mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-19 22:50:55 +08:00
fix: enforce archive encryption and clean failed S3 uploads
This commit is contained in:
@@ -71,7 +71,7 @@ jobs:
|
||||
ARTEX_PG_DSN: postgres://artex:testpass@localhost:5432/artextest_race?sslmode=disable
|
||||
|
||||
run: >-
|
||||
go test -race -p 1 ./agent ./cmd/artex ./config ./db ./evidence ./guard
|
||||
go test -race -p 1 ./agent ./archivestore ./cmd/artex ./config ./db ./evidence ./guard
|
||||
./intercept ./llmpool ./llmrec ./mcphttp ./notify ./server ./sidequestion
|
||||
./traffic ./traffic/rewrite
|
||||
|
||||
|
||||
@@ -76,6 +76,13 @@ func TestArchiveEncryptionConfiguration(t *testing.T) {
|
||||
t.Fatal("KMS key accepted with SSE-S3")
|
||||
}
|
||||
c.SSE = "aws:kms"
|
||||
for _, key := range []string{"alias/archive", "bare-key-id", "arn:aws:kms:us-east-1:123456789012:alias/archive"} {
|
||||
c.KMSKeyID = key
|
||||
if _, err := New(c); err == nil {
|
||||
t.Fatalf("accepted noncanonical KMS key %q", key)
|
||||
}
|
||||
}
|
||||
c.KMSKeyID = "arn:aws:kms:us-east-1:123456789012:key/test-key"
|
||||
if _, err := New(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
+69
-20
@@ -17,6 +17,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/arn"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/retry"
|
||||
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
@@ -93,24 +96,37 @@ func New(c Config) (*Store, error) {
|
||||
if c.KMSKeyID != "" && c.SSE != "aws:kms" {
|
||||
return nil, errors.New("archive S3 KMS key requires SSE=aws:kms")
|
||||
}
|
||||
if c.KMSKeyID != "" {
|
||||
key, err := arn.Parse(c.KMSKeyID)
|
||||
if err != nil || key.Service != "kms" || key.Region == "" || key.AccountID == "" ||
|
||||
!strings.HasPrefix(key.Resource, "key/") || len(key.Resource) <= len("key/") {
|
||||
return nil, errors.New("archive S3 KMS key must be a full key ARN, not an alias or bare key ID")
|
||||
}
|
||||
}
|
||||
if c.Region == "" {
|
||||
c.Region = "us-east-1"
|
||||
}
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.ResponseHeaderTimeout = 60 * time.Second
|
||||
httpClient := &http.Client{Transport: transport}
|
||||
cfg := aws.Config{Region: c.Region, HTTPClient: httpClient}
|
||||
// Load shared AWS settings (including custom CAs) for both credential modes.
|
||||
httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(t *http.Transport) {
|
||||
t.ResponseHeaderTimeout = 60 * time.Second
|
||||
})
|
||||
options := []func(*awsconfig.LoadOptions) error{
|
||||
awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient),
|
||||
}
|
||||
if explicitCredentials {
|
||||
cfg.Credentials = credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken)
|
||||
} else {
|
||||
// Keep the SDK's refreshable default chain: standard AWS environment/shared
|
||||
// profiles, web identity, ECS task roles and EC2 instance roles (IMDSv2).
|
||||
var err error
|
||||
cfg, err = awsconfig.LoadDefaultConfig(context.Background(),
|
||||
awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err)
|
||||
}
|
||||
options = append(options, awsconfig.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken)))
|
||||
}
|
||||
cfg, err := awsconfig.LoadDefaultConfig(context.Background(), options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err)
|
||||
}
|
||||
// The SDK's default client still follows 307/308. Use its CA-configured
|
||||
// transport, but never redirect a signed archive request to another origin.
|
||||
configuredClient := cfg.HTTPClient.(*awshttp.BuildableClient)
|
||||
cfg.HTTPClient = &http.Client{
|
||||
Transport: configuredClient.GetTransport(), Timeout: configuredClient.GetTimeout(),
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
}
|
||||
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(c.Endpoint)
|
||||
@@ -166,6 +182,7 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro
|
||||
uploader := manager.NewUploader(s.client, func(u *manager.Uploader) {
|
||||
u.PartSize = 8 << 20
|
||||
u.Concurrency = 2
|
||||
u.LeavePartsOnError = true // Abort below with a fresh, bounded cleanup context.
|
||||
u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
|
||||
})
|
||||
input := &s3.PutObjectInput{
|
||||
@@ -176,11 +193,36 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro
|
||||
if s.kmsKeyID != "" {
|
||||
input.SSEKMSKeyId = aws.String(s.kmsKeyID)
|
||||
}
|
||||
_, err = uploader.Upload(ctx, input)
|
||||
result, err := uploader.Upload(ctx, input)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload archive: %w", err)
|
||||
uploadErr := fmt.Errorf("upload archive: %w", err)
|
||||
var multipart manager.MultiUploadFailure
|
||||
if errors.As(err, &multipart) {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
defer cancel()
|
||||
_, cleanupErr := s.client.AbortMultipartUpload(cleanupCtx, &s3.AbortMultipartUploadInput{
|
||||
Bucket: aws.String(s.bucket), Key: aws.String(key), UploadId: aws.String(multipart.UploadID()),
|
||||
})
|
||||
if cleanupErr != nil {
|
||||
return errors.Join(uploadErr, fmt.Errorf("abort archive multipart upload: %w", cleanupErr))
|
||||
}
|
||||
}
|
||||
return uploadErr
|
||||
}
|
||||
return s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true)
|
||||
if err := s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true); err != nil {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
defer cancel()
|
||||
// Versioned buckets require deleting the version just uploaded; an
|
||||
// unversioned DELETE would only hide it behind a delete marker.
|
||||
_, cleanupErr := s.client.DeleteObject(cleanupCtx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket), Key: aws.String(key), VersionId: result.VersionID,
|
||||
})
|
||||
if cleanupErr != nil {
|
||||
return errors.Join(err, fmt.Errorf("delete unverified archive: %w", cleanupErr))
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64, requireEncryption bool) error {
|
||||
@@ -195,7 +237,8 @@ func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, che
|
||||
return fmt.Errorf("inspect archive: %w", err)
|
||||
}
|
||||
if requireEncryption && (object.ServerSideEncryption != s.encryption ||
|
||||
(s.encryption == types.ServerSideEncryptionAwsKms && aws.ToString(object.SSEKMSKeyId) == "")) {
|
||||
(s.encryption == types.ServerSideEncryptionAwsKms && (aws.ToString(object.SSEKMSKeyId) == "" ||
|
||||
(s.kmsKeyID != "" && aws.ToString(object.SSEKMSKeyId) != s.kmsKeyID)))) {
|
||||
return errors.New("remote archive did not confirm the requested server-side encryption; source data retained")
|
||||
}
|
||||
if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" {
|
||||
@@ -265,11 +308,17 @@ func (s *Store) readArchiveChunkOnce(ctx context.Context, key, etag string, star
|
||||
object, err := s.client.GetObject(requestCtx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket), Key: aws.String(key), IfMatch: aws.String(etag),
|
||||
Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)),
|
||||
}, func(o *s3.Options) {
|
||||
// This loop owns both HTTP and interrupted-body retries: at most five
|
||||
// wire requests per chunk, rather than five times the SDK's attempts.
|
||||
o.RetryMaxAttempts = 1
|
||||
})
|
||||
if err != nil {
|
||||
var networkError net.Error
|
||||
retry := errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded)
|
||||
return nil, retry, fmt.Errorf("read archive range: %w", err)
|
||||
var responseError *awshttp.ResponseError
|
||||
throttled := errors.As(err, &responseError) && responseError.HTTPStatusCode() == http.StatusTooManyRequests
|
||||
shouldRetry := throttled || retry.NewStandard().IsErrorRetryable(err) || errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded)
|
||||
return nil, shouldRetry, fmt.Errorf("read archive range: %w", err)
|
||||
}
|
||||
defer object.Body.Close()
|
||||
want := end - start + 1
|
||||
|
||||
@@ -340,6 +340,8 @@ func TestS3UploadRejectsUnconfirmedEncryption(t *testing.T) {
|
||||
switch r.Method {
|
||||
case "PUT":
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
case "DELETE":
|
||||
w.WriteHeader(204)
|
||||
case "HEAD":
|
||||
w.Header().Set("ETag", `"test-etag"`)
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
package archivestore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func archiveSource(t *testing.T, data []byte) (string, string) {
|
||||
t.Helper()
|
||||
filename := filepath.Join(t.TempDir(), "source")
|
||||
if err := os.WriteFile(filename, data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return filename, hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func TestS3VerificationFailureCleansUploadedVersion(t *testing.T) {
|
||||
for _, failure := range []string{"wrong-kms", "checksum", "cancelled", "delete-denied"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
data := []byte("private archive")
|
||||
source, checksum := archiveSource(t, data)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
var present, deleted atomic.Bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case "PUT":
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
present.Store(true)
|
||||
w.Header().Set("X-Amz-Version-Id", "uploaded-version")
|
||||
case "HEAD":
|
||||
w.Header().Set("ETag", `"test-etag"`)
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
|
||||
w.Header().Set("X-Amz-Server-Side-Encryption", "aws:kms")
|
||||
kms := "arn:aws:kms:us-east-1:123456789012:key/expected"
|
||||
if failure == "wrong-kms" {
|
||||
kms = "arn:aws:kms:us-east-1:123456789012:key/different"
|
||||
}
|
||||
w.Header().Set("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id", kms)
|
||||
if failure == "cancelled" {
|
||||
cancel()
|
||||
}
|
||||
case "GET":
|
||||
corrupt := bytes.Clone(data)
|
||||
corrupt[0] ^= 0xff
|
||||
serveArchiveRange(t, w, r, corrupt)
|
||||
case "DELETE":
|
||||
if r.URL.Query().Get("versionId") != "uploaded-version" {
|
||||
t.Error("cleanup did not target uploaded version")
|
||||
}
|
||||
deleted.Store(true)
|
||||
if failure == "delete-denied" {
|
||||
w.WriteHeader(403)
|
||||
return
|
||||
}
|
||||
present.Store(false)
|
||||
w.WriteHeader(204)
|
||||
default:
|
||||
t.Errorf("unexpected request %s", r.Method)
|
||||
w.WriteHeader(400)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
store, err := New(Config{Endpoint: srv.URL, Bucket: "test", AccessKey: "test", SecretKey: "test", PathStyle: true,
|
||||
SSE: "aws:kms", KMSKeyID: "arn:aws:kms:us-east-1:123456789012:key/expected"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = store.Upload(ctx, store.URI(1, 2, checksum), source, checksum)
|
||||
if err == nil {
|
||||
t.Fatal("accepted failed verification")
|
||||
}
|
||||
if failure == "wrong-kms" && !strings.Contains(err.Error(), "encryption") {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if failure == "cancelled" && !errors.Is(err, context.Canceled) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if failure == "delete-denied" && (!strings.Contains(err.Error(), "SHA-256") || !strings.Contains(err.Error(), "delete unverified archive")) {
|
||||
t.Fatalf("verification and cleanup errors must both survive: %v", err)
|
||||
}
|
||||
if !deleted.Load() || present.Load() != (failure == "delete-denied") {
|
||||
t.Fatal("unverified object cleanup was not attempted/completed")
|
||||
}
|
||||
if got, readErr := os.ReadFile(source); readErr != nil || !bytes.Equal(got, data) {
|
||||
t.Fatal("source changed", readErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3CancelledMultipartAbortsWithFreshContext(t *testing.T) {
|
||||
for _, denyAbort := range []bool{false, true} {
|
||||
t.Run(strconv.FormatBool(denyAbort), func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
var aborted atomic.Bool
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
switch {
|
||||
case r.Method == "POST" && q.Has("uploads"):
|
||||
fmt.Fprint(w, `<InitiateMultipartUploadResult><UploadId>cancelled-upload</UploadId></InitiateMultipartUploadResult>`)
|
||||
case r.Method == "PUT" && q.Has("partNumber"):
|
||||
cancel()
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
w.WriteHeader(403)
|
||||
case r.Method == "DELETE" && q.Get("uploadId") == "cancelled-upload":
|
||||
aborted.Store(true)
|
||||
if denyAbort {
|
||||
w.WriteHeader(403)
|
||||
} else {
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
default:
|
||||
t.Errorf("unexpected request %s", r.Method)
|
||||
w.WriteHeader(400)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
data := bytes.Repeat([]byte("x"), 9<<20)
|
||||
source, sum := archiveSource(t, data)
|
||||
store := testRangeStore(t, srv.URL)
|
||||
err := store.Upload(ctx, store.URI(1, 2, sum), source, sum)
|
||||
if !errors.Is(err, context.Canceled) || !aborted.Load() {
|
||||
t.Fatalf("cancelled upload not aborted: %v", err)
|
||||
}
|
||||
if denyAbort && !strings.Contains(err.Error(), "abort archive multipart upload") {
|
||||
t.Fatalf("cleanup failure lost: %v", err)
|
||||
}
|
||||
if got, readErr := os.ReadFile(source); readErr != nil || !bytes.Equal(got, data) {
|
||||
t.Fatal("source changed", readErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3SignedRequestsNeverFollowRedirects(t *testing.T) {
|
||||
var reached atomic.Int32
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reached.Add(1); w.WriteHeader(204) }))
|
||||
defer target.Close()
|
||||
for _, status := range []int{301, 302, 303, 307, 308} {
|
||||
t.Run(strconv.Itoa(status), func(t *testing.T) {
|
||||
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") == "" || r.Header.Get("X-Amz-Security-Token") != "fixture-session" {
|
||||
t.Error("fixture request must be signed with session credentials")
|
||||
}
|
||||
w.Header().Set("Location", target.URL+"/redirected")
|
||||
w.WriteHeader(status)
|
||||
}))
|
||||
defer origin.Close()
|
||||
store, err := New(Config{Endpoint: origin.URL, Bucket: "test", AccessKey: "test", SecretKey: "test", SessionToken: "fixture-session", PathStyle: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Delete(context.Background(), store.URI(1, 2, "test")); err == nil {
|
||||
t.Fatal("redirect accepted")
|
||||
}
|
||||
if reached.Load() != 0 {
|
||||
t.Fatal("signed request followed redirect")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3CustomCABundleForBothCredentialModes(t *testing.T) {
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) }))
|
||||
defer srv.Close()
|
||||
for _, explicit := range []bool{false, true} {
|
||||
t.Run(strconv.FormatBool(explicit), func(t *testing.T) {
|
||||
isolateAWSCredentials(t)
|
||||
ca := filepath.Join(t.TempDir(), "ca.pem")
|
||||
if err := os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("AWS_CA_BUNDLE", ca)
|
||||
c := Config{Endpoint: srv.URL, Bucket: "test", PathStyle: true}
|
||||
if explicit {
|
||||
c.AccessKey, c.SecretKey = "fixture", "fixture"
|
||||
} else {
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "fixture")
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture")
|
||||
}
|
||||
store, err := New(c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Delete(context.Background(), store.URI(1, 2, "test")); err != nil {
|
||||
t.Fatalf("configured CA not used: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3RangeRetriesHaveOneWireAttemptEach(t *testing.T) {
|
||||
for _, status := range []int{403, 429, 503} {
|
||||
t.Run(strconv.Itoa(status), func(t *testing.T) {
|
||||
var gets atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gets.Add(1)
|
||||
w.WriteHeader(status)
|
||||
}))
|
||||
defer srv.Close()
|
||||
store := testRangeStore(t, srv.URL)
|
||||
if _, err := store.readArchiveChunk(context.Background(), "key", `"etag"`, 0, 9, 10); err == nil {
|
||||
t.Fatal("failed request accepted")
|
||||
}
|
||||
want := int32(archiveReadAttempts)
|
||||
if status == 403 {
|
||||
want = 1
|
||||
}
|
||||
if got := gets.Load(); got != want {
|
||||
t.Fatalf("wire requests=%d want=%d", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user