diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f949165..2fac967 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: ARTEX_PG_DSN: postgres://artex:testpass@localhost:5432/artextest_race?sslmode=disable run: >- - go test -race -p 1 ./agent ./cmd/artex ./config ./db ./evidence ./guard + go test -race -p 1 ./agent ./archivestore ./cmd/artex ./config ./db ./evidence ./guard ./intercept ./llmpool ./llmrec ./mcphttp ./notify ./server ./sidequestion ./traffic ./traffic/rewrite diff --git a/archivestore/config_test.go b/archivestore/config_test.go index 73afc73..1fe54a5 100644 --- a/archivestore/config_test.go +++ b/archivestore/config_test.go @@ -76,6 +76,13 @@ func TestArchiveEncryptionConfiguration(t *testing.T) { t.Fatal("KMS key accepted with SSE-S3") } c.SSE = "aws:kms" + for _, key := range []string{"alias/archive", "bare-key-id", "arn:aws:kms:us-east-1:123456789012:alias/archive"} { + c.KMSKeyID = key + if _, err := New(c); err == nil { + t.Fatalf("accepted noncanonical KMS key %q", key) + } + } + c.KMSKeyID = "arn:aws:kms:us-east-1:123456789012:key/test-key" if _, err := New(c); err != nil { t.Fatal(err) } diff --git a/archivestore/s3.go b/archivestore/s3.go index 27c50af..5761a83 100644 --- a/archivestore/s3.go +++ b/archivestore/s3.go @@ -17,6 +17,9 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/aws/aws-sdk-go-v2/aws/retry" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" @@ -93,24 +96,37 @@ func New(c Config) (*Store, error) { if c.KMSKeyID != "" && c.SSE != "aws:kms" { return nil, errors.New("archive S3 KMS key requires SSE=aws:kms") } + if c.KMSKeyID != "" { + key, err := arn.Parse(c.KMSKeyID) + if err != nil || key.Service != "kms" || key.Region == "" || key.AccountID == "" || + !strings.HasPrefix(key.Resource, "key/") || len(key.Resource) <= len("key/") { + return nil, errors.New("archive S3 KMS key must be a full key ARN, not an alias or bare key ID") + } + } if c.Region == "" { c.Region = "us-east-1" } - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.ResponseHeaderTimeout = 60 * time.Second - httpClient := &http.Client{Transport: transport} - cfg := aws.Config{Region: c.Region, HTTPClient: httpClient} + // Load shared AWS settings (including custom CAs) for both credential modes. + httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(t *http.Transport) { + t.ResponseHeaderTimeout = 60 * time.Second + }) + options := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient), + } if explicitCredentials { - cfg.Credentials = credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken) - } else { - // Keep the SDK's refreshable default chain: standard AWS environment/shared - // profiles, web identity, ECS task roles and EC2 instance roles (IMDSv2). - var err error - cfg, err = awsconfig.LoadDefaultConfig(context.Background(), - awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient)) - if err != nil { - return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err) - } + options = append(options, awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken))) + } + cfg, err := awsconfig.LoadDefaultConfig(context.Background(), options...) + if err != nil { + return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err) + } + // The SDK's default client still follows 307/308. Use its CA-configured + // transport, but never redirect a signed archive request to another origin. + configuredClient := cfg.HTTPClient.(*awshttp.BuildableClient) + cfg.HTTPClient = &http.Client{ + Transport: configuredClient.GetTransport(), Timeout: configuredClient.GetTimeout(), + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, } client := s3.NewFromConfig(cfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(c.Endpoint) @@ -166,6 +182,7 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro uploader := manager.NewUploader(s.client, func(u *manager.Uploader) { u.PartSize = 8 << 20 u.Concurrency = 2 + u.LeavePartsOnError = true // Abort below with a fresh, bounded cleanup context. u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired }) input := &s3.PutObjectInput{ @@ -176,11 +193,36 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro if s.kmsKeyID != "" { input.SSEKMSKeyId = aws.String(s.kmsKeyID) } - _, err = uploader.Upload(ctx, input) + result, err := uploader.Upload(ctx, input) if err != nil { - return fmt.Errorf("upload archive: %w", err) + uploadErr := fmt.Errorf("upload archive: %w", err) + var multipart manager.MultiUploadFailure + if errors.As(err, &multipart) { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _, cleanupErr := s.client.AbortMultipartUpload(cleanupCtx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(s.bucket), Key: aws.String(key), UploadId: aws.String(multipart.UploadID()), + }) + if cleanupErr != nil { + return errors.Join(uploadErr, fmt.Errorf("abort archive multipart upload: %w", cleanupErr)) + } + } + return uploadErr } - return s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true) + if err := s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true); err != nil { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + // Versioned buckets require deleting the version just uploaded; an + // unversioned DELETE would only hide it behind a delete marker. + _, cleanupErr := s.client.DeleteObject(cleanupCtx, &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), Key: aws.String(key), VersionId: result.VersionID, + }) + if cleanupErr != nil { + return errors.Join(err, fmt.Errorf("delete unverified archive: %w", cleanupErr)) + } + return err + } + return nil } func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64, requireEncryption bool) error { @@ -195,7 +237,8 @@ func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, che return fmt.Errorf("inspect archive: %w", err) } if requireEncryption && (object.ServerSideEncryption != s.encryption || - (s.encryption == types.ServerSideEncryptionAwsKms && aws.ToString(object.SSEKMSKeyId) == "")) { + (s.encryption == types.ServerSideEncryptionAwsKms && (aws.ToString(object.SSEKMSKeyId) == "" || + (s.kmsKeyID != "" && aws.ToString(object.SSEKMSKeyId) != s.kmsKeyID)))) { return errors.New("remote archive did not confirm the requested server-side encryption; source data retained") } if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" { @@ -265,11 +308,17 @@ func (s *Store) readArchiveChunkOnce(ctx context.Context, key, etag string, star object, err := s.client.GetObject(requestCtx, &s3.GetObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(key), IfMatch: aws.String(etag), Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)), + }, func(o *s3.Options) { + // This loop owns both HTTP and interrupted-body retries: at most five + // wire requests per chunk, rather than five times the SDK's attempts. + o.RetryMaxAttempts = 1 }) if err != nil { var networkError net.Error - retry := errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded) - return nil, retry, fmt.Errorf("read archive range: %w", err) + var responseError *awshttp.ResponseError + throttled := errors.As(err, &responseError) && responseError.HTTPStatusCode() == http.StatusTooManyRequests + shouldRetry := throttled || retry.NewStandard().IsErrorRetryable(err) || errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded) + return nil, shouldRetry, fmt.Errorf("read archive range: %w", err) } defer object.Body.Close() want := end - start + 1 diff --git a/archivestore/s3_test.go b/archivestore/s3_test.go index dc618c1..84049e3 100644 --- a/archivestore/s3_test.go +++ b/archivestore/s3_test.go @@ -340,6 +340,8 @@ func TestS3UploadRejectsUnconfirmedEncryption(t *testing.T) { switch r.Method { case "PUT": _, _ = io.Copy(io.Discard, r.Body) + case "DELETE": + w.WriteHeader(204) case "HEAD": w.Header().Set("ETag", `"test-etag"`) w.Header().Set("Content-Length", strconv.Itoa(len(data))) diff --git a/archivestore/safety_test.go b/archivestore/safety_test.go new file mode 100644 index 0000000..9afe47b --- /dev/null +++ b/archivestore/safety_test.go @@ -0,0 +1,231 @@ +package archivestore + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" +) + +func archiveSource(t *testing.T, data []byte) (string, string) { + t.Helper() + filename := filepath.Join(t.TempDir(), "source") + if err := os.WriteFile(filename, data, 0600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(data) + return filename, hex.EncodeToString(sum[:]) +} + +func TestS3VerificationFailureCleansUploadedVersion(t *testing.T) { + for _, failure := range []string{"wrong-kms", "checksum", "cancelled", "delete-denied"} { + t.Run(failure, func(t *testing.T) { + data := []byte("private archive") + source, checksum := archiveSource(t, data) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var present, deleted atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "PUT": + _, _ = io.Copy(io.Discard, r.Body) + present.Store(true) + w.Header().Set("X-Amz-Version-Id", "uploaded-version") + case "HEAD": + w.Header().Set("ETag", `"test-etag"`) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + w.Header().Set("X-Amz-Server-Side-Encryption", "aws:kms") + kms := "arn:aws:kms:us-east-1:123456789012:key/expected" + if failure == "wrong-kms" { + kms = "arn:aws:kms:us-east-1:123456789012:key/different" + } + w.Header().Set("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id", kms) + if failure == "cancelled" { + cancel() + } + case "GET": + corrupt := bytes.Clone(data) + corrupt[0] ^= 0xff + serveArchiveRange(t, w, r, corrupt) + case "DELETE": + if r.URL.Query().Get("versionId") != "uploaded-version" { + t.Error("cleanup did not target uploaded version") + } + deleted.Store(true) + if failure == "delete-denied" { + w.WriteHeader(403) + return + } + present.Store(false) + w.WriteHeader(204) + default: + t.Errorf("unexpected request %s", r.Method) + w.WriteHeader(400) + } + })) + defer srv.Close() + store, err := New(Config{Endpoint: srv.URL, Bucket: "test", AccessKey: "test", SecretKey: "test", PathStyle: true, + SSE: "aws:kms", KMSKeyID: "arn:aws:kms:us-east-1:123456789012:key/expected"}) + if err != nil { + t.Fatal(err) + } + err = store.Upload(ctx, store.URI(1, 2, checksum), source, checksum) + if err == nil { + t.Fatal("accepted failed verification") + } + if failure == "wrong-kms" && !strings.Contains(err.Error(), "encryption") { + t.Fatal(err) + } + if failure == "cancelled" && !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if failure == "delete-denied" && (!strings.Contains(err.Error(), "SHA-256") || !strings.Contains(err.Error(), "delete unverified archive")) { + t.Fatalf("verification and cleanup errors must both survive: %v", err) + } + if !deleted.Load() || present.Load() != (failure == "delete-denied") { + t.Fatal("unverified object cleanup was not attempted/completed") + } + if got, readErr := os.ReadFile(source); readErr != nil || !bytes.Equal(got, data) { + t.Fatal("source changed", readErr) + } + }) + } +} + +func TestS3CancelledMultipartAbortsWithFreshContext(t *testing.T) { + for _, denyAbort := range []bool{false, true} { + t.Run(strconv.FormatBool(denyAbort), func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var aborted atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case r.Method == "POST" && q.Has("uploads"): + fmt.Fprint(w, `cancelled-upload`) + case r.Method == "PUT" && q.Has("partNumber"): + cancel() + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(403) + case r.Method == "DELETE" && q.Get("uploadId") == "cancelled-upload": + aborted.Store(true) + if denyAbort { + w.WriteHeader(403) + } else { + w.WriteHeader(204) + } + default: + t.Errorf("unexpected request %s", r.Method) + w.WriteHeader(400) + } + })) + defer srv.Close() + data := bytes.Repeat([]byte("x"), 9<<20) + source, sum := archiveSource(t, data) + store := testRangeStore(t, srv.URL) + err := store.Upload(ctx, store.URI(1, 2, sum), source, sum) + if !errors.Is(err, context.Canceled) || !aborted.Load() { + t.Fatalf("cancelled upload not aborted: %v", err) + } + if denyAbort && !strings.Contains(err.Error(), "abort archive multipart upload") { + t.Fatalf("cleanup failure lost: %v", err) + } + if got, readErr := os.ReadFile(source); readErr != nil || !bytes.Equal(got, data) { + t.Fatal("source changed", readErr) + } + }) + } +} + +func TestS3SignedRequestsNeverFollowRedirects(t *testing.T) { + var reached atomic.Int32 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reached.Add(1); w.WriteHeader(204) })) + defer target.Close() + for _, status := range []int{301, 302, 303, 307, 308} { + t.Run(strconv.Itoa(status), func(t *testing.T) { + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") == "" || r.Header.Get("X-Amz-Security-Token") != "fixture-session" { + t.Error("fixture request must be signed with session credentials") + } + w.Header().Set("Location", target.URL+"/redirected") + w.WriteHeader(status) + })) + defer origin.Close() + store, err := New(Config{Endpoint: origin.URL, Bucket: "test", AccessKey: "test", SecretKey: "test", SessionToken: "fixture-session", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + if err := store.Delete(context.Background(), store.URI(1, 2, "test")); err == nil { + t.Fatal("redirect accepted") + } + if reached.Load() != 0 { + t.Fatal("signed request followed redirect") + } + }) + } +} + +func TestS3CustomCABundleForBothCredentialModes(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) })) + defer srv.Close() + for _, explicit := range []bool{false, true} { + t.Run(strconv.FormatBool(explicit), func(t *testing.T) { + isolateAWSCredentials(t) + ca := filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("AWS_CA_BUNDLE", ca) + c := Config{Endpoint: srv.URL, Bucket: "test", PathStyle: true} + if explicit { + c.AccessKey, c.SecretKey = "fixture", "fixture" + } else { + t.Setenv("AWS_ACCESS_KEY_ID", "fixture") + t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture") + } + store, err := New(c) + if err != nil { + t.Fatal(err) + } + if err := store.Delete(context.Background(), store.URI(1, 2, "test")); err != nil { + t.Fatalf("configured CA not used: %v", err) + } + }) + } +} + +func TestS3RangeRetriesHaveOneWireAttemptEach(t *testing.T) { + for _, status := range []int{403, 429, 503} { + t.Run(strconv.Itoa(status), func(t *testing.T) { + var gets atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gets.Add(1) + w.WriteHeader(status) + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + if _, err := store.readArchiveChunk(context.Background(), "key", `"etag"`, 0, 9, 10); err == nil { + t.Fatal("failed request accepted") + } + want := int32(archiveReadAttempts) + if status == 403 { + want = 1 + } + if got := gets.Load(); got != want { + t.Fatalf("wire requests=%d want=%d", got, want) + } + }) + } +}