diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f949165..2fac967 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: ARTEX_PG_DSN: postgres://artex:testpass@localhost:5432/artextest_race?sslmode=disable run: >- - go test -race -p 1 ./agent ./cmd/artex ./config ./db ./evidence ./guard + go test -race -p 1 ./agent ./archivestore ./cmd/artex ./config ./db ./evidence ./guard ./intercept ./llmpool ./llmrec ./mcphttp ./notify ./server ./sidequestion ./traffic ./traffic/rewrite diff --git a/archivestore/config_test.go b/archivestore/config_test.go new file mode 100644 index 0000000..1fe54a5 --- /dev/null +++ b/archivestore/config_test.go @@ -0,0 +1,89 @@ +package archivestore + +import ( + "strings" + "testing" +) + +func TestArchiveStorageEnvironment(t *testing.T) { + for _, key := range []string{ + "ARTEX_ARCHIVE_STORAGE", "ARTEX_ARCHIVE_S3_ENDPOINT", "ARTEX_ARCHIVE_S3_REGION", + "ARTEX_ARCHIVE_S3_BUCKET", "ARTEX_ARCHIVE_S3_PREFIX", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", + "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_PATH_STYLE", + "ARTEX_ARCHIVE_S3_SESSION_TOKEN", "ARTEX_ARCHIVE_S3_SSE", "ARTEX_ARCHIVE_S3_KMS_KEY_ID", + } { + t.Setenv(key, "") + } + for _, backend := range []string{"", "local"} { + t.Setenv("ARTEX_ARCHIVE_STORAGE", backend) + if store, err := FromEnv(); err != nil || store != nil { + t.Fatalf("local default: %v %v", store, err) + } + } + for _, backend := range []string{"unknown", "s3"} { + t.Setenv("ARTEX_ARCHIVE_STORAGE", backend) + if _, err := FromEnv(); err == nil { + t.Fatalf("%q silently fell back to local", backend) + } + } + t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", "http://127.0.0.1:9000") + t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "test-bucket") + t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "test-key") + t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "test-secret") + t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true") + store, err := FromEnv() + if err != nil { + t.Fatal(err) + } + if uri := store.URI(1, 2, "checksum"); uri != "s3://test-bucket/artex/tasks/task-1-2-checksum.tar.zst" { + t.Fatal("unexpected default object location", uri) + } + t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "") + if _, err := FromEnv(); err == nil { + t.Fatal("missing credentials accepted") + } +} + +func TestArchiveEndpointValidation(t *testing.T) { + config := Config{Bucket: "test", AccessKey: "test-key", SecretKey: "test-secret"} + for _, endpoint := range []string{"https://objects.example.test", "http://127.0.0.1:9000", "http://127.0.0.2:9000", "http://[::1]:9000", "http://localhost:9000"} { + config.Endpoint = endpoint + if _, err := New(config); err != nil { + t.Errorf("rejected %s: %v", endpoint, err) + } + } + for _, endpoint := range []string{"http://objects.example.test", "https://user:test-secret@objects.example.test", "https://objects.example.test/bucket", "https://objects.example.test?token=test-secret", "https://objects.example.test#fragment"} { + config.Endpoint = endpoint + if _, err := New(config); err == nil { + t.Error("accepted endpoint with non-loopback HTTP, credentials, path, query or fragment") + } else if strings.Contains(err.Error(), "test-secret") { + t.Fatal("configuration error disclosed a secret") + } + } +} + +func TestArchiveEncryptionConfiguration(t *testing.T) { + c := Config{Endpoint: "https://objects.example.test", Bucket: "test", AccessKey: "test", SecretKey: "test"} + for _, mode := range []string{"none", "invalid"} { + c.SSE = mode + if _, err := New(c); err == nil { + t.Fatal("unencrypted/invalid mode accepted") + } + } + c.SSE = "AES256" + c.KMSKeyID = "alias/archive" + if _, err := New(c); err == nil { + t.Fatal("KMS key accepted with SSE-S3") + } + c.SSE = "aws:kms" + for _, key := range []string{"alias/archive", "bare-key-id", "arn:aws:kms:us-east-1:123456789012:alias/archive"} { + c.KMSKeyID = key + if _, err := New(c); err == nil { + t.Fatalf("accepted noncanonical KMS key %q", key) + } + } + c.KMSKeyID = "arn:aws:kms:us-east-1:123456789012:key/test-key" + if _, err := New(c); err != nil { + t.Fatal(err) + } +} diff --git a/archivestore/credentials_test.go b/archivestore/credentials_test.go new file mode 100644 index 0000000..6c667ed --- /dev/null +++ b/archivestore/credentials_test.go @@ -0,0 +1,132 @@ +package archivestore + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func isolateAWSCredentials(t *testing.T) { + t.Helper() + for _, key := range []string{ + "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_PROFILE", "AWS_DEFAULT_PROFILE", + "AWS_ROLE_ARN", "AWS_ROLE_SESSION_NAME", "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", + "AWS_EC2_METADATA_SERVICE_ENDPOINT", "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE", + } { + t.Setenv(key, "") + } + dir := t.TempDir() + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(dir, "credentials")) + t.Setenv("AWS_CONFIG_FILE", filepath.Join(dir, "config")) + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") +} + +func TestArchiveCredentialChain(t *testing.T) { + config := Config{Endpoint: "https://objects.example.test", Bucket: "test"} + for _, mode := range []string{"explicit-session", "environment", "shared-profile", "ecs", "imds"} { + t.Run(mode, func(t *testing.T) { + isolateAWSCredentials(t) + c := config + switch mode { + case "explicit-session": + t.Setenv("AWS_ACCESS_KEY_ID", "other-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "other-secret") + c.AccessKey, c.SecretKey, c.SessionToken = "fixture-key", "fixture-secret", "fixture-session" + case "environment": + t.Setenv("AWS_ACCESS_KEY_ID", "fixture-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret") + t.Setenv("AWS_SESSION_TOKEN", "fixture-session") + case "shared-profile": + t.Setenv("AWS_PROFILE", "archive") + data := `[archive] +aws_access_key_id=fixture-key +aws_secret_access_key=fixture-secret +aws_session_token=fixture-session +` + if err := os.WriteFile(os.Getenv("AWS_SHARED_CREDENTIALS_FILE"), []byte(data), 0600); err != nil { + t.Fatal(err) + } + case "ecs", "imds": + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if mode == "imds" { + switch r.URL.Path { + case "/latest/api/token": + if r.Method != "PUT" { + t.Error("IMDS did not use token request") + } + w.Header().Set("X-aws-ec2-metadata-token-ttl-seconds", "21600") + fmt.Fprint(w, "fixture-imds-token") + return + case "/latest/meta-data/iam/security-credentials/": + if r.Header.Get("X-aws-ec2-metadata-token") != "fixture-imds-token" { + t.Error("missing IMDSv2 token") + } + fmt.Fprint(w, "archive-role") + return + case "/latest/meta-data/iam/security-credentials/archive-role": + if r.Header.Get("X-aws-ec2-metadata-token") != "fixture-imds-token" { + t.Error("missing IMDSv2 token") + } + default: + t.Errorf("unexpected IMDS request %s", r.URL.Path) + w.WriteHeader(404) + return + } + } else if r.Header.Get("Authorization") != "fixture-container-token" { + t.Error("missing ECS authorization token") + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "Code": "Success", "AccessKeyId": "fixture-key", "SecretAccessKey": "fixture-secret", + "Token": "fixture-session", "Expiration": time.Now().Add(time.Hour).UTC().Format(time.RFC3339), + }) + })) + defer srv.Close() + if mode == "imds" { + t.Setenv("AWS_EC2_METADATA_DISABLED", "false") + t.Setenv("AWS_EC2_METADATA_V1_DISABLED", "true") + t.Setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", srv.URL) + } else { + t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", srv.URL+"/credentials") + t.Setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "fixture-container-token") + } + } + store, err := New(c) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + credentials, err := store.client.Options().Credentials.Retrieve(ctx) + if err != nil { + t.Fatal(err) + } + if credentials.AccessKeyID != "fixture-key" || credentials.SecretAccessKey != "fixture-secret" || credentials.SessionToken != "fixture-session" { + t.Fatal("wrong credential provider or session token") + } + if (mode == "imds" || mode == "ecs") && !credentials.CanExpire { + t.Fatal("role credentials must remain refreshable") + } + }) + } +} + +func TestArchiveIncompleteCredentialOverride(t *testing.T) { + isolateAWSCredentials(t) + t.Setenv("AWS_ACCESS_KEY_ID", "valid-ambient-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "valid-ambient-secret") + for _, c := range []Config{{AccessKey: "private-value"}, {SecretKey: "private-value"}, {SessionToken: "private-value"}} { + c.Endpoint, c.Bucket = "https://objects.example.test", "test" + if _, err := New(c); err == nil || strings.Contains(err.Error(), "private-value") { + t.Fatalf("partial override must fail without disclosing credentials: %v", err) + } + } +} diff --git a/archivestore/s3.go b/archivestore/s3.go new file mode 100644 index 0000000..5761a83 --- /dev/null +++ b/archivestore/s3.go @@ -0,0 +1,372 @@ +// Package archivestore stores immutable cold-task packages in S3-compatible storage. +package archivestore + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/aws/aws-sdk-go-v2/aws/retry" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +type Config struct { + Endpoint, Region, Bucket, Prefix, AccessKey, SecretKey string + PathStyle bool + SessionToken, SSE, KMSKeyID string +} + +type Store struct { + client *s3.Client + bucket, prefix string + encryption types.ServerSideEncryption + kmsKeyID string +} + +// FromEnv returns nil for the default local backend. A configured but invalid +// backend is an error, never a silent fallback to the machine's local disk. +func FromEnv() (*Store, error) { + switch os.Getenv("ARTEX_ARCHIVE_STORAGE") { + case "", "local": + return nil, nil + case "s3": + return New(Config{ + Endpoint: os.Getenv("ARTEX_ARCHIVE_S3_ENDPOINT"), Region: os.Getenv("ARTEX_ARCHIVE_S3_REGION"), + Bucket: os.Getenv("ARTEX_ARCHIVE_S3_BUCKET"), Prefix: os.Getenv("ARTEX_ARCHIVE_S3_PREFIX"), + AccessKey: os.Getenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID"), SecretKey: os.Getenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY"), + PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true", + SessionToken: os.Getenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN"), + SSE: os.Getenv("ARTEX_ARCHIVE_S3_SSE"), KMSKeyID: os.Getenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID"), + }) + default: + return nil, errors.New("ARTEX_ARCHIVE_STORAGE must be local or s3") + } +} + +func New(c Config) (*Store, error) { + u, err := url.Parse(c.Endpoint) + loopback := false + if err == nil { + loopback = u.Hostname() == "localhost" + if ip := net.ParseIP(u.Hostname()); ip != nil { + loopback = ip.IsLoopback() + } + } + if err != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") || + (u.Scheme != "https" && !(u.Scheme == "http" && loopback)) { + return nil, errors.New("archive S3 endpoint must be an HTTPS origin (HTTP allowed only on loopback)") + } + c.Prefix = strings.Trim(c.Prefix, "/") + if c.Prefix == "" { + c.Prefix = "artex/tasks" + } + if path.Clean(c.Prefix) != c.Prefix || strings.Contains(c.Prefix, "..") || strings.ContainsAny(c.Prefix, "\\?#") { + return nil, errors.New("invalid archive S3 prefix") + } + if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") { + return nil, errors.New("a valid archive S3 bucket is required") + } + explicitCredentials := c.AccessKey != "" || c.SecretKey != "" || c.SessionToken != "" + if explicitCredentials && (c.AccessKey == "" || c.SecretKey == "") { + return nil, errors.New("archive S3 access key and secret key must both be set when overriding AWS credentials") + } + if c.SSE == "" { + c.SSE = "AES256" + } + if c.SSE != "AES256" && c.SSE != "aws:kms" { + return nil, errors.New("archive S3 SSE must be AES256 or aws:kms; encryption cannot be disabled") + } + if c.KMSKeyID != "" && c.SSE != "aws:kms" { + return nil, errors.New("archive S3 KMS key requires SSE=aws:kms") + } + if c.KMSKeyID != "" { + key, err := arn.Parse(c.KMSKeyID) + if err != nil || key.Service != "kms" || key.Region == "" || key.AccountID == "" || + !strings.HasPrefix(key.Resource, "key/") || len(key.Resource) <= len("key/") { + return nil, errors.New("archive S3 KMS key must be a full key ARN, not an alias or bare key ID") + } + } + if c.Region == "" { + c.Region = "us-east-1" + } + // Load shared AWS settings (including custom CAs) for both credential modes. + httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(t *http.Transport) { + t.ResponseHeaderTimeout = 60 * time.Second + }) + options := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient), + } + if explicitCredentials { + options = append(options, awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken))) + } + cfg, err := awsconfig.LoadDefaultConfig(context.Background(), options...) + if err != nil { + return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err) + } + // The SDK's default client still follows 307/308. Use its CA-configured + // transport, but never redirect a signed archive request to another origin. + configuredClient := cfg.HTTPClient.(*awshttp.BuildableClient) + cfg.HTTPClient = &http.Client{ + Transport: configuredClient.GetTransport(), Timeout: configuredClient.GetTimeout(), + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + } + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(c.Endpoint) + o.UsePathStyle = c.PathStyle + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + }) + return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/", + encryption: types.ServerSideEncryption(c.SSE), kmsKeyID: c.KMSKeyID}, nil +} + +func (s *Store) URI(taskID, archiveID int64, checksum string) string { + return "s3://" + s.bucket + "/" + s.prefix + fmt.Sprintf("task-%d-%d-%s.tar.zst", taskID, archiveID, checksum) +} + +func (s *Store) key(uri string) (string, error) { + if s == nil { + return "", errors.New("S3 archive storage is not configured") + } + u, err := url.Parse(uri) + if err != nil || u.Scheme != "s3" || u.Host != s.bucket || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return "", errors.New("archive object is outside the configured bucket") + } + key := strings.TrimPrefix(u.Path, "/") + if !strings.HasPrefix(key, s.prefix) || len(key) <= len(s.prefix) || path.Clean(key) != key || strings.Contains(key, "\\") { + return "", errors.New("archive object is outside the configured prefix") + } + return key, nil +} + +func (s *Store) Validate(uri string) error { _, err := s.key(uri); return err } + +// Upload verifies the entire uploaded object by reading it back, including +// multipart uploads whose ETag is not a checksum. Callers retain source data +// until verification succeeds and PostgreSQL commits the remote location. +func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) error { + key, err := s.key(uri) + if err != nil { + return err + } + if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size { + return errors.New("invalid archive SHA-256") + } + f, err := os.Open(filename) + if err != nil { + return err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return err + } + uploader := manager.NewUploader(s.client, func(u *manager.Uploader) { + u.PartSize = 8 << 20 + u.Concurrency = 2 + u.LeavePartsOnError = true // Abort below with a fresh, bounded cleanup context. + u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + }) + input := &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), Key: aws.String(key), Body: f, + ContentType: aws.String("application/zstd"), Metadata: map[string]string{"sha256": checksum}, + ServerSideEncryption: s.encryption, + } + if s.kmsKeyID != "" { + input.SSEKMSKeyId = aws.String(s.kmsKeyID) + } + result, err := uploader.Upload(ctx, input) + if err != nil { + uploadErr := fmt.Errorf("upload archive: %w", err) + var multipart manager.MultiUploadFailure + if errors.As(err, &multipart) { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _, cleanupErr := s.client.AbortMultipartUpload(cleanupCtx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(s.bucket), Key: aws.String(key), UploadId: aws.String(multipart.UploadID()), + }) + if cleanupErr != nil { + return errors.Join(uploadErr, fmt.Errorf("abort archive multipart upload: %w", cleanupErr)) + } + } + return uploadErr + } + if err := s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true); err != nil { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + // Versioned buckets require deleting the version just uploaded; an + // unversioned DELETE would only hide it behind a delete marker. + _, cleanupErr := s.client.DeleteObject(cleanupCtx, &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), Key: aws.String(key), VersionId: result.VersionID, + }) + if cleanupErr != nil { + return errors.Join(err, fmt.Errorf("delete unverified archive: %w", cleanupErr)) + } + return err + } + return nil +} + +func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64, requireEncryption bool) error { + if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size { + return errors.New("invalid archive SHA-256") + } + if expectedSize < 0 { + return errors.New("invalid archive size") + } + object, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + if err != nil { + return fmt.Errorf("inspect archive: %w", err) + } + if requireEncryption && (object.ServerSideEncryption != s.encryption || + (s.encryption == types.ServerSideEncryptionAwsKms && (aws.ToString(object.SSEKMSKeyId) == "" || + (s.kmsKeyID != "" && aws.ToString(object.SSEKMSKeyId) != s.kmsKeyID)))) { + return errors.New("remote archive did not confirm the requested server-side encryption; source data retained") + } + if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" { + return errors.New("remote archive size mismatch or missing ETag; source data retained") + } + // Commit only complete, validated chunks to the output and hash. A broken + // connection can then retry its chunk without duplicating bytes or restarting + // a multi-gigabyte read. IfMatch pins every request to the same object. + h := sha256.New() + for offset := int64(0); offset < expectedSize; { + end := offset + min(archiveReadChunkSize, expectedSize-offset) - 1 + data, err := s.readArchiveChunk(ctx, key, aws.ToString(object.ETag), offset, end, expectedSize) + if err != nil { + return err + } + n, err := out.Write(data) + if err != nil { + return err + } + if n != len(data) { + return io.ErrShortWrite + } + _, _ = h.Write(data) + offset += int64(len(data)) + } + if !strings.EqualFold(hex.EncodeToString(h.Sum(nil)), checksum) { + return errors.New("remote archive size or SHA-256 mismatch; source data retained") + } + return nil +} + +const ( + archiveReadChunkSize = int64(8 << 20) + archiveReadAttempts = 5 +) + +func (s *Store) readArchiveChunk(ctx context.Context, key, etag string, start, end, total int64) ([]byte, error) { + var lastErr error + for attempt := 0; attempt < archiveReadAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return nil, err + } + if attempt > 0 { + timer := time.NewTimer((250 * time.Millisecond) << (attempt - 1)) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + data, retry, err := s.readArchiveChunkOnce(ctx, key, etag, start, end, total) + if err == nil { + return data, nil + } + if !retry { + return nil, err + } + lastErr = err + } + return nil, fmt.Errorf("read archive bytes %d-%d after %d attempts: %w", start, end, archiveReadAttempts, lastErr) +} + +func (s *Store) readArchiveChunkOnce(ctx context.Context, key, etag string, start, end, total int64) ([]byte, bool, error) { + requestCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + object, err := s.client.GetObject(requestCtx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), Key: aws.String(key), IfMatch: aws.String(etag), + Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)), + }, func(o *s3.Options) { + // This loop owns both HTTP and interrupted-body retries: at most five + // wire requests per chunk, rather than five times the SDK's attempts. + o.RetryMaxAttempts = 1 + }) + if err != nil { + var networkError net.Error + var responseError *awshttp.ResponseError + throttled := errors.As(err, &responseError) && responseError.HTTPStatusCode() == http.StatusTooManyRequests + shouldRetry := throttled || retry.NewStandard().IsErrorRetryable(err) || errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded) + return nil, shouldRetry, fmt.Errorf("read archive range: %w", err) + } + defer object.Body.Close() + want := end - start + 1 + if aws.ToString(object.ETag) != etag || aws.ToString(object.ContentRange) != fmt.Sprintf("bytes %d-%d/%d", start, end, total) || object.ContentLength == nil || *object.ContentLength != want { + return nil, false, errors.New("remote archive changed or returned an invalid byte range; source data retained") + } + data, err := io.ReadAll(io.LimitReader(object.Body, want+1)) + if err != nil { + return nil, true, err + } + if int64(len(data)) < want { + return nil, true, io.ErrUnexpectedEOF + } + if int64(len(data)) != want { + return nil, false, errors.New("remote archive range exceeds its declared size") + } + return data, false, nil +} + +func (s *Store) Download(ctx context.Context, uri, filename, checksum string, size int64) (err error) { + key, err := s.key(uri) + if err != nil { + return err + } + if size <= 0 { + return errors.New("invalid archive size") + } + f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + defer func() { + err = errors.Join(err, f.Close()) + if err != nil { + _ = os.Remove(filename) + } + }() + if err = s.readVerified(ctx, key, f, checksum, size, false); err != nil { + return err + } + return f.Sync() +} + +func (s *Store) Delete(ctx context.Context, uri string) error { + key, err := s.key(uri) + if err != nil { + return err + } + _, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + return err +} diff --git a/archivestore/s3_test.go b/archivestore/s3_test.go new file mode 100644 index 0000000..84049e3 --- /dev/null +++ b/archivestore/s3_test.go @@ -0,0 +1,420 @@ +package archivestore + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" +) + +func TestS3RoundTripAndIntegrity(t *testing.T) { + for _, sse := range []string{"AES256", "aws:kms"} { + kms := "" + if sse == "aws:kms" { + kms = "arn:aws:kms:us-east-1:123456789012:key/test-key" + } + for _, size := range []int{1024, 10 << 20} { + t.Run(sse+"/"+strconv.Itoa(size), func(t *testing.T) { + var mu sync.Mutex + var object []byte + parts := map[int][]byte{} + corrupt, multipart := false, false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + q := r.URL.Query() + if (r.Method == "PUT" && !q.Has("partNumber")) || (r.Method == "POST" && q.Has("uploads")) { + if r.Header.Get("X-Amz-Server-Side-Encryption") != sse || r.Header.Get("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id") != kms { + t.Error("upload did not explicitly request the configured encryption") + } + } + if r.Method == "HEAD" { + if r.Header.Get("X-Amz-Server-Side-Encryption") != "" { + t.Error("encryption write header leaked into HEAD") + } + w.Header().Set("X-Amz-Server-Side-Encryption", sse) + if kms != "" { + w.Header().Set("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id", kms) + } + } + w.Header().Set("ETag", `"test-etag"`) + switch { + case r.Method == "POST" && q.Has("uploads"): + multipart = true + fmt.Fprint(w, `test-upload`) + case r.Method == "PUT" && q.Has("partNumber"): + n, _ := strconv.Atoi(q.Get("partNumber")) + parts[n], _ = io.ReadAll(r.Body) + case r.Method == "POST" && q.Has("uploadId"): + for i := 1; i <= len(parts); i++ { + object = append(object, parts[i]...) + } + fmt.Fprint(w, `test-etag`) + case r.Method == "PUT": + object, _ = io.ReadAll(r.Body) + case r.Method == "HEAD": + w.Header().Set("Content-Length", strconv.Itoa(len(object))) + case r.Method == "GET": + data := object + if corrupt { + data = bytes.Clone(object) + data[0] ^= 0xff + } + serveArchiveRange(t, w, r, data) + case r.Method == "DELETE": + object = nil + w.WriteHeader(204) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL) + w.WriteHeader(400) + } + })) + defer srv.Close() + store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true, SSE: sse, KMSKeyID: kms}) + if err != nil { + t.Fatal(err) + } + data := bytes.Repeat([]byte{0x42}, size) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + root := t.TempDir() + source := filepath.Join(root, "source") + if err := os.WriteFile(source, data, 0600); err != nil { + t.Fatal(err) + } + uri := store.URI(1, 2, checksum) + ctx := context.Background() + if err := store.Upload(ctx, uri, source, checksum); err != nil { + t.Fatal(err) + } + if multipart != (size > 8<<20) { + t.Fatalf("multipart=%v size=%d", multipart, size) + } + dest := filepath.Join(root, "download") + if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil { + t.Fatal(err) + } + got, _ := os.ReadFile(dest) + if !bytes.Equal(got, data) { + t.Fatal("download differs") + } + mu.Lock() + corrupt = true + mu.Unlock() + bad := filepath.Join(root, "bad") + if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil { + t.Fatal("accepted corrupt object") + } + if _, err := os.Stat(bad); !os.IsNotExist(err) { + t.Fatal("left corrupt download") + } + if err := store.Upload(ctx, uri, source, checksum); err == nil { + t.Fatal("upload accepted failed readback") + } + if err := store.Delete(ctx, uri); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func serveArchiveRange(t *testing.T, w http.ResponseWriter, r *http.Request, data []byte) { + t.Helper() + var start, end int64 + if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) { + t.Errorf("invalid range %q", r.Header.Get("Range")) + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return + } + if r.Header.Get("If-Match") != `"test-etag"` { + t.Errorf("missing object version condition: %q", r.Header.Get("If-Match")) + w.WriteHeader(http.StatusPreconditionFailed) + return + } + w.Header().Set("ETag", `"test-etag"`) + w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[start : end+1]) +} + +func TestS3ReadbackRetriesInterruptedRange(t *testing.T) { + data := append(bytes.Repeat([]byte{0x42}, int(archiveReadChunkSize)), bytes.Repeat([]byte{0x73}, 1024)...) + var mu sync.Mutex + var ranges []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + w.Header().Set("ETag", `"test-etag"`) + if r.Method == "HEAD" { + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + return + } + ranges = append(ranges, r.Header.Get("Range")) + if len(ranges) == 2 { + w.Header().Set("Content-Length", strconv.FormatInt(int64(len(data))-archiveReadChunkSize, 10)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", archiveReadChunkSize, len(data)-1, len(data))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[archiveReadChunkSize : archiveReadChunkSize+128]) // Close during a later chunk. + return + } + serveArchiveRange(t, w, r, data) + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + dest := filepath.Join(t.TempDir(), "archive") + if err := store.Download(context.Background(), store.URI(1, 2, checksum), dest, checksum, int64(len(data))); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(dest) + if err != nil || !bytes.Equal(got, data) { + t.Fatalf("retried download differs: %v", err) + } + mu.Lock() + defer mu.Unlock() + if len(ranges) != 3 || ranges[1] != ranges[2] || ranges[0] != fmt.Sprintf("bytes=0-%d", archiveReadChunkSize-1) || ranges[2] != fmt.Sprintf("bytes=%d-%d", archiveReadChunkSize, len(data)-1) { + t.Fatalf("retry did not stay within the interrupted range: %v", ranges) + } +} + +func testRangeStore(t *testing.T, endpoint string) *Store { + t.Helper() + s, err := New(Config{Endpoint: endpoint, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + return s +} + +func TestS3ReadbackRejectsInvalidRangesAndExhaustedRetries(t *testing.T) { + for _, mode := range []string{"ignored-range", "changed-etag", "changed-object", "truncated"} { + t.Run(mode, func(t *testing.T) { + data := bytes.Repeat([]byte{0x42}, 1024) + var mu sync.Mutex + gets := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + w.Header().Set("ETag", `"test-etag"`) + if r.Method == "HEAD" { + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + return + } + gets++ + if mode == "changed-object" { + w.WriteHeader(http.StatusPreconditionFailed) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + if mode == "ignored-range" { + _, _ = w.Write(data) + return + } + if mode == "changed-etag" { + w.Header().Set("ETag", `"different-object"`) + } + w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", len(data)-1, len(data))) + w.WriteHeader(http.StatusPartialContent) + if mode == "truncated" { + _, _ = w.Write(data[:128]) + } else { + _, _ = w.Write(data) + } + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + dest := filepath.Join(t.TempDir(), "archive") + if err := store.Download(context.Background(), store.URI(1, 2, checksum), dest, checksum, int64(len(data))); err == nil { + t.Fatal("accepted invalid or incomplete readback") + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatal("left a failed download on disk") + } + mu.Lock() + defer mu.Unlock() + want := 1 + if mode == "truncated" { + want = archiveReadAttempts + } + if gets != want { + t.Fatalf("GET count %d, want %d", gets, want) + } + }) + } +} + +type failedArchiveWriter struct{ err error } + +func (w failedArchiveWriter) Write([]byte) (int, error) { return 0, w.err } + +func TestS3ReadbackPropagatesOutputFailure(t *testing.T) { + data := []byte("test archive") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"test-etag"`) + if r.Method == "HEAD" { + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + return + } + serveArchiveRange(t, w, r, data) + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + h := sha256.Sum256(data) + outputError := errors.New("output disk full") + err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data)), false) + if !errors.Is(err, outputError) { + t.Fatalf("lost output error: %v", err) + } +} + +func TestS3ReadbackStopsOnCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + data := bytes.Repeat([]byte{0x42}, 1024) + gets := make(chan struct{}, archiveReadAttempts+1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"test-etag"`) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + if r.Method == "HEAD" { + return + } + gets <- struct{}{} + w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", len(data)-1, len(data))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[:128]) + w.(http.Flusher).Flush() + cancel() + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + dest := filepath.Join(t.TempDir(), "archive") + err := store.Download(ctx, store.URI(1, 2, checksum), dest, checksum, int64(len(data))) + if !errors.Is(err, context.Canceled) || len(gets) != 1 { + t.Fatalf("cancellation was not honored: error=%v GETs=%d", err, len(gets)) + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatal("left a cancelled download on disk") + } +} + +func TestS3RejectsForeignLocations(t *testing.T) { + s, err := New(Config{Endpoint: "https://sfo3.digitaloceanspaces.com", Bucket: "allowed", Prefix: "archives/tasks", AccessKey: "test", SecretKey: "test"}) + if err != nil { + t.Fatal(err) + } + for _, uri := range []string{"s3://other/archives/tasks/a", "s3://allowed/archives/other/a", "s3://allowed/archives/tasks/../a", "s3://allowed/archives/tasks/%2e%2e/a", "s3://allowed/archives/tasks/a?secret=x", "s3://user@allowed/archives/tasks/a", "file:///etc/passwd"} { + if err := s.Validate(uri); err == nil { + t.Errorf("accepted %s", uri) + } + } + var absent *Store + if absent.Validate("s3://allowed/archives/tasks/a") == nil { + t.Fatal("missing config accepted") + } +} + +func TestS3UploadRejectsUnconfirmedEncryption(t *testing.T) { + for _, mode := range []string{"missing", "different", "kms-without-key"} { + t.Run(mode, func(t *testing.T) { + data := []byte("private archive") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "PUT": + _, _ = io.Copy(io.Discard, r.Body) + case "DELETE": + w.WriteHeader(204) + case "HEAD": + w.Header().Set("ETag", `"test-etag"`) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + if mode != "missing" { + w.Header().Set("X-Amz-Server-Side-Encryption", "aws:kms") + } + default: + t.Errorf("content must not be accepted before encryption confirmation: %s", r.Method) + w.WriteHeader(400) + } + })) + defer srv.Close() + c := Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true} + if mode == "kms-without-key" { + c.SSE = "aws:kms" + } + store, err := New(c) + if err != nil { + t.Fatal(err) + } + filename := filepath.Join(t.TempDir(), "source") + if err := os.WriteFile(filename, data, 0600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(data) + checksum := hex.EncodeToString(sum[:]) + if err := store.Upload(context.Background(), store.URI(1, 2, checksum), filename, checksum); err == nil || !strings.Contains(err.Error(), "encryption") { + t.Fatalf("unconfirmed encryption accepted: %v", err) + } + if got, err := os.ReadFile(filename); err != nil || !bytes.Equal(got, data) { + t.Fatal("source was not retained", err) + } + }) + } +} + +func TestS3MultipartFailureAbortsUpload(t *testing.T) { + var aborted atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case r.Method == "POST" && q.Has("uploads"): + if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" { + t.Error("multipart encryption missing") + } + fmt.Fprint(w, `failed-upload`) + case r.Method == "PUT" && q.Has("partNumber"): + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(403) + case r.Method == "DELETE" && q.Get("uploadId") == "failed-upload": + aborted.Store(true) + w.WriteHeader(204) + default: + t.Errorf("unexpected failed-upload request: %s", r.Method) + w.WriteHeader(400) + } + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + data := bytes.Repeat([]byte("x"), 9<<20) + checksum := sha256.Sum256(data) + sum := hex.EncodeToString(checksum[:]) + filename := filepath.Join(t.TempDir(), "source") + if err := os.WriteFile(filename, data, 0600); err != nil { + t.Fatal(err) + } + if err := store.Upload(context.Background(), store.URI(1, 2, sum), filename, sum); err == nil { + t.Fatal("failed part accepted") + } + if !aborted.Load() { + t.Fatal("failed multipart upload was not aborted") + } + if info, err := os.Stat(filename); err != nil || info.Size() != int64(len(data)) { + t.Fatal("source was not retained", err) + } +} diff --git a/archivestore/safety_test.go b/archivestore/safety_test.go new file mode 100644 index 0000000..9afe47b --- /dev/null +++ b/archivestore/safety_test.go @@ -0,0 +1,231 @@ +package archivestore + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" +) + +func archiveSource(t *testing.T, data []byte) (string, string) { + t.Helper() + filename := filepath.Join(t.TempDir(), "source") + if err := os.WriteFile(filename, data, 0600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(data) + return filename, hex.EncodeToString(sum[:]) +} + +func TestS3VerificationFailureCleansUploadedVersion(t *testing.T) { + for _, failure := range []string{"wrong-kms", "checksum", "cancelled", "delete-denied"} { + t.Run(failure, func(t *testing.T) { + data := []byte("private archive") + source, checksum := archiveSource(t, data) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var present, deleted atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "PUT": + _, _ = io.Copy(io.Discard, r.Body) + present.Store(true) + w.Header().Set("X-Amz-Version-Id", "uploaded-version") + case "HEAD": + w.Header().Set("ETag", `"test-etag"`) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + w.Header().Set("X-Amz-Server-Side-Encryption", "aws:kms") + kms := "arn:aws:kms:us-east-1:123456789012:key/expected" + if failure == "wrong-kms" { + kms = "arn:aws:kms:us-east-1:123456789012:key/different" + } + w.Header().Set("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id", kms) + if failure == "cancelled" { + cancel() + } + case "GET": + corrupt := bytes.Clone(data) + corrupt[0] ^= 0xff + serveArchiveRange(t, w, r, corrupt) + case "DELETE": + if r.URL.Query().Get("versionId") != "uploaded-version" { + t.Error("cleanup did not target uploaded version") + } + deleted.Store(true) + if failure == "delete-denied" { + w.WriteHeader(403) + return + } + present.Store(false) + w.WriteHeader(204) + default: + t.Errorf("unexpected request %s", r.Method) + w.WriteHeader(400) + } + })) + defer srv.Close() + store, err := New(Config{Endpoint: srv.URL, Bucket: "test", AccessKey: "test", SecretKey: "test", PathStyle: true, + SSE: "aws:kms", KMSKeyID: "arn:aws:kms:us-east-1:123456789012:key/expected"}) + if err != nil { + t.Fatal(err) + } + err = store.Upload(ctx, store.URI(1, 2, checksum), source, checksum) + if err == nil { + t.Fatal("accepted failed verification") + } + if failure == "wrong-kms" && !strings.Contains(err.Error(), "encryption") { + t.Fatal(err) + } + if failure == "cancelled" && !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if failure == "delete-denied" && (!strings.Contains(err.Error(), "SHA-256") || !strings.Contains(err.Error(), "delete unverified archive")) { + t.Fatalf("verification and cleanup errors must both survive: %v", err) + } + if !deleted.Load() || present.Load() != (failure == "delete-denied") { + t.Fatal("unverified object cleanup was not attempted/completed") + } + if got, readErr := os.ReadFile(source); readErr != nil || !bytes.Equal(got, data) { + t.Fatal("source changed", readErr) + } + }) + } +} + +func TestS3CancelledMultipartAbortsWithFreshContext(t *testing.T) { + for _, denyAbort := range []bool{false, true} { + t.Run(strconv.FormatBool(denyAbort), func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var aborted atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case r.Method == "POST" && q.Has("uploads"): + fmt.Fprint(w, `cancelled-upload`) + case r.Method == "PUT" && q.Has("partNumber"): + cancel() + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(403) + case r.Method == "DELETE" && q.Get("uploadId") == "cancelled-upload": + aborted.Store(true) + if denyAbort { + w.WriteHeader(403) + } else { + w.WriteHeader(204) + } + default: + t.Errorf("unexpected request %s", r.Method) + w.WriteHeader(400) + } + })) + defer srv.Close() + data := bytes.Repeat([]byte("x"), 9<<20) + source, sum := archiveSource(t, data) + store := testRangeStore(t, srv.URL) + err := store.Upload(ctx, store.URI(1, 2, sum), source, sum) + if !errors.Is(err, context.Canceled) || !aborted.Load() { + t.Fatalf("cancelled upload not aborted: %v", err) + } + if denyAbort && !strings.Contains(err.Error(), "abort archive multipart upload") { + t.Fatalf("cleanup failure lost: %v", err) + } + if got, readErr := os.ReadFile(source); readErr != nil || !bytes.Equal(got, data) { + t.Fatal("source changed", readErr) + } + }) + } +} + +func TestS3SignedRequestsNeverFollowRedirects(t *testing.T) { + var reached atomic.Int32 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reached.Add(1); w.WriteHeader(204) })) + defer target.Close() + for _, status := range []int{301, 302, 303, 307, 308} { + t.Run(strconv.Itoa(status), func(t *testing.T) { + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") == "" || r.Header.Get("X-Amz-Security-Token") != "fixture-session" { + t.Error("fixture request must be signed with session credentials") + } + w.Header().Set("Location", target.URL+"/redirected") + w.WriteHeader(status) + })) + defer origin.Close() + store, err := New(Config{Endpoint: origin.URL, Bucket: "test", AccessKey: "test", SecretKey: "test", SessionToken: "fixture-session", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + if err := store.Delete(context.Background(), store.URI(1, 2, "test")); err == nil { + t.Fatal("redirect accepted") + } + if reached.Load() != 0 { + t.Fatal("signed request followed redirect") + } + }) + } +} + +func TestS3CustomCABundleForBothCredentialModes(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) })) + defer srv.Close() + for _, explicit := range []bool{false, true} { + t.Run(strconv.FormatBool(explicit), func(t *testing.T) { + isolateAWSCredentials(t) + ca := filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("AWS_CA_BUNDLE", ca) + c := Config{Endpoint: srv.URL, Bucket: "test", PathStyle: true} + if explicit { + c.AccessKey, c.SecretKey = "fixture", "fixture" + } else { + t.Setenv("AWS_ACCESS_KEY_ID", "fixture") + t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture") + } + store, err := New(c) + if err != nil { + t.Fatal(err) + } + if err := store.Delete(context.Background(), store.URI(1, 2, "test")); err != nil { + t.Fatalf("configured CA not used: %v", err) + } + }) + } +} + +func TestS3RangeRetriesHaveOneWireAttemptEach(t *testing.T) { + for _, status := range []int{403, 429, 503} { + t.Run(strconv.Itoa(status), func(t *testing.T) { + var gets atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gets.Add(1) + w.WriteHeader(status) + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + if _, err := store.readArchiveChunk(context.Background(), "key", `"etag"`, 0, 9, 10); err == nil { + t.Fatal("failed request accepted") + } + want := int32(archiveReadAttempts) + if status == 403 { + want = 1 + } + if got := gets.Load(); got != want { + t.Fatalf("wire requests=%d want=%d", got, want) + } + }) + } +} diff --git a/cmd/artex-archive-check/main.go b/cmd/artex-archive-check/main.go new file mode 100644 index 0000000..af0efd9 --- /dev/null +++ b/cmd/artex-archive-check/main.go @@ -0,0 +1,64 @@ +// artex-archive-check tests the configured object store without touching tasks. +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "log" + "os" + "path/filepath" + "time" + + "github.com/Autumn-27/artex/archivestore" +) + +func main() { + if err := check(); err != nil { + log.Fatal(err) + } +} + +func check() error { + s, err := archivestore.FromEnv() + if err != nil { + return err + } + if s == nil { + return fmt.Errorf("S3 archive storage is not configured") + } + dir, err := os.MkdirTemp("", "artex-archive-check-") + if err != nil { + return err + } + defer os.RemoveAll(dir) + data := []byte("ARTEX object storage connectivity check " + time.Now().UTC().Format(time.RFC3339Nano)) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + uri := s.URI(0, time.Now().UnixNano(), checksum) + file := filepath.Join(dir, "check") + if err := os.WriteFile(file, data, 0600); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + defer func() { + cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := s.Delete(cleanup, uri); err != nil { + log.Printf("test object cleanup failed for %s: %v", uri, err) + } + }() + if err := s.Upload(ctx, uri, file, checksum); err != nil { + return err + } + if err := s.Download(ctx, uri, filepath.Join(dir, "download"), checksum, int64(len(data))); err != nil { + return err + } + if err := s.Delete(ctx, uri); err != nil { + return err + } + fmt.Println("S3 upload, full SHA-256 verification, download and delete passed") + return nil +} diff --git a/go.mod b/go.mod index 8160b04..353f837 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,11 @@ go 1.26.10 require ( github.com/Autumn-27/norma v0.4.11 + github.com/aws/aws-sdk-go-v2 v1.47.5 + github.com/aws/aws-sdk-go-v2/config v1.33.13 + github.com/aws/aws-sdk-go-v2/credentials v1.20.13 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 @@ -22,6 +27,20 @@ require ( require ( github.com/andybalholm/brotli v1.2.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect + github.com/aws/smithy-go v1.28.7 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/creack/pty v1.1.24 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/go.sum b/go.sum index de01417..a950b0c 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,44 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= +github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 h1:MW0Iql1SOTgQPj8mCxrU/eH/K/AIDWi08ItOsXaWRLU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25/go.mod h1:+ucVYmHYB3JqVueY0E1RHahdXAnKQbIHqybSbvOUbOM= +github.com/aws/aws-sdk-go-v2/config v1.33.13 h1:u52x6ISQOx172tpwM0VRxhJzEsTJYpMcEYUGM2A1wrw= +github.com/aws/aws-sdk-go-v2/config v1.33.13/go.mod h1:/ixtz6XPHb42zekjQ0LqD9XjTpTysNWhIGwufY1OzI4= +github.com/aws/aws-sdk-go-v2/credentials v1.20.13 h1:j4z41OtspmQ3ftQycaDunJ26b69a9SHfaSVUPqgRv8k= +github.com/aws/aws-sdk-go-v2/credentials v1.20.13/go.mod h1:SgPA1oS3b+dlMxZb618jDmGNscpXSCaxz15igS52OlE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19 h1:+JWfo9Mkg5HWwDwnpRPu7WNiiyEmiQRTaNSFsZ/1KIY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19/go.mod h1:l4/tI+TfymypMqq2oRCkhMAB7Gi7nlt5JaoUUz+cUbI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 h1:GACgECI422Xt5n4Pojr3O0aocPv8lfH4PJe2Aon5Wv8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9/go.mod h1:lPfnRkHpn1qWZJLbVEw10rIqSuCee1pNQTyGkUjAYh8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 h1:RS1Wt+K0r4X0ci9Vf4JYTUfHzsQihYA2LgGJeEneKpk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8/go.mod h1:adMnTy5kyoUW86jhXnGSVc/mvMQI4GZd1qYWMeaS8M8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 h1:to7+Vr3rLkpYRwVKJ5QRriI0eE306vrVN/6TnVHgxvY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6/go.mod h1:Jc89ASa2k7HMAskj3bSnt8bf+k+p2IApzkoXnEjkgUw= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 h1:WZLC+k/CZx11kZGQDqu9mpA8yhiXGX2I//jhuJE1FDo= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.1/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 h1:4VB/dnDo7pmXSLQMszSoBzWndENuc7mzwTpD6lQn77A= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.1/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 h1:w8yui2ZqdHmGwVBDGvazoIvKw/DnLENBLkYnN8q8iqo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw= +github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= +github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=