chore: replace aws-sdk-go v1 with aws-sdk-go-v2

.github/workflows/release.yml

@@ -30,7 +30,11 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          gh release create "$GITHUB_REF_NAME" \
-            --title "$GITHUB_REF_NAME" \
-            --generate-notes \
-            goofys goofys-amd64 goofys-arm64
+          if gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then
+            gh release upload "$GITHUB_REF_NAME" --clobber goofys goofys-amd64 goofys-arm64
+          else
+            gh release create "$GITHUB_REF_NAME" \
+              --title "$GITHUB_REF_NAME" \
+              --generate-notes \
+              goofys goofys-amd64 goofys-arm64
+          fi

README.md

@@ -126,7 +126,7 @@ Additionally, goofys also works with the following non-S3 object stores:
 # References
 
   * Data is stored on [Amazon S3](https://aws.amazon.com/s3/)
-  * [Amazon SDK for Go](https://github.com/aws/aws-sdk-go)
+  * [AWS SDK for Go v2](https://github.com/aws/aws-sdk-go-v2)
   * Other related fuse filesystems
     * [catfs](https://github.com/kahing/catfs): caching layer that can be used with goofys
     * [s3fs](https://github.com/s3fs-fuse/s3fs-fuse): another popular filesystem for S3

api/common/conf_s3.go

@@ -15,16 +15,20 @@
 package common
 
 import (
+	"context"
 	"crypto/md5"
 	"encoding/base64"
 	"fmt"
 	"net/http"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/client"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
-	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/ratelimit"
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
+	"github.com/aws/aws-sdk-go-v2/service/sts"
+	"github.com/aws/smithy-go/logging"
 )
 
 type S3Config struct {
@@ -51,14 +55,12 @@ type S3Config struct {
 
 	Subdomain bool
 
-	Credentials *credentials.Credentials
-	Session     *session.Session
+	Credentials aws.CredentialsProvider
+	Session     *aws.Config
 
 	BucketOwner string
 }
 
-var s3Session *session.Session
-
 func (c *S3Config) Init() *S3Config {
 	if c.Region == "" {
 		c.Region = "us-east-1"
@@ -70,58 +72,77 @@ func (c *S3Config) Init() *S3Config {
 }
 
 func (c *S3Config) ToAwsConfig(flags *FlagStorage) (*aws.Config, error) {
-	awsConfig := (&aws.Config{
-		Region: &c.Region,
-		Logger: GetLogger("s3"),
-	}).WithHTTPClient(&http.Client{
+	ctx := context.Background()
+	httpClient := &http.Client{
 		Transport: &defaultHTTPTransport,
 		Timeout:   flags.HTTPTimeout,
+	}
+	log := GetLogger("s3")
+	sdkLogger := logging.LoggerFunc(func(_ logging.Classification, format string, args ...interface{}) {
+		log.Debugf(format, args...)
 	})
+	var logMode aws.ClientLogMode
 	if flags.DebugS3 {
-		awsConfig.LogLevel = aws.LogLevel(aws.LogDebug | aws.LogDebugWithRequestErrors)
+		logMode = aws.LogRequest | aws.LogResponse | aws.LogRetries
 	}
-
-	if c.Credentials == nil {
-		if c.AccessKey != "" {
-			c.Credentials = credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, "")
-		} else if c.Profile != "" {
-			c.Credentials = newSharedFileCredentials(c.Profile)
-		}
+	loadOptions := []func(*config.LoadOptions) error{
+		config.WithRegion(c.Region),
+		config.WithHTTPClient(httpClient),
+		config.WithLogger(sdkLogger),
+		config.WithClientLogMode(logMode),
 	}
-	if flags.Endpoint != "" {
-		awsConfig.Endpoint = &flags.Endpoint
+	if c.Credentials == nil && c.AccessKey != "" {
+		c.Credentials = credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, "")
 	}
-
-	awsConfig.S3ForcePathStyle = aws.Bool(!c.Subdomain)
-
 	if c.Session == nil {
-		if s3Session == nil {
-			var err error
-			s3Session, err = session.NewSessionWithOptions(session.Options{
-				Profile:           c.Profile,
-				SharedConfigState: session.SharedConfigEnable,
-			})
-			if err != nil {
-				return nil, err
-			}
+		options := sharedConfigLoadOptions(c.Profile, loadOptions)
+		if c.Credentials != nil {
+			options = append(options, config.WithCredentialsProvider(c.Credentials))
 		}
-		c.Session = s3Session
+		loaded, err := config.LoadDefaultConfig(ctx, options...)
+		if err != nil {
+			return nil, err
+		}
+		if c.Credentials == nil {
+			loaded.Credentials = &sharedFileProvider{profile: c.Profile, loadOptions: loadOptions}
+		}
+		c.Session = &loaded
+	} else if c.Credentials == nil && c.Profile != "" {
+		c.Credentials = &sharedFileProvider{profile: c.Profile, loadOptions: loadOptions}
 	}
 
-	if c.RoleArn != "" {
-		c.Credentials = stscreds.NewCredentials(stsConfigProvider{c}, c.RoleArn,
-			func(p *stscreds.AssumeRoleProvider) {
-				if c.RoleExternalId != "" {
-					p.ExternalID = &c.RoleExternalId
-				}
-				p.RoleSessionName = c.RoleSessionName
+	awsConfig := c.Session.Copy()
+	awsConfig.Region = c.Region
+	awsConfig.HTTPClient = httpClient
+	awsConfig.Logger = sdkLogger
+	awsConfig.ClientLogMode = logMode
+	if awsConfig.Retryer == nil {
+		awsConfig.Retryer = func() aws.Retryer {
+			return retry.NewStandard(func(options *retry.StandardOptions) {
+				options.MaxAttempts = 4
+				options.RateLimiter = ratelimit.None
 			})
+		}
 	}
-
 	if c.Credentials != nil {
 		awsConfig.Credentials = c.Credentials
 	}
 
+	if c.RoleArn != "" {
+		stsClient := sts.NewFromConfig(awsConfig, func(options *sts.Options) {
+			if c.StsEndpoint != "" {
+				options.BaseEndpoint = aws.String(c.StsEndpoint)
+			}
+		})
+		awsConfig.Credentials = aws.NewCredentialsCache(stscreds.NewAssumeRoleProvider(stsClient, c.RoleArn,
+			func(options *stscreds.AssumeRoleOptions) {
+				if c.RoleExternalId != "" {
+					options.ExternalID = aws.String(c.RoleExternalId)
+				}
+				options.RoleSessionName = c.RoleSessionName
+			}))
+	}
+
 	if c.SseC != "" {
 		key, err := base64.StdEncoding.DecodeString(c.SseC)
 		if err != nil {
@@ -133,21 +154,5 @@ func (c *S3Config) ToAwsConfig(flags *FlagStorage) (*aws.Config, error) {
 		c.SseCDigest = base64.StdEncoding.EncodeToString(m[:])
 	}
 
-	return awsConfig, nil
-}
-
-type stsConfigProvider struct {
-	*S3Config
-}
-
-func (c stsConfigProvider) ClientConfig(serviceName string, cfgs ...*aws.Config) client.Config {
-	config := c.Session.ClientConfig(serviceName, cfgs...)
-	if c.Credentials != nil {
-		config.Config.Credentials = c.Credentials
-	}
-	if c.StsEndpoint != "" {
-		config.Endpoint = c.StsEndpoint
-	}
-
-	return config
+	return &awsConfig, nil
 }

api/common/conf_s3_credentials.go

@@ -15,13 +15,13 @@
 package common
 
 import (
+	"context"
 	"os"
 	"sync"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/defaults"
-	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
 )
 
 const sharedCredentialsStatInterval = 10 * time.Second
@@ -38,45 +38,36 @@ func (s sharedFileState) equal(other sharedFileState) bool {
 }
 
 type sharedFileProvider struct {
-	mu         sync.Mutex
-	profile    string
-	resolved   *credentials.Credentials
-	value      credentials.Value
-	state      sharedFileState
-	loaded     bool
-	expired    bool
-	unreadable bool
-	checkedAt  time.Time
+	mu          sync.Mutex
+	profile     string
+	loadOptions []func(*config.LoadOptions) error
+	value       aws.Credentials
+	state       sharedFileState
+	loaded      bool
+	expired     bool
+	unreadable  bool
+	checkedAt   time.Time
 }
 
-func newSharedFileCredentials(profile string) *credentials.Credentials {
-	creds := credentials.NewCredentials(&sharedFileProvider{profile: profile})
-	if _, err := creds.Get(); err != nil {
-		credentialsLog.Warnf("cannot resolve credentials for profile %v: %v", profile, err)
-	}
-	return creds
-}
-
-func (p *sharedFileProvider) Retrieve() (credentials.Value, error) {
+func (p *sharedFileProvider) Retrieve(ctx context.Context) (aws.Credentials, error) {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
+	if !p.isExpired() {
+		return p.value, nil
+	}
 	state, stated := p.fileState()
 
-	sess, err := session.NewSessionWithOptions(session.Options{
-		Profile:           p.profile,
-		SharedConfigState: session.SharedConfigEnable,
-	})
+	cfg, err := config.LoadDefaultConfig(ctx, sharedConfigLoadOptions(p.profile, p.loadOptions)...)
 	if err != nil {
 		return p.keepLoaded(err)
 	}
 
-	value, err := sess.Config.Credentials.Get()
+	value, err := cfg.Credentials.Retrieve(ctx)
 	if err != nil {
 		return p.keepLoaded(err)
 	}
 
-	p.resolved = sess.Config.Credentials
 	p.value = value
 	p.loaded = true
 	p.expired = false
@@ -91,22 +82,31 @@ func (p *sharedFileProvider) Retrieve() (credentials.Value, error) {
 func (p *sharedFileProvider) IsExpired() bool {
 	p.mu.Lock()
 	defer p.mu.Unlock()
+	return p.isExpired()
+}
+
+func (p *sharedFileProvider) Expire() {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.expired = true
+}
 
+func (p *sharedFileProvider) isExpired() bool {
 	if !p.loaded || p.expired {
 		return true
 	}
 
+	if p.value.Expired() {
+		p.expired = true
+		return true
+	}
+
 	now := time.Now()
 	if now.Sub(p.checkedAt) < sharedCredentialsStatInterval {
 		return false
 	}
 	p.checkedAt = now
 
-	if p.resolved.IsExpired() {
-		p.expired = true
-		return true
-	}
-
 	state, stated := p.fileState()
 	if !stated {
 		if !p.unreadable {
@@ -122,9 +122,9 @@ func (p *sharedFileProvider) IsExpired() bool {
 	return p.expired
 }
 
-func (p *sharedFileProvider) keepLoaded(err error) (credentials.Value, error) {
+func (p *sharedFileProvider) keepLoaded(err error) (aws.Credentials, error) {
 	if !p.loaded {
-		return credentials.Value{}, err
+		return aws.Credentials{}, err
 	}
 
 	p.expired = false
@@ -134,6 +134,14 @@ func (p *sharedFileProvider) keepLoaded(err error) (credentials.Value, error) {
 	return p.value, nil
 }
 
+func sharedConfigLoadOptions(profile string, base []func(*config.LoadOptions) error) []func(*config.LoadOptions) error {
+	options := append([]func(*config.LoadOptions) error{}, base...)
+	if profile != "" {
+		options = append(options, config.WithSharedConfigProfile(profile))
+	}
+	return options
+}
+
 func (p *sharedFileProvider) fileState() (sharedFileState, bool) {
 	info, err := os.Stat(sharedCredentialsFilename())
 	if err != nil {
@@ -146,5 +154,5 @@ func sharedCredentialsFilename() string {
 	if filename := os.Getenv("AWS_SHARED_CREDENTIALS_FILE"); filename != "" {
 		return filename
 	}
-	return defaults.SharedCredentialsFilename()
+	return config.DefaultSharedCredentialsFilename()
 }