Skip to content

Commit

Permalink
feat: Add support for Bucket Request configuration
Browse files Browse the repository at this point in the history 
Related to issue #358
  • Loading branch information
@oxyno-zeta
oxyno-zeta committed May 8, 2023
1 parent 7d7b92c commit a593c6c
Show file tree
Hide file tree
Showing 5 changed files with 113 additions and 19 deletions.
9 changes: 9 additions & 0 deletions conf/config-example.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -480,3 +480,12 @@ targets:
# env: AWS_ACCESS_KEY_ID
# secretKey:
# path: secret_key_file
# requestConfig:
# listHeaders:
# Accept-Encoding: gzip
# getHeaders:
# Accept-Encoding: gzip
# putHeaders:
# Accept-Encoding: gzip
# deleteHeaders:
# Accept-Encoding: gzip
9 changes: 9 additions & 0 deletions docs/configuration/example.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -490,4 +490,13 @@ targets:
# env: AWS_ACCESS_KEY_ID
# secretKey:
# path: secret_key_file
# requestConfig:
# listHeaders:
# Accept-Encoding: gzip
# getHeaders:
# Accept-Encoding: gzip
# putHeaders:
# Accept-Encoding: gzip
# deleteHeaders:
# Accept-Encoding: gzip
```
28 changes: 19 additions & 9 deletions docs/configuration/structure.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,15 +302,25 @@ You can found more information [here](../feature-guide/webhooks.md) about webhoo

## BucketConfiguration

| Key | Type | Required | Default | Description |
| ------------- | --------------------------------------------------------------- | -------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| name | String | Yes | None | Bucket name in S3 provider |
| prefix | String | No | None | Bucket prefix |
| region | String | No | `us-east-1` | Bucket region |
| s3Endpoint | String | No | None | Custom S3 Endpoint for non AWS S3 bucket |
| credentials | [BucketCredentialConfiguration](#bucketcredentialconfiguration) | No | None | Credentials to access S3 bucket |
| disableSSL | Boolean | No | `false` | Disable SSL connection |
| s3ListMaxKeys | Integer | No | `1000` | This flag will be used for the max pagination list management of files and "folders" in S3. In S3 list requests, the limit is fixed to 1000 items maximum. S3-Proxy will allow to increase this by making multiple requests to S3. Warning: This will increase the memory and CPU usage. |
| Key | Type | Required | Default | Description |
| ------------- | --------------------------------------------------------------------- | -------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| name | String | Yes | None | Bucket name in S3 provider |
| prefix | String | No | None | Bucket prefix |
| region | String | No | `us-east-1` | Bucket region |
| s3Endpoint | String | No | None | Custom S3 Endpoint for non AWS S3 bucket |
| credentials | [BucketCredentialConfiguration](#bucketcredentialconfiguration) | No | None | Credentials to access S3 bucket |
| disableSSL | Boolean | No | `false` | Disable SSL connection |
| s3ListMaxKeys | Integer | No | `1000` | This flag will be used for the max pagination list management of files and "folders" in S3. In S3 list requests, the limit is fixed to 1000 items maximum. S3-Proxy will allow to increase this by making multiple requests to S3. Warning: This will increase the memory and CPU usage. |
| requestConfig | [BucketRequestConfigConfiguration](#bucketrequestconfigconfiguration) | No | `nil` | This will allow to customize requests sent to your S3 backend. |

## BucketRequestConfigConfiguration

| Key | Type | Required | Default | Description |
| ------------- | ----------------- | -------- | ------- | ---------------------------------------------------------------------- |
| listHeaders | Map[string]string | No | `nil` | Will allow to customize headers made to S3 provider on List requests |
| getHeaders | Map[string]string | No | `nil` | Will allow to customize headers made to S3 provider on Get requests |
| putHeaders | Map[string]string | No | `nil` | Will allow to customize headers made to S3 provider on Put requests |
| deleteHeaders | Map[string]string | No | `nil` | Will allow to customize headers made to S3 provider on Delete requests |

## BucketCredentialConfiguration

Expand Down
9 changes: 9 additions & 0 deletions pkg/s3-proxy/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,7 @@ type OPAServerAuthorization struct {
// BucketConfig Bucket configuration.
type BucketConfig struct {
Credentials *BucketCredentialConfig `mapstructure:"credentials" validate:"omitempty,dive"`
RequestConfig *BucketRequestConfig `mapstructure:"requestConfig" validate:"omitempty,dive"`
Name string `mapstructure:"name" validate:"required"`
Prefix string `mapstructure:"prefix"`
Region string `mapstructure:"region"`
Expand All @@ -475,6 +476,14 @@ type BucketConfig struct {
DisableSSL bool `mapstructure:"disableSSL"`
}

// BucketRequestConfig Bucket request configuration.
type BucketRequestConfig struct {
ListHeaders map[string]string `mapstructure:"listHeaders"`
GetHeaders map[string]string `mapstructure:"getHeaders"`
PutHeaders map[string]string `mapstructure:"putHeaders"`
DeleteHeaders map[string]string `mapstructure:"deleteHeaders"`
}

// BucketCredentialConfig Bucket Credentials configurations.
type BucketCredentialConfig struct {
AccessKey *CredentialConfig `mapstructure:"accessKey" validate:"omitempty,dive"`
Expand Down
77 changes: 67 additions & 10 deletions pkg/s3-proxy/s3client/s3Context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"emperror.dev/errors"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/config"
Expand Down Expand Up @@ -113,6 +114,12 @@ func (s3ctx *s3Context) ListFilesAndDirectories(ctx context.Context, key string)
// Get trace
parentTrace := tracing.GetTraceFromContext(ctx)

// Init & get request headers
var requestHeaders map[string]string
if s3ctx.target.Bucket.RequestConfig != nil {
requestHeaders = s3ctx.target.Bucket.RequestConfig.ListHeaders
}

// Loop
for loopControl {
// Create child trace
Expand Down Expand Up @@ -177,6 +184,7 @@ func (s3ctx *s3Context) ListFilesAndDirectories(ctx context.Context, key string)

return lastPage
},
addHeadersToRequest(requestHeaders),
)

// Metrics
Expand Down Expand Up @@ -223,7 +231,17 @@ func (s3ctx *s3Context) GetObject(ctx context.Context, input *GetInput) (*GetOut
// Build input
s3Input := s3ctx.buildGetObjectInputFromInput(input)

obj, err := s3ctx.svcClient.GetObjectWithContext(ctx, s3Input)
// Init & get request headers
var requestHeaders map[string]string
if s3ctx.target.Bucket.RequestConfig != nil {
requestHeaders = s3ctx.target.Bucket.RequestConfig.GetHeaders
}

obj, err := s3ctx.svcClient.GetObjectWithContext(
ctx,
s3Input,
addHeadersToRequest(requestHeaders),
)
// Metrics
s3ctx.metricsCtx.IncS3Operations(s3ctx.target.Name, s3ctx.target.Bucket.Name, GetObjectOperation)
// Check if error exists
Expand Down Expand Up @@ -356,8 +374,18 @@ func (s3ctx *s3Context) PutObject(ctx context.Context, input *PutInput) (*Result
inp.StorageClass = aws.String(input.StorageClass)
}

// Init & get request headers
var requestHeaders map[string]string
if s3ctx.target.Bucket.RequestConfig != nil {
requestHeaders = s3ctx.target.Bucket.RequestConfig.PutHeaders
}

// Upload to S3 bucket
_, err := s3ctx.svcClient.PutObjectWithContext(ctx, inp)
_, err := s3ctx.svcClient.PutObjectWithContext(
ctx,
inp,
addHeadersToRequest(requestHeaders),
)
// Check error
if err != nil {
return nil, errors.WithStack(err)
Expand Down Expand Up @@ -391,11 +419,21 @@ func (s3ctx *s3Context) HeadObject(ctx context.Context, key string) (*HeadOutput

defer childTrace.Finish()

// Init & get request headers
var requestHeaders map[string]string
if s3ctx.target.Bucket.RequestConfig != nil {
requestHeaders = s3ctx.target.Bucket.RequestConfig.GetHeaders
}

// Head object in bucket
_, err := s3ctx.svcClient.HeadObjectWithContext(ctx, &s3.HeadObjectInput{
Bucket: aws.String(s3ctx.target.Bucket.Name),
Key: aws.String(key),
})
_, err := s3ctx.svcClient.HeadObjectWithContext(
ctx,
&s3.HeadObjectInput{
Bucket: aws.String(s3ctx.target.Bucket.Name),
Key: aws.String(key),
},
addHeadersToRequest(requestHeaders),
)
// Metrics
s3ctx.metricsCtx.IncS3Operations(s3ctx.target.Name, s3ctx.target.Bucket.Name, HeadObjectOperation)
// Test error
Expand Down Expand Up @@ -434,11 +472,21 @@ func (s3ctx *s3Context) DeleteObject(ctx context.Context, key string) (*ResultIn

defer childTrace.Finish()

// Init & get request headers
var requestHeaders map[string]string
if s3ctx.target.Bucket.RequestConfig != nil {
requestHeaders = s3ctx.target.Bucket.RequestConfig.DeleteHeaders
}

// Delete object
_, err := s3ctx.svcClient.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s3ctx.target.Bucket.Name),
Key: aws.String(key),
})
_, err := s3ctx.svcClient.DeleteObjectWithContext(
ctx,
&s3.DeleteObjectInput{
Bucket: aws.String(s3ctx.target.Bucket.Name),
Key: aws.String(key),
},
addHeadersToRequest(requestHeaders),
)
// Check error
if err != nil {
return nil, errors.WithStack(err)
Expand All @@ -458,3 +506,12 @@ func (s3ctx *s3Context) DeleteObject(ctx context.Context, key string) (*ResultIn
// Return
return info, nil
}

func addHeadersToRequest(headers map[string]string) func(r *request.Request) {
return func(r *request.Request) {
// Loop over them
for k, v := range headers {
r.HTTPRequest.Header.Set(k, v)
}
}
}

0 comments on commit a593c6c

Please sign in to comment.