Skip to content

Commit

Permalink
feat(obj-prefix): enable prefixes on object keys (#5)
Browse files Browse the repository at this point in the history
* feat: enable to attach a prefix to the object key

* fix(obj-prefix): move regex compiling out of WithObjectPrefix function

* docs(obj-prefix): update WithObjectPrefix comment

* docs(obj-prefix): more invalid prefixes based in the characters to avoid

* feat(obj-prefix): abstract object key name to a func and add unit tests

* test(obj-prefix): modify TestWithObjectPrefix to use table-driven tests

* test(obj-prefix): use test.expectedErr in TestWithObjectPrefix

* style(obj-prefix): change an assertion to first pass the expected value

* refactor(obj-prefix): private method of client to generate the s3Key
  • Loading branch information
EmmanuelMr18 authored Oct 18, 2023
1 parent 89491b8 commit c03a75d
Show file tree
Hide file tree
Showing 2 changed files with 90 additions and 6 deletions.
35 changes: 29 additions & 6 deletions client.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package sqsextendedclient
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"regexp"
Expand All @@ -28,9 +29,11 @@ const (
)

var (
jsonUnmarshal = json.Unmarshal
jsonMarshal = json.Marshal
jsonUnmarshal = json.Unmarshal
jsonMarshal = json.Marshal
ErrObjectPrefix = errors.New("object prefix contains invalid characters")
)
var validObjectNameRegex = regexp.MustCompile("^[0-9a-zA-Z!_.*'()-]+$")

type S3Client interface {
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
Expand All @@ -49,6 +52,7 @@ type Client struct {
alwaysThroughS3 bool
pointerClass string
reservedAttrs []string
objectPrefix string
}

type ClientOption func(*Client) error
Expand Down Expand Up @@ -131,6 +135,25 @@ func WithPointerClass(pointerClass string) ClientOption {
}
}

// WithObjectPrefix attaches a prefix to the object key (prefix/uuid)
func WithObjectPrefix(prefix string) ClientOption {
return func(c *Client) error {
if !validObjectNameRegex.MatchString(prefix) {
return ErrObjectPrefix
}
c.objectPrefix = prefix
return nil
}
}

// s3Key returns a new string object key and prepends c.ObjectPrefix if it exists.
func (c *Client) s3Key(filename string) string {
if c.objectPrefix != "" {
return fmt.Sprintf("%s/%s", c.objectPrefix, filename)
}
return filename
}

// messageExceedsThreshold determines if the size of the body and attributes exceeds the configured
// message size threshold
func (c *Client) messageExceedsThreshold(body *string, attributes map[string]types.MessageAttributeValue) bool {
Expand Down Expand Up @@ -222,8 +245,8 @@ func (c *Client) SendMessage(ctx context.Context, params *sqs.SendMessageInput,
input.QueueUrl = &queueURL

if c.alwaysThroughS3 || c.messageExceedsThreshold(input.MessageBody, input.MessageAttributes) {
// generate UUID filename
s3Key := uuid.New().String()
// generate s3 object key
s3Key := c.s3Key(uuid.New().String())

// upload large payload to S3
_, err := c.s3c.PutObject(ctx, &s3.PutObjectInput{
Expand Down Expand Up @@ -321,8 +344,8 @@ func (c *Client) SendMessageBatch(ctx context.Context, params *sqs.SendMessageBa
copyEntries[i] = e

if c.alwaysThroughS3 || c.messageExceedsThreshold(e.MessageBody, e.MessageAttributes) {
// generate UUID filename
s3Key := uuid.New().String()
// generate s3 object key
s3Key := c.s3Key(uuid.New().String())

// upload large payload to S3
g.Go(func() error {
Expand Down
61 changes: 61 additions & 0 deletions client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/google/uuid"
"io"
"strings"
"testing"
Expand Down Expand Up @@ -107,6 +108,7 @@ func TestNewClientOptions(t *testing.T) {
WithPointerClass("pointer.class"),
WithReservedAttributeNames([]string{"Reserved", "Attributes"}),
WithS3BucketName("BUCKET!"),
WithObjectPrefix("custom_prefix"),
)

assert.Nil(t, err)
Expand All @@ -117,6 +119,7 @@ func TestNewClientOptions(t *testing.T) {
assert.Equal(t, "pointer.class", c.pointerClass)
assert.Equal(t, []string{"Reserved", "Attributes"}, c.reservedAttrs)
assert.Equal(t, "BUCKET!", c.bucketName)
assert.Equal(t, "custom_prefix", c.objectPrefix)
}

func TestNewClientOptionsFailure(t *testing.T) {
Expand Down Expand Up @@ -227,6 +230,64 @@ func TestS3PointerUnmarshalInvalidLength(t *testing.T) {
assert.ErrorContains(t, err, "invalid pointer format, expected length 2, but received [3]")
}

func TestS3Key(t *testing.T) {
uuid := uuid.New().String()
tests := []struct {
name string
prefix string
filename string
expectedS3Key string
}{
{"with prefix", "test", uuid, fmt.Sprintf("%s/%s", "test", uuid)},
{"without prefix", "", uuid, uuid},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var c *Client
var err error
if test.prefix != "" {
c, err = New(nil, nil, WithObjectPrefix(test.prefix))
} else {
c, err = New(nil, nil)
}
assert.Nil(t, err)
s3Key := c.s3Key(test.filename)
assert.Equal(t, test.expectedS3Key, s3Key)
})
}
}

func TestWithObjectPrefix(t *testing.T) {
invalidPrefixes := []string{"../test", "./test", "tes&", "te$t", "testñ", "te@st", "test=", "test;", "test:", "+test", "te st", "te,st", "test?", "te\\st", "test{", "test^", "test}", "te`st", "]test", "test\"", "test>", "test]", "test~", "test<", "te#st", "|test"}
validPrefixes := []string{"test0", "test", "TESt", "te!st", "te-st", "te_st", "te.st", "test*", "'test'", "(test)"}

tests := []struct {
name string
prefixes []string
expectedErr error
}{
{"invalid prefixes", invalidPrefixes, ErrObjectPrefix},
{"valid prefixes", validPrefixes, nil},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for _, prefix := range test.prefixes {
c, err := New(
nil,
nil,
WithObjectPrefix(prefix),
)
if test.expectedErr == nil {
assert.Equal(t, c.objectPrefix, prefix)
} else {
assert.Equal(t, test.expectedErr, err)
}
}
})
}
}

func TestSendMessage(t *testing.T) {
key := new(string)

Expand Down

0 comments on commit c03a75d

Please sign in to comment.