[S3] - fixing s3 destination (#2425)

* fixing s3 destination * fixing s3 destination * making batch optional * fixing bad import
segmentio · Sep 19, 2024 · 4b75a0f · 4b75a0f
1 parent 508182a
commit 4b75a0f
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 12 deletions.
diff --git a/packages/destination-actions/src/destinations/s3/fields.ts b/packages/destination-actions/src/destinations/s3/fields.ts
@@ -170,15 +170,15 @@ export const commonFields: ActionDefinition<Settings>['fields'] = {
     type: 'boolean',
     label: 'Enable Batching',
     description: 'Enable Batching Hidden Field',
-    unsafe_hidden: true,
+    unsafe_hidden: false,
     required: true,
     default: true
   },
   batch_size: {
     label: 'Batch Size',
-    description: 'Maximum number of events to include in each batch. Actual batch sizes may be lower.',
+    description:
+      'Maximum number of events to include in each batch. Actual batch sizes may be lower. Max batch size is 25000.',
     type: 'number',
-    unsafe_hidden: true,
     required: false,
     default: 25000
   },

diff --git a/packages/destination-actions/src/destinations/s3/functions.ts b/packages/destination-actions/src/destinations/s3/functions.ts
@@ -2,8 +2,16 @@ import { Payload } from './syncToS3/generated-types'
 import { Settings } from './generated-types'
 import { Client } from './client'
 import { RawMapping } from './types'
+import { IntegrationError } from '@segment/actions-core'
 
 export async function send(payloads: Payload[], settings: Settings, rawMapping: RawMapping) {
+
+  const batchSize = payloads[0] && typeof payloads[0].batch_size === 'number' ? payloads[0].batch_size : 0
+
+  if(batchSize > 25000) {
+    throw new IntegrationError('Batch size cannot exceed 25000', 'Invalid Payload', 400)
+  }
+
   const headers = Object.keys(rawMapping.columns).map((column) => {
     return snakeCase(column)
   })

diff --git a/packages/destination-actions/src/destinations/s3/syncToS3/index.ts b/packages/destination-actions/src/destinations/s3/syncToS3/index.ts
@@ -3,7 +3,7 @@ import type { Settings } from '../generated-types'
 import type { Payload } from './generated-types'
 import { commonFields } from '../fields'
 import { send } from '../functions'
-import { SingleData, BatchData, RawMapping } from '../types'
+import { Data, RawMapping } from '../types'
 
 const action: ActionDefinition<Settings, Payload> = {
   title: 'Sync to S3',
@@ -12,13 +12,13 @@ const action: ActionDefinition<Settings, Payload> = {
   fields: commonFields,
   perform: async (_, data) => {
     const { payload, settings } = data
-    const rawMapping: RawMapping = (data as unknown as SingleData).rawMapping
+    const rawMapping: RawMapping = (data as unknown as Data).rawMapping
     return send([payload], settings, rawMapping)
   },
   performBatch: async (_, data) => {
     const { payload, settings } = data
-    const rawMapping: RawMapping[] = (data as unknown as BatchData).rawMapping
-    return send(payload, settings, rawMapping[0])
+    const rawMapping: RawMapping = (data as unknown as Data).rawMapping
+    return send(payload, settings, rawMapping)
   }
 }
 

diff --git a/packages/destination-actions/src/destinations/s3/types.ts b/packages/destination-actions/src/destinations/s3/types.ts
@@ -4,14 +4,10 @@ export interface Credentials {
   sessionToken: string
 }
 
-export interface SingleData {
+export interface Data {
   rawMapping: RawMapping
 }
 
-export interface BatchData {
-  rawMapping: RawMapping[]
-}
-
 export interface RawMapping {
   columns: {
     [k: string]: unknown