Merge remote-tracking branch 'origin/main' into responsys-audiences-a…

…s-pets

packages/destination-actions/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@segment/action-destinations",
   "description": "Destination Actions engine and definitions.",
-  "version": "3.327.0",
+  "version": "3.328.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/segmentio/action-destinations",

packages/destination-actions/src/destinations/s3/fields.ts

@@ -144,6 +144,15 @@ export const commonFields: ActionDefinition<Settings>['fields'] = {
     disabledInputMethods: ['variable', 'function', 'enrichment'],
     default: 'audience_action'
   },
+  batch_size_column_name: {
+    label: 'Batch Size Column Name',
+    description:
+      'Specify the column name to store the batch size when the event is sent to S3. Leave blank if no column is required',
+    type: 'string',
+    required: false,
+    disabledInputMethods: ['variable', 'function', 'enrichment'],
+    default: 'batch_size'
+  },
   traits_or_props: {
     label: 'Traits or Props - Hidden Field',
     description: 'Field used to retrieve Audience value',

packages/destination-actions/src/destinations/s3/functions.ts

@@ -8,23 +8,28 @@ export async function send(payloads: Payload[], settings: Settings, rawMapping:
   const batchSize = payloads[0] && typeof payloads[0].batch_size === 'number' ? payloads[0].batch_size : 0
   const delimiter = payloads[0]?.delimiter
   const actionColName = payloads[0]?.audience_action_column_name
+  const batchColName = payloads[0]?.batch_size_column_name
 
-  if (batchSize > 25000) {
-    throw new IntegrationError('Batch size cannot exceed 25000', 'Invalid Payload', 400)
+  if (batchSize > 5000) {
+    throw new IntegrationError('Batch size cannot exceed 5000', 'Invalid Payload', 400)
   }
 
   const headers: ColumnHeader[] = Object.entries(rawMapping.columns)
     .filter(([_, value]) => value !== '')
     .map(([column]) => {
-      return { cleanName: clean(delimiter, column), originalName: column };
-    });
+      return { cleanName: clean(delimiter, column), originalName: column }
+    })
 
   if (actionColName) {
-    headers.push({cleanName: clean(delimiter, actionColName), originalName: actionColName} )
+    headers.push({ cleanName: clean(delimiter, actionColName), originalName: actionColName })
+  }
+
+  if (batchColName) {
+    headers.push({ cleanName: clean(delimiter, batchColName), originalName: batchColName })
   }
 
-  const fileContent = generateFile(payloads, headers, delimiter, actionColName)
-  
+  const fileContent = generateFile(payloads, headers, delimiter, actionColName, batchColName)
+
   const s3Client = new Client(settings.s3_aws_region, settings.iam_role_arn, settings.iam_external_id)
 
   await s3Client.uploadS3(
@@ -55,17 +60,24 @@ function processField(row: string[], value: unknown | undefined) {
   )
 }
 
-export function generateFile(payloads: Payload[], headers: ColumnHeader[], delimiter: string, actionColName?: string): string {
-  const rows: string[] = [];
-  rows.push(`${headers.map(header => header.cleanName).join(delimiter === 'tab' ? '\t' : delimiter)}\n`)
+export function generateFile(
+  payloads: Payload[],
+  headers: ColumnHeader[],
+  delimiter: string,
+  actionColName?: string,
+  batchColName?: string
+): string {
+  const rows: string[] = []
+  rows.push(`${headers.map((header) => header.cleanName).join(delimiter === 'tab' ? '\t' : delimiter)}\n`)
 
   payloads.forEach((payload, index) => {
     const isLastRow = index === payloads.length - 1
     const row: string[] = []
-
     headers.forEach((header) => {
       if (header.originalName === actionColName) {
         processField(row, getAudienceAction(payload))
+      } else if (header.originalName === batchColName) {
+        processField(row, payloads.length)
       } else {
         processField(row, payload.columns[header.originalName])
       }
@@ -86,4 +98,4 @@ export function getAudienceAction(payload: Payload): boolean | undefined {
   }
 
   return (payload?.traits_or_props as Record<string, boolean> | undefined)?.[payload.computation_key] ?? undefined
-}
+}

packages/destination-actions/src/destinations/s3/syncToS3/__tests__/index.test.ts

@@ -97,6 +97,7 @@ describe('generateFile', () => {
         'Custom Field 2': 'Custom Field Value 2'
       },
       audience_action_column_name: 'audience_action',
+      batch_size_column_name: 'batch_size',
       traits_or_props: {
         audience_name_1: true,
         prop_str: 'Hello String!',
@@ -135,13 +136,14 @@ describe('generateFile', () => {
     { cleanName: 'audience_space_id', originalName: 'audience_space_id' },
     { cleanName: 'Custom Field 1', originalName: 'Custom Field 1' },
     { cleanName: 'Custom Field 2', originalName: 'Custom Field 2' },
-    { cleanName: 'audience_action', originalName: 'audience_action' }
+    { cleanName: 'audience_action', originalName: 'audience_action' },
+    { cleanName: 'batch_size', originalName: 'batch_size' }
   ]
 
-  const output = `event_name,event_type,user_id,anonymous_id,email,properties,traits,context,timestamp,message_id,integrations,audience_name,audience_id,audience_space_id,Custom Field 1,Custom Field 2,audience_action\n"Custom Event 1","track","user_id_1","anonymous_id_1","[email protected]","{""prop_str"":""Hello String!"",""prop_num"":123.45,""prop_bool"":true,""prop_datetime"":""2024-01-08T13:52:50.212Z"",""prop_date"":""2024-01-08"",""prop_obj"":{""key1"":""value1"",""key2"":""value2""},""prop_arr"":[""value1"",""value2""],""custom_field_1"":""Custom Field Value 1"",""custom_field_2"":""Custom Field Value 2""}","{""first_name"":""John"",""last_name"":""Doe"",""email"":""[email protected]""}","{""traits"":{""first_name"":""John"",""last_name"":""Doe"",""email"":""[email protected]""},""personas"":{""computation_key"":""audience_name_1"",""computation_id"":""audience_id_1"",""space_id"":""space_id_1""}}","2024-01-08T13:52:50.212Z","aaa-bbb-ccc","{}","audience_name_1","audience_id_1","space_id_1","Custom Field Value 1","Custom Field Value 2","true"`
+  const output = `event_name,event_type,user_id,anonymous_id,email,properties,traits,context,timestamp,message_id,integrations,audience_name,audience_id,audience_space_id,Custom Field 1,Custom Field 2,audience_action,batch_size\n"Custom Event 1","track","user_id_1","anonymous_id_1","[email protected]","{""prop_str"":""Hello String!"",""prop_num"":123.45,""prop_bool"":true,""prop_datetime"":""2024-01-08T13:52:50.212Z"",""prop_date"":""2024-01-08"",""prop_obj"":{""key1"":""value1"",""key2"":""value2""},""prop_arr"":[""value1"",""value2""],""custom_field_1"":""Custom Field Value 1"",""custom_field_2"":""Custom Field Value 2""}","{""first_name"":""John"",""last_name"":""Doe"",""email"":""[email protected]""}","{""traits"":{""first_name"":""John"",""last_name"":""Doe"",""email"":""[email protected]""},""personas"":{""computation_key"":""audience_name_1"",""computation_id"":""audience_id_1"",""space_id"":""space_id_1""}}","2024-01-08T13:52:50.212Z","aaa-bbb-ccc","{}","audience_name_1","audience_id_1","space_id_1","Custom Field Value 1","Custom Field Value 2","true","1"`
 
   it('should generate a CSV file with correct content', () => {
-    const result = generateFile(payloads, headers, ',', 'audience_action')
+    const result = generateFile(payloads, headers, ',', 'audience_action', 'batch_size')
     expect(result).toEqual(output)
   })
 })