diff --git a/packages/destination-actions/package.json b/packages/destination-actions/package.json index 26e7322ebe..93a6c3f915 100644 --- a/packages/destination-actions/package.json +++ b/packages/destination-actions/package.json @@ -1,7 +1,7 @@ { "name": "@segment/action-destinations", "description": "Destination Actions engine and definitions.", - "version": "3.327.0", + "version": "3.328.0", "repository": { "type": "git", "url": "https://github.com/segmentio/action-destinations", diff --git a/packages/destination-actions/src/destinations/s3/fields.ts b/packages/destination-actions/src/destinations/s3/fields.ts index db1edf01bc..feda00d79d 100644 --- a/packages/destination-actions/src/destinations/s3/fields.ts +++ b/packages/destination-actions/src/destinations/s3/fields.ts @@ -144,6 +144,15 @@ export const commonFields: ActionDefinition['fields'] = { disabledInputMethods: ['variable', 'function', 'enrichment'], default: 'audience_action' }, + batch_size_column_name: { + label: 'Batch Size Column Name', + description: + 'Specify the column name to store the batch size when the event is sent to S3. Leave blank if no column is required', + type: 'string', + required: false, + disabledInputMethods: ['variable', 'function', 'enrichment'], + default: 'batch_size' + }, traits_or_props: { label: 'Traits or Props - Hidden Field', description: 'Field used to retrieve Audience value', diff --git a/packages/destination-actions/src/destinations/s3/functions.ts b/packages/destination-actions/src/destinations/s3/functions.ts index 535e28a05d..f63ac01343 100644 --- a/packages/destination-actions/src/destinations/s3/functions.ts +++ b/packages/destination-actions/src/destinations/s3/functions.ts @@ -8,23 +8,28 @@ export async function send(payloads: Payload[], settings: Settings, rawMapping: const batchSize = payloads[0] && typeof payloads[0].batch_size === 'number' ? payloads[0].batch_size : 0 const delimiter = payloads[0]?.delimiter const actionColName = payloads[0]?.audience_action_column_name + const batchColName = payloads[0]?.batch_size_column_name - if (batchSize > 25000) { - throw new IntegrationError('Batch size cannot exceed 25000', 'Invalid Payload', 400) + if (batchSize > 5000) { + throw new IntegrationError('Batch size cannot exceed 5000', 'Invalid Payload', 400) } const headers: ColumnHeader[] = Object.entries(rawMapping.columns) .filter(([_, value]) => value !== '') .map(([column]) => { - return { cleanName: clean(delimiter, column), originalName: column }; - }); + return { cleanName: clean(delimiter, column), originalName: column } + }) if (actionColName) { - headers.push({cleanName: clean(delimiter, actionColName), originalName: actionColName} ) + headers.push({ cleanName: clean(delimiter, actionColName), originalName: actionColName }) + } + + if (batchColName) { + headers.push({ cleanName: clean(delimiter, batchColName), originalName: batchColName }) } - const fileContent = generateFile(payloads, headers, delimiter, actionColName) - + const fileContent = generateFile(payloads, headers, delimiter, actionColName, batchColName) + const s3Client = new Client(settings.s3_aws_region, settings.iam_role_arn, settings.iam_external_id) await s3Client.uploadS3( @@ -55,17 +60,24 @@ function processField(row: string[], value: unknown | undefined) { ) } -export function generateFile(payloads: Payload[], headers: ColumnHeader[], delimiter: string, actionColName?: string): string { - const rows: string[] = []; - rows.push(`${headers.map(header => header.cleanName).join(delimiter === 'tab' ? '\t' : delimiter)}\n`) +export function generateFile( + payloads: Payload[], + headers: ColumnHeader[], + delimiter: string, + actionColName?: string, + batchColName?: string +): string { + const rows: string[] = [] + rows.push(`${headers.map((header) => header.cleanName).join(delimiter === 'tab' ? '\t' : delimiter)}\n`) payloads.forEach((payload, index) => { const isLastRow = index === payloads.length - 1 const row: string[] = [] - headers.forEach((header) => { if (header.originalName === actionColName) { processField(row, getAudienceAction(payload)) + } else if (header.originalName === batchColName) { + processField(row, payloads.length) } else { processField(row, payload.columns[header.originalName]) } @@ -86,4 +98,4 @@ export function getAudienceAction(payload: Payload): boolean | undefined { } return (payload?.traits_or_props as Record | undefined)?.[payload.computation_key] ?? undefined -} \ No newline at end of file +} diff --git a/packages/destination-actions/src/destinations/s3/syncToS3/__tests__/index.test.ts b/packages/destination-actions/src/destinations/s3/syncToS3/__tests__/index.test.ts index 044913188a..3ff10e6228 100644 --- a/packages/destination-actions/src/destinations/s3/syncToS3/__tests__/index.test.ts +++ b/packages/destination-actions/src/destinations/s3/syncToS3/__tests__/index.test.ts @@ -97,6 +97,7 @@ describe('generateFile', () => { 'Custom Field 2': 'Custom Field Value 2' }, audience_action_column_name: 'audience_action', + batch_size_column_name: 'batch_size', traits_or_props: { audience_name_1: true, prop_str: 'Hello String!', @@ -135,13 +136,14 @@ describe('generateFile', () => { { cleanName: 'audience_space_id', originalName: 'audience_space_id' }, { cleanName: 'Custom Field 1', originalName: 'Custom Field 1' }, { cleanName: 'Custom Field 2', originalName: 'Custom Field 2' }, - { cleanName: 'audience_action', originalName: 'audience_action' } + { cleanName: 'audience_action', originalName: 'audience_action' }, + { cleanName: 'batch_size', originalName: 'batch_size' } ] - const output = `event_name,event_type,user_id,anonymous_id,email,properties,traits,context,timestamp,message_id,integrations,audience_name,audience_id,audience_space_id,Custom Field 1,Custom Field 2,audience_action\n"Custom Event 1","track","user_id_1","anonymous_id_1","test@test.com","{""prop_str"":""Hello String!"",""prop_num"":123.45,""prop_bool"":true,""prop_datetime"":""2024-01-08T13:52:50.212Z"",""prop_date"":""2024-01-08"",""prop_obj"":{""key1"":""value1"",""key2"":""value2""},""prop_arr"":[""value1"",""value2""],""custom_field_1"":""Custom Field Value 1"",""custom_field_2"":""Custom Field Value 2""}","{""first_name"":""John"",""last_name"":""Doe"",""email"":""test@test.com""}","{""traits"":{""first_name"":""John"",""last_name"":""Doe"",""email"":""test@test.com""},""personas"":{""computation_key"":""audience_name_1"",""computation_id"":""audience_id_1"",""space_id"":""space_id_1""}}","2024-01-08T13:52:50.212Z","aaa-bbb-ccc","{}","audience_name_1","audience_id_1","space_id_1","Custom Field Value 1","Custom Field Value 2","true"` + const output = `event_name,event_type,user_id,anonymous_id,email,properties,traits,context,timestamp,message_id,integrations,audience_name,audience_id,audience_space_id,Custom Field 1,Custom Field 2,audience_action,batch_size\n"Custom Event 1","track","user_id_1","anonymous_id_1","test@test.com","{""prop_str"":""Hello String!"",""prop_num"":123.45,""prop_bool"":true,""prop_datetime"":""2024-01-08T13:52:50.212Z"",""prop_date"":""2024-01-08"",""prop_obj"":{""key1"":""value1"",""key2"":""value2""},""prop_arr"":[""value1"",""value2""],""custom_field_1"":""Custom Field Value 1"",""custom_field_2"":""Custom Field Value 2""}","{""first_name"":""John"",""last_name"":""Doe"",""email"":""test@test.com""}","{""traits"":{""first_name"":""John"",""last_name"":""Doe"",""email"":""test@test.com""},""personas"":{""computation_key"":""audience_name_1"",""computation_id"":""audience_id_1"",""space_id"":""space_id_1""}}","2024-01-08T13:52:50.212Z","aaa-bbb-ccc","{}","audience_name_1","audience_id_1","space_id_1","Custom Field Value 1","Custom Field Value 2","true","1"` it('should generate a CSV file with correct content', () => { - const result = generateFile(payloads, headers, ',', 'audience_action') + const result = generateFile(payloads, headers, ',', 'audience_action', 'batch_size') expect(result).toEqual(output) }) }) diff --git a/packages/destination-actions/src/destinations/s3/syncToS3/generated-types.ts b/packages/destination-actions/src/destinations/s3/syncToS3/generated-types.ts index 2196303b98..d9b8ab0f58 100644 --- a/packages/destination-actions/src/destinations/s3/syncToS3/generated-types.ts +++ b/packages/destination-actions/src/destinations/s3/syncToS3/generated-types.ts @@ -75,6 +75,10 @@ export interface Payload { * Name of the column that will contain the action for the audience. true if the user is in the audience, false if not. */ audience_action_column_name?: string + /** + * Specify the column name to store the batch size when the event is sent to S3. Leave blank if no column is required + */ + batch_size_column_name?: string /** * Field used to retrieve Audience value */