From 79bc9681561f03a9f8df494654a1225acda27d28 Mon Sep 17 00:00:00 2001 From: Julien Richard Date: Wed, 28 Feb 2024 10:46:19 +0100 Subject: [PATCH] [backend] Improve csv feed parse to allow partial reject (#6151) --- .../ingestionCsv/IngestionCsvCreation.tsx | 1 + .../data/ingestionCsv/IngestionCsvLine.tsx | 3 ++- .../opencti-graphql/src/parser/csv-bundler.ts | 26 ++++++++++++------- .../opencti-graphql/src/parser/csv-parser.ts | 19 ++++++++++++-- 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/opencti-platform/opencti-front/src/private/components/data/ingestionCsv/IngestionCsvCreation.tsx b/opencti-platform/opencti-front/src/private/components/data/ingestionCsv/IngestionCsvCreation.tsx index 00db4b117b79..d128c00ea314 100644 --- a/opencti-platform/opencti-front/src/private/components/data/ingestionCsv/IngestionCsvCreation.tsx +++ b/opencti-platform/opencti-front/src/private/components/data/ingestionCsv/IngestionCsvCreation.tsx @@ -181,6 +181,7 @@ const IngestionCsvCreation: FunctionComponent = ({ pa label: t_i18n('Import from date (empty = all Csv possible items)'), variant: 'standard', fullWidth: true, + style: fieldSpacingContainerStyle, }} /> className={classes.bodyItem} style={{ width: dataColumns.current_state_date.width }} > - {nsdt(data.current_state_date)} + {data.current_state_date ? nsdt(data.current_state_date) : data.current_state_hash} } diff --git a/opencti-platform/opencti-graphql/src/parser/csv-bundler.ts b/opencti-platform/opencti-graphql/src/parser/csv-bundler.ts index 86c030ac21c2..2715922dff33 100644 --- a/opencti-platform/opencti-graphql/src/parser/csv-bundler.ts +++ b/opencti-platform/opencti-graphql/src/parser/csv-bundler.ts @@ -13,6 +13,8 @@ import { parsingProcess } from './csv-parser'; import { isStixDomainObjectContainer } from '../schema/stixDomainObject'; import { objects } from '../schema/stixRefRelationship'; import { isEmptyField } from '../database/utils'; +import { logApp } from '../config/conf'; +import { UnknownError } from '../config/errors'; const validateInput = async (context: AuthContext, user: AuthUser, inputs: Record[]) => { await Promise.all(inputs.map(async (input) => { @@ -46,16 +48,20 @@ export const bundleProcess = async ( if (skipLine) { skipLine = false; } else if (!isEmptyLine) { - // Compute input by representation - const inputs = await mappingProcess(context, user, sanitizedMapper, record); - // Remove inline elements - const withoutInlineInputs = inputs.filter((input) => !inlineEntityTypes.includes(input.entity_type as string)); - // Validate elements - await validateInput(context, user, withoutInlineInputs); - // Transform entity to stix - const stixObjects = withoutInlineInputs.map((input) => convertStoreToStix(input as unknown as StoreCommon)); - // Add to bundle - bundleBuilder.addObjects(stixObjects); + try { + // Compute input by representation + const inputs = await mappingProcess(context, user, sanitizedMapper, record); + // Remove inline elements + const withoutInlineInputs = inputs.filter((input) => !inlineEntityTypes.includes(input.entity_type as string)); + // Validate elements + await validateInput(context, user, withoutInlineInputs); + // Transform entity to stix + const stixObjects = withoutInlineInputs.map((input) => convertStoreToStix(input as unknown as StoreCommon)); + // Add to bundle + bundleBuilder.addObjects(stixObjects); + } catch (e) { + logApp.error(UnknownError('Error CSV mapping record', { cause: e })); + } } }))); } diff --git a/opencti-platform/opencti-graphql/src/parser/csv-parser.ts b/opencti-platform/opencti-graphql/src/parser/csv-parser.ts index ec03bee84617..ca496eae038c 100644 --- a/opencti-platform/opencti-graphql/src/parser/csv-parser.ts +++ b/opencti-platform/opencti-graphql/src/parser/csv-parser.ts @@ -4,6 +4,8 @@ import fs from 'node:fs'; import { parse } from 'csv-parse/sync'; import * as readline from 'readline'; import { Readable } from 'stream'; +import { logApp } from '../config/conf'; +import { isNotEmptyField } from '../database/utils'; const parserOption = (delimiter: string, comment: string) => ({ delimiter, @@ -49,8 +51,21 @@ export const parseCsvBufferContent = (buffer: Buffer, delimiter: string, skipLin }) .on('end', () => { try { - const parsing = parse(Buffer.concat(chunks).toString('utf8'), parserOption(delimiter, skipLineChar)); - resolve(parsing); + const parsingResult = []; + const data = Buffer.concat(chunks).toString('utf8'); + const lines = data.split('\n'); + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]; + try { + const parsing = parse(line, parserOption(delimiter, skipLineChar)); + if (isNotEmptyField(parsing[0])) { + parsingResult.push(parsing[0]); + } + } catch (err) { + logApp.error('Error parsing CSV line', { line, cause: err }); + } + } + resolve(parsingResult); } catch (error) { reject(error); }