[backend] Improve csv feed parse to allow partial reject (#6151)

OpenCTI-Platform · Feb 28, 2024 · 79bc968 · 79bc968
1 parent cde7fe5
commit 79bc968
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 13 deletions.
diff --git a/...-platform/opencti-front/src/private/components/data/ingestionCsv/IngestionCsvCreation.tsx b/...-platform/opencti-front/src/private/components/data/ingestionCsv/IngestionCsvCreation.tsx
@@ -181,6 +181,7 @@ const IngestionCsvCreation: FunctionComponent<IngestionCsvCreationProps> = ({ pa
                   label: t_i18n('Import from date (empty = all Csv possible items)'),
                   variant: 'standard',
                   fullWidth: true,
+                  style: fieldSpacingContainerStyle,
                 }}
               />
               <Field

diff --git a/opencti-platform/opencti-front/src/private/components/data/ingestionCsv/IngestionCsvLine.tsx b/opencti-platform/opencti-front/src/private/components/data/ingestionCsv/IngestionCsvLine.tsx
@@ -57,6 +57,7 @@ const ingestionCsvLineFragment = graphql`
     uri
     ingestion_running
     current_state_date
+    current_state_hash
   }
 `;
 
@@ -102,7 +103,7 @@ export const IngestionCsvLineComponent: FunctionComponent<IngestionCsvLineProps>
               className={classes.bodyItem}
               style={{ width: dataColumns.current_state_date.width }}
             >
-              {nsdt(data.current_state_date)}
+              {data.current_state_date ? nsdt(data.current_state_date) : data.current_state_hash}
             </div>
           </div>
         }

diff --git a/opencti-platform/opencti-graphql/src/parser/csv-bundler.ts b/opencti-platform/opencti-graphql/src/parser/csv-bundler.ts
@@ -13,6 +13,8 @@ import { parsingProcess } from './csv-parser';
 import { isStixDomainObjectContainer } from '../schema/stixDomainObject';
 import { objects } from '../schema/stixRefRelationship';
 import { isEmptyField } from '../database/utils';
+import { logApp } from '../config/conf';
+import { UnknownError } from '../config/errors';
 
 const validateInput = async (context: AuthContext, user: AuthUser, inputs: Record<string, InputType>[]) => {
   await Promise.all(inputs.map(async (input) => {
@@ -46,16 +48,20 @@ export const bundleProcess = async (
       if (skipLine) {
         skipLine = false;
       } else if (!isEmptyLine) {
-        // Compute input by representation
-        const inputs = await mappingProcess(context, user, sanitizedMapper, record);
-        // Remove inline elements
-        const withoutInlineInputs = inputs.filter((input) => !inlineEntityTypes.includes(input.entity_type as string));
-        // Validate elements
-        await validateInput(context, user, withoutInlineInputs);
-        // Transform entity to stix
-        const stixObjects = withoutInlineInputs.map((input) => convertStoreToStix(input as unknown as StoreCommon));
-        // Add to bundle
-        bundleBuilder.addObjects(stixObjects);
+        try {
+          // Compute input by representation
+          const inputs = await mappingProcess(context, user, sanitizedMapper, record);
+          // Remove inline elements
+          const withoutInlineInputs = inputs.filter((input) => !inlineEntityTypes.includes(input.entity_type as string));
+          // Validate elements
+          await validateInput(context, user, withoutInlineInputs);
+          // Transform entity to stix
+          const stixObjects = withoutInlineInputs.map((input) => convertStoreToStix(input as unknown as StoreCommon));
+          // Add to bundle
+          bundleBuilder.addObjects(stixObjects);
+        } catch (e) {
+          logApp.error(UnknownError('Error CSV mapping record', { cause: e }));
+        }
       }
     })));
   }

diff --git a/opencti-platform/opencti-graphql/src/parser/csv-parser.ts b/opencti-platform/opencti-graphql/src/parser/csv-parser.ts
@@ -4,6 +4,8 @@ import fs from 'node:fs';
 import { parse } from 'csv-parse/sync';
 import * as readline from 'readline';
 import { Readable } from 'stream';
+import { logApp } from '../config/conf';
+import { isNotEmptyField } from '../database/utils';
 
 const parserOption = (delimiter: string, comment: string) => ({
   delimiter,
@@ -49,8 +51,21 @@ export const parseCsvBufferContent = (buffer: Buffer, delimiter: string, skipLin
       })
       .on('end', () => {
         try {
-          const parsing = parse(Buffer.concat(chunks).toString('utf8'), parserOption(delimiter, skipLineChar));
-          resolve(parsing);
+          const parsingResult = [];
+          const data = Buffer.concat(chunks).toString('utf8');
+          const lines = data.split('\n');
+          for (let index = 0; index < lines.length; index += 1) {
+            const line = lines[index];
+            try {
+              const parsing = parse(line, parserOption(delimiter, skipLineChar));
+              if (isNotEmptyField(parsing[0])) {
+                parsingResult.push(parsing[0]);
+              }
+            } catch (err) {
+              logApp.error('Error parsing CSV line', { line, cause: err });
+            }
+          }
+          resolve(parsingResult);
         } catch (error) {
           reject(error);
         }