From ba9aaf13a7657a5a5bff0bab5deca1c7e17d3d3f Mon Sep 17 00:00:00 2001 From: Jeremy Cloarec Date: Fri, 27 Sep 2024 16:59:00 +0200 Subject: [PATCH] [backend] handle elIndex in draft --- .../opencti-graphql/src/database/engine.js | 17 +++++++++++++---- .../opencti-graphql/src/database/file-search.js | 4 ++-- .../opencti-graphql/src/database/utils.js | 2 +- .../src/domain/backgroundTask-common.js | 2 +- .../src/domain/backgroundTask.js | 4 ++-- .../opencti-graphql/src/domain/retentionRule.js | 4 ++-- .../opencti-graphql/src/domain/stream.js | 2 +- .../opencti-graphql/src/domain/taxii.js | 2 +- .../opencti-graphql/src/domain/work.js | 2 +- .../internal/document/document-domain.ts | 4 ++-- .../01-database/elasticSearch-test.js | 2 +- .../04-manager/connectorManager-test.ts | 2 +- 12 files changed, 28 insertions(+), 19 deletions(-) diff --git a/opencti-platform/opencti-graphql/src/database/engine.js b/opencti-platform/opencti-graphql/src/database/engine.js index 0035d590f2915..03730c31aab4b 100644 --- a/opencti-platform/opencti-graphql/src/database/engine.js +++ b/opencti-platform/opencti-graphql/src/database/engine.js @@ -17,6 +17,7 @@ import { INDEX_DELETED_OBJECTS, INDEX_DRAFT, INDEX_INTERNAL_OBJECTS, + INDEX_INTERNAL_RELATIONSHIPS, inferIndexFromConceptType, isEmptyField, isInferredIndex, @@ -3269,17 +3270,25 @@ export const elBulk = async (args) => { }); }; /* v8 ignore next */ -export const elIndex = async (indexName, documentBody, opts = {}) => { +export const elIndex = async (context, user, indexName, documentBody, opts = {}) => { const { refresh = true, pipeline } = opts; const documentId = documentBody.internal_id; const entityType = documentBody.entity_type ? documentBody.entity_type : ''; logApp.debug(`[SEARCH] index > ${entityType} ${documentId} in ${indexName}`, { documentBody }); + const draftContext = inDraftContext(context, user); + let indexNameToUse = indexName; + let documentToIndex = documentBody; + if (draftContext) { + if (indexName === INDEX_INTERNAL_OBJECTS || indexName === INDEX_INTERNAL_RELATIONSHIPS) throw new Error('Cannot index internal element in draft context'); + indexNameToUse = INDEX_DRAFT; + documentToIndex = { ...documentBody, draft_ids: [draftContext] }; + } let indexParams = { - index: indexName, - id: documentBody.internal_id, + index: indexNameToUse, + id: documentToIndex.internal_id, refresh, timeout: '60m', - body: R.dissoc('_index', documentBody), + body: R.dissoc('_index', documentToIndex), }; if (pipeline) { indexParams = { ...indexParams, pipeline }; diff --git a/opencti-platform/opencti-graphql/src/database/file-search.js b/opencti-platform/opencti-graphql/src/database/file-search.js index b074111e2402d..f941e1126aad5 100644 --- a/opencti-platform/opencti-graphql/src/database/file-search.js +++ b/opencti-platform/opencti-graphql/src/database/file-search.js @@ -63,13 +63,13 @@ export const elIndexFiles = async (context, user, files) => { }; const documentBody = buildIndexFileBody(internal_id, fileObject, entity); try { - await elIndex(INDEX_FILES, documentBody, { pipeline: 'attachment' }); + await elIndex(context, user, INDEX_FILES, documentBody, { pipeline: 'attachment' }); } catch (err) { // catch & log error logApp.error('Error on file indexing', { message: err.message, causeStack: err.data?.cause?.stack, stack: err.stack, file_id }); // try to index without file content const documentWithoutFileData = R.dissoc('file_data', documentBody); - await elIndex(INDEX_FILES, documentWithoutFileData).catch((e) => { + await elIndex(context, user, INDEX_FILES, documentWithoutFileData).catch((e) => { logApp.error('Error in fallback file indexing', { message: e.message, cause: e.cause, file_id }); }); } diff --git a/opencti-platform/opencti-graphql/src/database/utils.js b/opencti-platform/opencti-graphql/src/database/utils.js index 046bd0d94a8be..350cb01f8b2bb 100644 --- a/opencti-platform/opencti-graphql/src/database/utils.js +++ b/opencti-platform/opencti-graphql/src/database/utils.js @@ -49,7 +49,7 @@ const INDEX_STIX_CYBER_OBSERVABLES = `${ES_INDEX_PREFIX}_stix_cyber_observables` export const READ_INDEX_STIX_CYBER_OBSERVABLES = `${INDEX_STIX_CYBER_OBSERVABLES}*`; // Relations -const INDEX_INTERNAL_RELATIONSHIPS = `${ES_INDEX_PREFIX}_internal_relationships`; +export const INDEX_INTERNAL_RELATIONSHIPS = `${ES_INDEX_PREFIX}_internal_relationships`; export const READ_INDEX_INTERNAL_RELATIONSHIPS = `${INDEX_INTERNAL_RELATIONSHIPS}*`; export const INDEX_STIX_CORE_RELATIONSHIPS = `${ES_INDEX_PREFIX}_stix_core_relationships`; export const READ_INDEX_STIX_CORE_RELATIONSHIPS = `${INDEX_STIX_CORE_RELATIONSHIPS}*`; diff --git a/opencti-platform/opencti-graphql/src/domain/backgroundTask-common.js b/opencti-platform/opencti-graphql/src/domain/backgroundTask-common.js index e10b61e34d686..2ca0dafd4560e 100644 --- a/opencti-platform/opencti-graphql/src/domain/backgroundTask-common.js +++ b/opencti-platform/opencti-graphql/src/domain/backgroundTask-common.js @@ -299,6 +299,6 @@ export const createListTask = async (context, user, input) => { message: 'creates `background task`', context_data: { entity_type: ENTITY_TYPE_BACKGROUND_TASK, input: listTask } }); - await elIndex(INDEX_INTERNAL_OBJECTS, listTask); + await elIndex(context, user, INDEX_INTERNAL_OBJECTS, listTask); return listTask; }; diff --git a/opencti-platform/opencti-graphql/src/domain/backgroundTask.js b/opencti-platform/opencti-graphql/src/domain/backgroundTask.js index 8c562dd2e2205..c3d9783507b10 100644 --- a/opencti-platform/opencti-graphql/src/domain/backgroundTask.js +++ b/opencti-platform/opencti-graphql/src/domain/backgroundTask.js @@ -121,7 +121,7 @@ export const createRuleTask = async (context, user, ruleDefinition, input) => { const countExpected = queryData.pageInfo.globalCount; const task = createDefaultTask(user, input, TASK_TYPE_RULE, countExpected); const ruleTask = { ...task, rule, enable }; - await elIndex(INDEX_INTERNAL_OBJECTS, ruleTask); + await elIndex(context, user, INDEX_INTERNAL_OBJECTS, ruleTask); return ruleTask; }; @@ -146,7 +146,7 @@ export const createQueryTask = async (context, user, input) => { message: 'creates `background task`', context_data: { entity_type: ENTITY_TYPE_BACKGROUND_TASK, input: queryTask } }); - await elIndex(INDEX_INTERNAL_OBJECTS, queryTask); + await elIndex(context, user, INDEX_INTERNAL_OBJECTS, queryTask); return queryTask; }; diff --git a/opencti-platform/opencti-graphql/src/domain/retentionRule.js b/opencti-platform/opencti-graphql/src/domain/retentionRule.js index bbe04d2f27fa9..c06ddc935f500 100644 --- a/opencti-platform/opencti-graphql/src/domain/retentionRule.js +++ b/opencti-platform/opencti-graphql/src/domain/retentionRule.js @@ -52,7 +52,7 @@ export const checkRetentionRule = async (context, input) => { }; // input { name, filters } -export const createRetentionRule = async (_, user, input) => { +export const createRetentionRule = async (context, user, input) => { // filters must be a valid json let { filters } = input; if (!filters) { // filters is undefined or an empty string @@ -77,7 +77,7 @@ export const createRetentionRule = async (_, user, input) => { filters, ...input, }; - await elIndex(INDEX_INTERNAL_OBJECTS, retentionRule); + await elIndex(context, user, INDEX_INTERNAL_OBJECTS, retentionRule); await publishUserAction({ user, event_type: 'mutation', diff --git a/opencti-platform/opencti-graphql/src/domain/stream.js b/opencti-platform/opencti-graphql/src/domain/stream.js index 6b5744833862b..957ba1305e37f 100644 --- a/opencti-platform/opencti-graphql/src/domain/stream.js +++ b/opencti-platform/opencti-graphql/src/domain/stream.js @@ -33,7 +33,7 @@ export const createStreamCollection = async (context, user, input) => { authorized_authorities: [TAXIIAPI_SETCOLLECTIONS], ...input }; - await elIndex(INDEX_INTERNAL_OBJECTS, data); + await elIndex(context, user, INDEX_INTERNAL_OBJECTS, data); await publishUserAction({ user, event_type: 'mutation', diff --git a/opencti-platform/opencti-graphql/src/domain/taxii.js b/opencti-platform/opencti-graphql/src/domain/taxii.js index 39cc3cfb6ff38..3c4b6c730391d 100644 --- a/opencti-platform/opencti-graphql/src/domain/taxii.js +++ b/opencti-platform/opencti-graphql/src/domain/taxii.js @@ -29,7 +29,7 @@ export const createTaxiiCollection = async (context, user, input) => { authorized_authorities: [TAXIIAPI_SETCOLLECTIONS], ...input, }; - await elIndex(INDEX_INTERNAL_OBJECTS, data); + await elIndex(context, user, INDEX_INTERNAL_OBJECTS, data); await publishUserAction({ user, event_type: 'mutation', diff --git a/opencti-platform/opencti-graphql/src/domain/work.js b/opencti-platform/opencti-graphql/src/domain/work.js index a809900b906a6..58a609c380511 100644 --- a/opencti-platform/opencti-graphql/src/domain/work.js +++ b/opencti-platform/opencti-graphql/src/domain/work.js @@ -195,7 +195,7 @@ export const createWork = async (context, user, connector, friendlyName, sourceI errors: [], [buildRefRelationKey(RELATION_OBJECT_MARKING)]: [...fileMarkings] }; - await elIndex(INDEX_HISTORY, work); + await elIndex(context, user, INDEX_HISTORY, work); return loadWorkById(context, user, workId); }; diff --git a/opencti-platform/opencti-graphql/src/modules/internal/document/document-domain.ts b/opencti-platform/opencti-graphql/src/modules/internal/document/document-domain.ts index 7feb0179f4c3f..a4a78b381e883 100644 --- a/opencti-platform/opencti-graphql/src/modules/internal/document/document-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/internal/document/document-domain.ts @@ -57,9 +57,9 @@ export const indexFileToDocument = async (context: AuthContext, file: any) => { const internalFile = await storeLoadById(context, SYSTEM_USER, data.internal_id, ENTITY_TYPE_INTERNAL_FILE); if (internalFile) { // update existing internalFile (if file has been saved in another index) - return elIndex(internalFile._index, data); + return elIndex(context, context.user, internalFile._index, data); } - return elIndex(INDEX_INTERNAL_OBJECTS, data); + return elIndex(context, context.user, INDEX_INTERNAL_OBJECTS, data); }; export const deleteDocumentIndex = async (context: AuthContext, user: AuthUser, id: string) => { diff --git a/opencti-platform/opencti-graphql/tests/02-integration/01-database/elasticSearch-test.js b/opencti-platform/opencti-graphql/tests/02-integration/01-database/elasticSearch-test.js index da2ab4801ba2c..b34908960f37e 100644 --- a/opencti-platform/opencti-graphql/tests/02-integration/01-database/elasticSearch-test.js +++ b/opencti-platform/opencti-graphql/tests/02-integration/01-database/elasticSearch-test.js @@ -92,7 +92,7 @@ describe('Elasticsearch document loader', () => { entity_type: 'Campaign', parent_types: ['Campaign', 'Stix-Domain-Object', 'Stix-Core-Object', 'Stix-Object', 'Basic-Object'], }; - const indexedData = await elIndex('test_index', documentBody); + const indexedData = await elIndex({}, {}, 'test_index', documentBody); expect(indexedData).toEqual(documentBody); const documentWithIndex = R.assoc('_index', 'test_index-000001', documentBody); // Load by internal Id diff --git a/opencti-platform/opencti-graphql/tests/02-integration/04-manager/connectorManager-test.ts b/opencti-platform/opencti-graphql/tests/02-integration/04-manager/connectorManager-test.ts index f198a5d48f42f..3213c5876be74 100644 --- a/opencti-platform/opencti-graphql/tests/02-integration/04-manager/connectorManager-test.ts +++ b/opencti-platform/opencti-graphql/tests/02-integration/04-manager/connectorManager-test.ts @@ -52,7 +52,7 @@ describe('Old work of connector cleanup test', () => { user_id: '', }; - await elIndex(INDEX_HISTORY, eightDaysAgoWork as Work); + await elIndex({}, {}, INDEX_HISTORY, eightDaysAgoWork as Work); const workCreated = await findWorkById(testContext, ADMIN_USER, workId) as unknown as Work; expect(workCreated.id).toBeDefined(); expect(workCreated.status).toBe(status);