diff --git a/opencti-platform/opencti-graphql/src/database/engine.js b/opencti-platform/opencti-graphql/src/database/engine.js index c85dea47db61..ae1a532706cd 100644 --- a/opencti-platform/opencti-graphql/src/database/engine.js +++ b/opencti-platform/opencti-graphql/src/database/engine.js @@ -2286,17 +2286,17 @@ export const elPaginate = async (context, user, indexName, options = {}) => { } ); }; -export const elList = async (context, user, indexName, options = {}, noFiltersChecking = false) => { - const convertedFilters = (noFiltersChecking || !options.filters) - ? options.filters - : checkAndConvertFilters(options.filters); - const { first = MAX_SEARCH_SIZE, infinite = false } = options; +export const elList = async (context, user, indexName, opts = {}) => { + const { first = MAX_SEARCH_SIZE, maxSize = undefined } = opts; + const { filters, noFiltersChecking = false } = opts; + const convertedFilters = noFiltersChecking ? filters : checkAndConvertFilters(filters); + let emitSize = 0; let hasNextPage = true; let continueProcess = true; - let searchAfter = options.after; + let searchAfter = opts.after; const listing = []; const publish = async (elements) => { - const { callback } = options; + const { callback } = opts; if (callback) { const callbackResult = await callback(elements); continueProcess = callbackResult === true || callbackResult === undefined; @@ -2306,15 +2306,18 @@ export const elList = async (context, user, indexName, options = {}, noFiltersCh }; while (continueProcess && hasNextPage) { // Force options to prevent connection format and manage search after - const opts = { ...options, filters: convertedFilters, first, after: searchAfter, connectionFormat: false }; - const elements = await elPaginate(context, user, indexName, opts); - if (!infinite && (elements.length === 0 || elements.length < first)) { + const paginateOpts = { ...opts, filters: convertedFilters, first, after: searchAfter, connectionFormat: false }; + const elements = await elPaginate(context, user, indexName, paginateOpts); + emitSize += elements.length; + const noMoreElements = elements.length === 0 || elements.length < (first ?? MAX_SEARCH_SIZE); + const moreThanMax = maxSize ? emitSize >= maxSize : false; + if (noMoreElements || moreThanMax) { if (elements.length > 0) { await publish(elements); } hasNextPage = false; } else if (elements.length > 0) { - const { sort } = R.last(elements); + const { sort } = elements[elements.length - 1]; searchAfter = offsetToCursor(sort); await publish(elements); } diff --git a/opencti-platform/opencti-graphql/src/database/middleware-loader.ts b/opencti-platform/opencti-graphql/src/database/middleware-loader.ts index 7bd6cb07d51d..cf9c825bc705 100644 --- a/opencti-platform/opencti-graphql/src/database/middleware-loader.ts +++ b/opencti-platform/opencti-graphql/src/database/middleware-loader.ts @@ -1,13 +1,12 @@ import * as R from 'ramda'; import { buildPagination, - offsetToCursor, READ_DATA_INDICES, READ_DATA_INDICES_WITHOUT_INFERRED, READ_ENTITIES_INDICES, READ_RELATIONSHIPS_INDICES, } from './utils'; -import { elAggregationsList, elCount, elFindByIds, elLoadById, elPaginate } from './engine'; +import { elAggregationsList, elCount, elFindByIds, elList, elLoadById, elPaginate } from './engine'; import { buildRefRelationKey } from '../schema/general'; import type { AuthContext, AuthUser } from '../types/user'; import type { @@ -33,8 +32,6 @@ import { } from '../schema/stixRefRelationship'; import { ENTITY_TYPE_WORKSPACE } from '../modules/workspace/workspace-types'; -const MAX_SEARCH_SIZE = 5000; - export interface FiltersWithNested extends Filter { nested?: Array<{ key: string; // nested filters handle special cases for elastic, it's an internal format @@ -55,7 +52,6 @@ export interface ListFilter { useWildcardPrefix?: boolean; useWildcardSuffix?: boolean; first?: number | null; - infinite?: boolean; after?: string | undefined | null; orderBy?: any, orderMode?: InputMaybe; @@ -65,7 +61,7 @@ export interface ListFilter { } type InternalListEntities = -(context: AuthContext, user: AuthUser, entityTypes: Array, args: EntityOptions, noFiltersChecking?: boolean) => Promise>; +(context: AuthContext, user: AuthUser, entityTypes: Array, args: EntityOptions) => Promise>; // entities interface EntityFilters extends ListFilter { @@ -85,44 +81,12 @@ interface EntityFilters extends ListFilter { } export interface EntityOptions extends EntityFilters { - ids?: Array; - indices?: Array; + ids?: Array + indices?: Array + noFiltersChecking?: boolean includeAuthorities?: boolean | null } -export const elList = async (context: AuthContext, user: AuthUser, indices: Array, options: ListFilter = {}): Promise> => { - const { first = MAX_SEARCH_SIZE, infinite = false } = options; - let hasNextPage = true; - let continueProcess = true; - let searchAfter = options.after; - const listing: Array = []; - const publish = async (elements: Array) => { - const { callback } = options; - if (callback) { - const callbackResult = await callback(elements); - continueProcess = callbackResult || callbackResult === undefined; - } else { - listing.push(...elements); - } - }; - while (continueProcess && hasNextPage) { - // Force options to prevent connection format and manage search after - const opts = { ...options, first, after: searchAfter, connectionFormat: false }; - const elements = await elPaginate(context, user, indices, opts); - if (!infinite && (elements.length === 0 || elements.length < (first ?? MAX_SEARCH_SIZE))) { - if (elements.length > 0) { - await publish(elements); - } - hasNextPage = false; - } else if (elements.length > 0) { - const { sort } = elements[elements.length - 1]; - searchAfter = offsetToCursor(sort); - await publish(elements); - } - } - return listing; -}; - // relations interface RelationFilters extends ListFilter { connectionFormat?: boolean; @@ -456,10 +420,10 @@ export const listAllEntitiesForFilter = async (context: AuthContext, user: AuthU return buildPagination(0, null, nodeElements, nodeElements.length); }; -export const listEntities: InternalListEntities = async (context, user, entityTypes, args = {}, noFiltersChecking = false) => { +export const listEntities: InternalListEntities = async (context, user, entityTypes, args = {}) => { const { indices = READ_ENTITIES_INDICES } = args; - const { filters } = args; - const convertedFilters = (noFiltersChecking || !filters) ? filters : checkAndConvertFilters(filters); + const { filters, noFiltersChecking } = args; + const convertedFilters = noFiltersChecking ? filters : checkAndConvertFilters(filters); // TODO Reactivate this test after global migration to typescript // if (connectionFormat !== false) { // throw UnsupportedError('List connection require connectionFormat option to false'); @@ -468,9 +432,9 @@ export const listEntities: InternalListEntities = async (context, user, entityTy return elPaginate(context, user, indices, paginateArgs); }; export const listAllEntities = async (context: AuthContext, user: AuthUser, entityTypes: Array, - args: EntityOptions = {}, noFiltersChecking = false): Promise> => { - const { indices = READ_ENTITIES_INDICES } = args; - const filters = (noFiltersChecking || !args.filters) ? args.filters : checkAndConvertFilters(args.filters); + args: EntityOptions = {}): Promise> => { + const { indices = READ_ENTITIES_INDICES, noFiltersChecking = false } = args; + const filters = noFiltersChecking ? args.filters : checkAndConvertFilters(args.filters); const paginateArgs = buildEntityFilters({ entityTypes, filters, ...args }); return elList(context, user, indices, paginateArgs); }; diff --git a/opencti-platform/opencti-graphql/src/database/middleware.js b/opencti-platform/opencti-graphql/src/database/middleware.js index b59e9f945d72..b187add82681 100644 --- a/opencti-platform/opencti-graphql/src/database/middleware.js +++ b/opencti-platform/opencti-graphql/src/database/middleware.js @@ -309,8 +309,9 @@ const batchListThrough = async (context, user, sources, sourceSide, relationType filters, types: [relationType], connectionFormat: false, + noFiltersChecking: true, search, - }, true); + }); // For each relation resolved the target entity // Filter on element id if necessary let targetIds = R.uniq(relations.map((s) => s[`${opposite}Id`])); @@ -391,11 +392,9 @@ export const listThings = async (context, user, thingsTypes, args = {}) => { const paginateArgs = buildEntityFilters({ types: thingsTypes, ...args, filters: convertedFilters }); return elPaginate(context, user, indices, paginateArgs); }; -export const listAllThings = async (context, user, thingsTypes, args = {}, noFiltersChecking = false) => { - const { indices = READ_DATA_INDICES } = args; - const convertedFilters = (noFiltersChecking || !args.filters) - ? args.filters - : checkAndConvertFilters(args.filters); +export const listAllThings = async (context, user, thingsTypes, args = {}) => { + const { indices = READ_DATA_INDICES, noFiltersChecking = false } = args; + const convertedFilters = noFiltersChecking ? args.filters : checkAndConvertFilters(args.filters); const paginateArgs = buildEntityFilters({ types: thingsTypes, ...args, filters: convertedFilters }); return elList(context, user, indices, paginateArgs); }; @@ -1055,8 +1054,9 @@ const listEntitiesByHashes = async (context, user, type, hashes) => { filters: [{ key: 'hashes.*', values: searchHashes, operator: 'wildcard' }], filterGroups: [], }, + noFiltersChecking: true, connectionFormat: false, - }, true); + }); }; export const hashMergeValidation = (instances) => { // region Specific check for observables with hashes @@ -1879,9 +1879,10 @@ export const updateAttributeMetaResolved = async (context, user, initial, inputs filters: [{ key: 'user_email', values: [initial.contact_information] }], filterGroups: [], }, + noFiltersChecking: true, connectionFormat: false }; - const users = await listEntities(context, SYSTEM_USER, [ENTITY_TYPE_USER], args, true); + const users = await listEntities(context, SYSTEM_USER, [ENTITY_TYPE_USER], args); if (users.length > 0) { throw FunctionalError('Cannot update an individual corresponding to a user'); } @@ -2108,9 +2109,10 @@ export const updateAttributeMetaResolved = async (context, user, initial, inputs filters: [{ key: 'contact_information', values: [updatedInstance.user_email] }], filterGroups: [], }, + noFiltersChecking: true, connectionFormat: false }; - const individuals = await listEntities(context, user, [ENTITY_TYPE_IDENTITY_INDIVIDUAL], args, true); + const individuals = await listEntities(context, user, [ENTITY_TYPE_IDENTITY_INDIVIDUAL], args); if (individuals.length > 0) { const individualId = R.head(individuals).id; const patch = { @@ -3388,9 +3390,10 @@ export const internalDeleteElementById = async (context, user, id, opts = {}) => filters: [{ key: 'user_email', values: [element.contact_information] }], filterGroups: [], }, + noFiltersChecking: true, connectionFormat: false }; - const users = await listEntities(context, SYSTEM_USER, [ENTITY_TYPE_USER], args, true); + const users = await listEntities(context, SYSTEM_USER, [ENTITY_TYPE_USER], args); if (users.length > 0) { throw FunctionalError('Cannot delete an individual corresponding to a user'); } diff --git a/opencti-platform/opencti-graphql/src/domain/backgroundTask.js b/opencti-platform/opencti-graphql/src/domain/backgroundTask.js index 947435ab2db9..d63d4e318c85 100644 --- a/opencti-platform/opencti-graphql/src/domain/backgroundTask.js +++ b/opencti-platform/opencti-graphql/src/domain/backgroundTask.js @@ -34,8 +34,8 @@ export const findById = async (context, user, taskId) => { return storeLoadById(context, user, taskId, ENTITY_TYPE_BACKGROUND_TASK); }; -export const findAll = (context, user, args, noFiltersChecking = false) => { - return listEntities(context, user, [ENTITY_TYPE_BACKGROUND_TASK], args, noFiltersChecking); +export const findAll = (context, user, args) => { + return listEntities(context, user, [ENTITY_TYPE_BACKGROUND_TASK], args); }; const buildQueryFiltersContent = (adaptedFiltersGroup) => { diff --git a/opencti-platform/opencti-graphql/src/domain/user.js b/opencti-platform/opencti-graphql/src/domain/user.js index 5e4e07c3777e..bb1f016f7c69 100644 --- a/opencti-platform/opencti-graphql/src/domain/user.js +++ b/opencti-platform/opencti-graphql/src/domain/user.js @@ -1104,8 +1104,8 @@ export const buildCompleteUser = async (context, client) => { filters: [{ key: 'contact_information', values: [client.user_email] }], filterGroups: [], }; - const args = { filters: contactInformationFilter, connectionFormat: false }; - const individualsPromise = listEntities(context, SYSTEM_USER, [ENTITY_TYPE_IDENTITY_INDIVIDUAL], args, true); + const args = { filters: contactInformationFilter, connectionFormat: false, noFiltersChecking: true }; + const individualsPromise = listEntities(context, SYSTEM_USER, [ENTITY_TYPE_IDENTITY_INDIVIDUAL], args); const organizationsPromise = batchOrganizations(context, SYSTEM_USER, client.id, { ...batchOpts, withInferences: false }); const userGroupsPromise = listThroughGetTo(context, SYSTEM_USER, client.id, RELATION_MEMBER_OF, ENTITY_TYPE_GROUP); const settings = await getEntityFromCache(context, SYSTEM_USER, ENTITY_TYPE_SETTINGS); diff --git a/opencti-platform/opencti-graphql/src/http/httpRollingFeed.ts b/opencti-platform/opencti-graphql/src/http/httpRollingFeed.ts index 47f4d4d3cfd5..ef0cb70d41aa 100644 --- a/opencti-platform/opencti-graphql/src/http/httpRollingFeed.ts +++ b/opencti-platform/opencti-graphql/src/http/httpRollingFeed.ts @@ -8,7 +8,7 @@ import { ForbiddenAccess } from '../config/errors'; import { BYPASS, executionContext, SYSTEM_USER } from '../utils/access'; import { findById as findFeed } from '../domain/feed'; import type { AuthUser } from '../types/user'; -import { listThings } from '../database/middleware'; +import { listAllThings } from '../database/middleware'; import { minutesAgo } from '../utils/format'; import { isNotEmptyField } from '../database/utils'; import { convertFiltersToQueryOptions } from '../utils/filtering/filtering-resolution'; @@ -69,8 +69,9 @@ const initHttpRollingFeeds = (app: Express.Application) => { const field = feed.feed_date_attribute ?? 'created_at'; const extraOptions = { defaultTypes: feed.feed_types, field, orderMode: 'desc', after: fromDate }; const options = await convertFiltersToQueryOptions(context, user, filters, extraOptions); - const args = { connectionFormat: false, first: SIZE_LIMIT, ...options }; - const elements = await listThings(context, user, feed.feed_types, args); + const args = { connectionFormat: false, maxSize: SIZE_LIMIT, ...options }; + const paginateElements = await listAllThings(context, user, feed.feed_types, args); + const elements = R.take(SIZE_LIMIT, paginateElements); // Due to pagination, number of results can be slightly superior if (feed.include_header) { res.write(`${feed.feed_attributes.map((a) => a.attribute).join(',')}\r\n`); } diff --git a/opencti-platform/opencti-graphql/src/manager/activityManager.ts b/opencti-platform/opencti-graphql/src/manager/activityManager.ts index cfa2a94bc30c..7176b214e13a 100644 --- a/opencti-platform/opencti-graphql/src/manager/activityManager.ts +++ b/opencti-platform/opencti-graphql/src/manager/activityManager.ts @@ -199,7 +199,8 @@ const initActivityManager = () => { filters: [{ key: ['event_access'], values: ['EXISTS'] }], filterGroups: [], }, - }, true); + noFiltersChecking: true + }); let lastEventId = '0-0'; if (histoElements.length > 0) { const histoDate = histoElements[0].timestamp; diff --git a/opencti-platform/opencti-graphql/src/manager/cacheManager.ts b/opencti-platform/opencti-graphql/src/manager/cacheManager.ts index ea94bafc9dcd..e4fb86c34e9e 100644 --- a/opencti-platform/opencti-graphql/src/manager/cacheManager.ts +++ b/opencti-platform/opencti-graphql/src/manager/cacheManager.ts @@ -131,8 +131,8 @@ const platformRunningPlaybooks = (context: AuthContext) => { filters: [{ key: ['playbook_running'], values: ['true'] }], filterGroups: [], }; - const opts = { filters, connectionFormat: false }; - return findAllPlaybooks(context, SYSTEM_USER, opts, true); + const opts = { filters, noFiltersChecking: true, connectionFormat: false }; + return findAllPlaybooks(context, SYSTEM_USER, opts); }; return { values: null, fn: reloadPlaybooks }; }; diff --git a/opencti-platform/opencti-graphql/src/manager/connectorManager.js b/opencti-platform/opencti-graphql/src/manager/connectorManager.js index 2992d62c5e3d..23c612166736 100644 --- a/opencti-platform/opencti-graphql/src/manager/connectorManager.js +++ b/opencti-platform/opencti-graphql/src/manager/connectorManager.js @@ -3,8 +3,7 @@ import { lockResource, redisDeleteWorks, redisGetConnectorStatus, redisGetWork } import conf, { booleanConf, logApp } from '../config/conf'; import { TYPE_LOCK_ERROR } from '../config/errors'; import { connectors } from '../database/repository'; -import { elDeleteInstances, elUpdate } from '../database/engine'; -import { elList } from '../database/middleware-loader'; +import { elDeleteInstances, elList, elUpdate } from '../database/engine'; import { executionContext, SYSTEM_USER } from '../utils/access'; import { INDEX_HISTORY } from '../database/utils'; import { now, sinceNowInDays } from '../utils/format'; @@ -66,6 +65,7 @@ const closeOldWorks = async (context, connector) => { }; await elList(context, SYSTEM_USER, [INDEX_HISTORY], { filters, + noFiltersChecking: true, types: ['Work'], connectionFormat: false, callback: queryCallback @@ -93,6 +93,7 @@ const deleteCompletedWorks = async (context, connector) => { await elList(context, SYSTEM_USER, [INDEX_HISTORY], { filters, types: ['Work'], + noFiltersChecking: true, connectionFormat: false, callback: queryCallback }); diff --git a/opencti-platform/opencti-graphql/src/manager/expiredManager.js b/opencti-platform/opencti-graphql/src/manager/expiredManager.js index 06b6c1032104..a4fd2194b200 100644 --- a/opencti-platform/opencti-graphql/src/manager/expiredManager.js +++ b/opencti-platform/opencti-graphql/src/manager/expiredManager.js @@ -41,7 +41,7 @@ const revokedInstances = async (context) => { ], filterGroups: [], }; - const opts = { filters, connectionFormat: false, callback }; + const opts = { filters, noFiltersChecking: true, connectionFormat: false, callback }; await elList(context, SYSTEM_USER, READ_DATA_INDICES, opts); }; diff --git a/opencti-platform/opencti-graphql/src/manager/historyManager.ts b/opencti-platform/opencti-graphql/src/manager/historyManager.ts index e9905f1e51b8..873ef56c706d 100644 --- a/opencti-platform/opencti-graphql/src/manager/historyManager.ts +++ b/opencti-platform/opencti-graphql/src/manager/historyManager.ts @@ -199,7 +199,8 @@ const initHistoryManager = () => { filters: [{ key: ['event_access'], values: [], operator: FilterOperator.Nil }], filterGroups: [], }, - }, true); + noFiltersChecking: true + }); let lastEventId = '0-0'; if (histoElements.length > 0) { const histoDate = histoElements[0].timestamp; diff --git a/opencti-platform/opencti-graphql/src/manager/ingestionManager.ts b/opencti-platform/opencti-graphql/src/manager/ingestionManager.ts index 3fe11348da88..1555b39c57e3 100644 --- a/opencti-platform/opencti-graphql/src/manager/ingestionManager.ts +++ b/opencti-platform/opencti-graphql/src/manager/ingestionManager.ts @@ -180,7 +180,8 @@ const rssExecutor = async (context: AuthContext, turndownService: TurndownServic filters: [{ key: 'ingestion_running', values: [true] }], filterGroups: [], }; - const ingestions = await findAllRssIngestions(context, SYSTEM_USER, { filters, connectionFormat: false }, true); + const opts = { filters, connectionFormat: false, noFiltersChecking: true }; + const ingestions = await findAllRssIngestions(context, SYSTEM_USER, opts); const ingestionPromises = []; for (let i = 0; i < ingestions.length; i += 1) { const ingestion = ingestions[i]; @@ -249,7 +250,8 @@ const taxiiExecutor = async (context: AuthContext) => { filters: [{ key: 'ingestion_running', values: [true] }], filterGroups: [], }; - const ingestions = await findAllTaxiiIngestions(context, SYSTEM_USER, { filters, connectionFormat: false }, true); + const opts = { filters, connectionFormat: false, noFiltersChecking: true }; + const ingestions = await findAllTaxiiIngestions(context, SYSTEM_USER, opts); const ingestionPromises = []; for (let i = 0; i < ingestions.length; i += 1) { const ingestion = ingestions[i]; diff --git a/opencti-platform/opencti-graphql/src/manager/notificationManager.ts b/opencti-platform/opencti-graphql/src/manager/notificationManager.ts index a2170935609c..e458db032da6 100644 --- a/opencti-platform/opencti-graphql/src/manager/notificationManager.ts +++ b/opencti-platform/opencti-graphql/src/manager/notificationManager.ts @@ -437,7 +437,7 @@ export const buildTargetEvents = async ( const { data: { data }, event: eventType } = streamEvent; const { event_types, notifiers, instance_trigger, filters } = trigger; let finalFilters = filters ? JSON.parse(filters) : null; - if (useSideEventMatching) { // modifiy filters to look for instance trigger side events + if (useSideEventMatching) { // modify filters to look for instance trigger side events finalFilters = replaceFilterKey(JSON.parse(trigger.filters), CONNECTED_TO_INSTANCE_FILTER, CONNECTED_TO_INSTANCE_SIDE_EVENTS_FILTER); } let triggerEventTypes = event_types; diff --git a/opencti-platform/opencti-graphql/src/manager/ruleManager.ts b/opencti-platform/opencti-graphql/src/manager/ruleManager.ts index e7304de995c8..ed5b99a34528 100644 --- a/opencti-platform/opencti-graphql/src/manager/ruleManager.ts +++ b/opencti-platform/opencti-graphql/src/manager/ruleManager.ts @@ -27,7 +27,7 @@ import { ALREADY_DELETED_ERROR, TYPE_LOCK_ERROR } from '../config/errors'; import { getParentTypes } from '../schema/schemaUtils'; import { isBasicRelationship } from '../schema/stixRelationship'; import { isStixSightingRelationship } from '../schema/stixSightingRelationship'; -import { elList, internalLoadById } from '../database/middleware-loader'; +import { internalLoadById } from '../database/middleware-loader'; import type { RuleDefinition, RuleRuntime, RuleScope } from '../types/rules'; import type { BasicManagerEntity, BasicStoreCommon, BasicStoreEntity, StoreObject } from '../types/store'; import type { AuthContext, AuthUser } from '../types/user'; @@ -48,6 +48,7 @@ import { getActivatedRules, RULES_DECLARATION } from '../domain/rules'; import { executionContext, RULE_MANAGER_USER } from '../utils/access'; import { isModuleActivated } from '../domain/settings'; import { FilterMode, FilterOperator } from '../generated/graphql'; +import { elList } from '../database/engine'; const MIN_LIVE_STREAM_EVENT_VERSION = 4; @@ -192,7 +193,8 @@ const applyCleanupOnDependencyIds = async (deletionIds: Array) => { // eslint-disable-next-line @typescript-eslint/no-use-before-define await rulesCleanHandler(context, RULE_MANAGER_USER, elements, RULES_DECLARATION, deletionIds); }; - await elList(context, RULE_MANAGER_USER, READ_DATA_INDICES, { filters, callback }); + const opts = { filters, noFiltersChecking: true, callback }; + await elList(context, RULE_MANAGER_USER, READ_DATA_INDICES, opts); }; export const rulesApplyHandler = async (context: AuthContext, user: AuthUser, events: Array, forRules: Array = []) => { diff --git a/opencti-platform/opencti-graphql/src/manager/taskManager.js b/opencti-platform/opencti-graphql/src/manager/taskManager.js index 7544277a596c..97835bcb813b 100644 --- a/opencti-platform/opencti-graphql/src/manager/taskManager.js +++ b/opencti-platform/opencti-graphql/src/manager/taskManager.js @@ -92,7 +92,8 @@ const findTaskToExecute = async (context) => { filters: [{ key: 'completed', values: [false] }], filterGroups: [], }, - }, true); + noFiltersChecking: true + }); if (tasks.length === 0) { return null; } @@ -329,10 +330,15 @@ const executeUnshare = async (context, user, actionContext, element) => { for (let indexCreate = 0; indexCreate < values.length; indexCreate += 1) { const target = values[indexCreate]; // resolve all containers of this element - const args = { filters: - { mode: 'and', filters: [{ key: buildRefRelationKey(RELATION_OBJECT), values: [element.id] }], filterGroups: [] }, + const args = { + filters: { + mode: 'and', + filters: [{ key: buildRefRelationKey(RELATION_OBJECT), values: [element.id] }], + filterGroups: [] + }, + noFiltersChecking: true }; - const containers = await listAllThings(context, user, [ENTITY_TYPE_CONTAINER], args, true); + const containers = await listAllThings(context, user, [ENTITY_TYPE_CONTAINER], args); const grantedTo = containers.map((n) => n[buildRefRelationKey(RELATION_GRANTED_TO)]).flat(); if (!grantedTo.includes(target)) { await deleteRelationsByFromAndTo(context, user, element.id, target, RELATION_GRANTED_TO, ABSTRACT_BASIC_RELATIONSHIP); diff --git a/opencti-platform/opencti-graphql/src/migrations/1612381566895-clear_indicates_indexation.js b/opencti-platform/opencti-graphql/src/migrations/1612381566895-clear_indicates_indexation.js index 0d0ed6e8484c..f298976123ac 100644 --- a/opencti-platform/opencti-graphql/src/migrations/1612381566895-clear_indicates_indexation.js +++ b/opencti-platform/opencti-graphql/src/migrations/1612381566895-clear_indicates_indexation.js @@ -30,7 +30,7 @@ export const up = async (next) => { filters: [{ key: buildRefRelationKey(RELATION_INDICATES), values: ['EXISTS'] }], filterGroups: [], }; - const opts = { types: [ABSTRACT_STIX_CORE_OBJECT, ABSTRACT_STIX_CORE_RELATIONSHIP], filters, callback }; + const opts = { types: [ABSTRACT_STIX_CORE_OBJECT, ABSTRACT_STIX_CORE_RELATIONSHIP], filters, noFiltersChecking: true, callback }; await elList(context, SYSTEM_USER, READ_DATA_INDICES, opts); // Apply operations. let currentProcessing = 0; diff --git a/opencti-platform/opencti-graphql/src/migrations/1650287551439-remove_sector_locations.js b/opencti-platform/opencti-graphql/src/migrations/1650287551439-remove_sector_locations.js index 064ee8d3e1aa..8bb6fb1a99c0 100644 --- a/opencti-platform/opencti-graphql/src/migrations/1650287551439-remove_sector_locations.js +++ b/opencti-platform/opencti-graphql/src/migrations/1650287551439-remove_sector_locations.js @@ -19,7 +19,7 @@ export const up = async (next) => { filters: [{ key: 'fromType', values: [ENTITY_TYPE_IDENTITY_SECTOR] }], filterGroups: [], }; - const opts = { types: [RELATION_LOCATED_AT], filters, callback }; + const opts = { types: [RELATION_LOCATED_AT], filters, noFiltersChecking: true, callback }; await elList(context, SYSTEM_USER, READ_INDEX_STIX_CORE_RELATIONSHIPS, opts); logApp.info(`[MIGRATION] Cleaning located-at relationships between Sectors and Locations done in ${new Date() - start} ms`); next(); diff --git a/opencti-platform/opencti-graphql/src/migrations/1678399004322-rel-merge-adapt.js b/opencti-platform/opencti-graphql/src/migrations/1678399004322-rel-merge-adapt.js index ccd4fa8cccee..ccad102121af 100644 --- a/opencti-platform/opencti-graphql/src/migrations/1678399004322-rel-merge-adapt.js +++ b/opencti-platform/opencti-graphql/src/migrations/1678399004322-rel-merge-adapt.js @@ -17,6 +17,7 @@ export const up = async (next) => { filters: [{ key: 'event_type', values: ['merge'] }], filterGroups: [], }, + noFiltersChecking: true, orderBy: 'created_at', orderMode: 'asc' }); diff --git a/opencti-platform/opencti-graphql/src/migrations/1688674984685-threat-actor-split.js b/opencti-platform/opencti-graphql/src/migrations/1688674984685-threat-actor-split.js index b446563f438a..87186e5a5f01 100644 --- a/opencti-platform/opencti-graphql/src/migrations/1688674984685-threat-actor-split.js +++ b/opencti-platform/opencti-graphql/src/migrations/1688674984685-threat-actor-split.js @@ -1,7 +1,7 @@ import { executionContext, SYSTEM_USER } from '../utils/access'; import { generateStandardId } from '../schema/identifier'; import { ENTITY_TYPE_THREAT_ACTOR_INDIVIDUAL } from '../modules/threatActorIndividual/threatActorIndividual-types'; -import { elCount, elRawUpdateByQuery } from '../database/engine'; +import { elCount, elList, elRawUpdateByQuery } from '../database/engine'; import { READ_ENTITIES_INDICES, READ_INDEX_INTERNAL_OBJECTS, READ_INDEX_STIX_DOMAIN_OBJECTS, @@ -10,7 +10,7 @@ import { } from '../database/utils'; import { DatabaseError } from '../config/errors'; import { ENTITY_TYPE_THREAT_ACTOR_GROUP } from '../schema/stixDomainObject'; -import { elList, listAllEntities } from '../database/middleware-loader'; +import { listAllEntities } from '../database/middleware-loader'; import { ENTITY_TYPE_THREAT_ACTOR } from '../schema/general'; import { builtInOv, openVocabularies } from '../modules/vocabulary/vocabulary-utils'; import { ENTITY_TYPE_VOCABULARY } from '../modules/vocabulary/vocabulary-types'; @@ -84,7 +84,7 @@ const createIndividualThreatCategories = async (context) => { filters: [{ key: 'category', values: [group] }], filterGroups: [], }; - const args = { connectionFormat: false, filters }; + const args = { connectionFormat: false, filters, noFiltersChecking: true }; const vocabsFromGroup = await listAllEntities(context, SYSTEM_USER, [ENTITY_TYPE_VOCABULARY], args); const groupExistingVocabs = (vocabsFromGroup ?? []).map((v) => ({ key: v.name, description: v.description, aliases: v.aliases })); const groupVocabToMaintains = groupExistingVocabs.filter((g) => !individualVocabKeys.includes(g.key)); diff --git a/opencti-platform/opencti-graphql/src/migrations/1688710489709-add_order_to_opinion_ov.js b/opencti-platform/opencti-graphql/src/migrations/1688710489709-add_order_to_opinion_ov.js index be1fce5973d6..d63ff60caa57 100644 --- a/opencti-platform/opencti-graphql/src/migrations/1688710489709-add_order_to_opinion_ov.js +++ b/opencti-platform/opencti-graphql/src/migrations/1688710489709-add_order_to_opinion_ov.js @@ -53,6 +53,7 @@ export const up = async (next) => { indices: [READ_INDEX_STIX_META_OBJECTS], connectionFormat: false, filters, + noFiltersChecking: true, }); const updateVocabulary = async (vocabulary) => { diff --git a/opencti-platform/opencti-graphql/src/migrations/1689722008218-threat-actor-split-inferred.js b/opencti-platform/opencti-graphql/src/migrations/1689722008218-threat-actor-split-inferred.js index 2682e2ee0994..b830a955288a 100644 --- a/opencti-platform/opencti-graphql/src/migrations/1689722008218-threat-actor-split-inferred.js +++ b/opencti-platform/opencti-graphql/src/migrations/1689722008218-threat-actor-split-inferred.js @@ -1,11 +1,10 @@ import { executionContext, SYSTEM_USER } from '../utils/access'; -import { elCount, elRawUpdateByQuery } from '../database/engine'; +import { elCount, elList, elRawUpdateByQuery } from '../database/engine'; import { READ_ENTITIES_INDICES, READ_INDEX_INFERRED_RELATIONSHIPS, READ_INDEX_STIX_DOMAIN_OBJECTS, } from '../database/utils'; import { DatabaseError } from '../config/errors'; -import { elList } from '../database/middleware-loader'; import { ENTITY_TYPE_THREAT_ACTOR } from '../schema/general'; import { logApp } from '../config/conf'; diff --git a/opencti-platform/opencti-graphql/src/migrations/1691495378997-migrate-hidden-types-from-role-to-group.js b/opencti-platform/opencti-graphql/src/migrations/1691495378997-migrate-hidden-types-from-role-to-group.js index 02340f22fca2..d808d0984bb3 100644 --- a/opencti-platform/opencti-graphql/src/migrations/1691495378997-migrate-hidden-types-from-role-to-group.js +++ b/opencti-platform/opencti-graphql/src/migrations/1691495378997-migrate-hidden-types-from-role-to-group.js @@ -1,9 +1,8 @@ import { Promise } from 'bluebird'; import { executionContext, SYSTEM_USER } from '../utils/access'; -import { elRawUpdateByQuery, elReplace, ES_MAX_CONCURRENCY } from '../database/engine'; +import { elRawUpdateByQuery, elList, elReplace, ES_MAX_CONCURRENCY } from '../database/engine'; import { isNotEmptyField, READ_INDEX_INTERNAL_OBJECTS, } from '../database/utils'; import { DatabaseError } from '../config/errors'; -import { elList } from '../database/middleware-loader'; import { logApp } from '../config/conf'; import { ENTITY_TYPE_GROUP, ENTITY_TYPE_ROLE } from '../schema/internalObject'; import { RELATION_HAS_ROLE } from '../schema/internalRelationship'; diff --git a/opencti-platform/opencti-graphql/src/migrations/1698678031967-filters-refacto.js b/opencti-platform/opencti-graphql/src/migrations/1698678031967-filters-refacto.js index 3dcb1802c363..26f7a3d0415d 100644 --- a/opencti-platform/opencti-graphql/src/migrations/1698678031967-filters-refacto.js +++ b/opencti-platform/opencti-graphql/src/migrations/1698678031967-filters-refacto.js @@ -186,9 +186,9 @@ export const up = async (next) => { } ], filterGroups: [], - } - }, - true, + }, + noFiltersChecking: true + } ); let tasksFiltersConvertor = {}; diff --git a/opencti-platform/opencti-graphql/src/modules/ingestion/ingestion-rss-domain.ts b/opencti-platform/opencti-graphql/src/modules/ingestion/ingestion-rss-domain.ts index aaac8960be4c..fb294b48e53e 100644 --- a/opencti-platform/opencti-graphql/src/modules/ingestion/ingestion-rss-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/ingestion/ingestion-rss-domain.ts @@ -16,8 +16,8 @@ export const findAllPaginated = async (context: AuthContext, user: AuthUser, opt return listEntitiesPaginated(context, user, [ENTITY_TYPE_INGESTION_RSS], opts); }; -export const findAllRssIngestions = async (context: AuthContext, user: AuthUser, opts = {}, noFiltersChecking = false) => { - return listAllEntities(context, user, [ENTITY_TYPE_INGESTION_RSS], opts, noFiltersChecking); +export const findAllRssIngestions = async (context: AuthContext, user: AuthUser, opts = {}) => { + return listAllEntities(context, user, [ENTITY_TYPE_INGESTION_RSS], opts); }; export const addIngestion = async (context: AuthContext, user: AuthUser, input: IngestionRssAddInput) => { diff --git a/opencti-platform/opencti-graphql/src/modules/ingestion/ingestion-taxii-domain.ts b/opencti-platform/opencti-graphql/src/modules/ingestion/ingestion-taxii-domain.ts index 6be07d11e626..ec100d610a54 100644 --- a/opencti-platform/opencti-graphql/src/modules/ingestion/ingestion-taxii-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/ingestion/ingestion-taxii-domain.ts @@ -16,8 +16,8 @@ export const findAllPaginated = async (context: AuthContext, user: AuthUser, opt return listEntitiesPaginated(context, user, [ENTITY_TYPE_INGESTION_TAXII], opts); }; -export const findAllTaxiiIngestions = async (context: AuthContext, user: AuthUser, opts = {}, noFiltersChecking = false) => { - return listAllEntities(context, user, [ENTITY_TYPE_INGESTION_TAXII], opts, noFiltersChecking); +export const findAllTaxiiIngestions = async (context: AuthContext, user: AuthUser, opts = {}) => { + return listAllEntities(context, user, [ENTITY_TYPE_INGESTION_TAXII], opts); }; export const addIngestion = async (context: AuthContext, user: AuthUser, input: IngestionTaxiiAddInput) => { diff --git a/opencti-platform/opencti-graphql/src/modules/playbook/playbook-domain.ts b/opencti-platform/opencti-graphql/src/modules/playbook/playbook-domain.ts index 014bb1f36bc8..42371c2171ac 100644 --- a/opencti-platform/opencti-graphql/src/modules/playbook/playbook-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/playbook/playbook-domain.ts @@ -48,8 +48,8 @@ export const findAll = (context: AuthContext, user: AuthUser, opts: EntityOption return listEntitiesPaginated(context, user, [ENTITY_TYPE_PLAYBOOK], opts); }; -export const findAllPlaybooks = (context: AuthContext, user: AuthUser, opts: EntityOptions, noFiltersChecking = false) => { - return listAllEntities(context, user, [ENTITY_TYPE_PLAYBOOK], opts, noFiltersChecking); +export const findAllPlaybooks = (context: AuthContext, user: AuthUser, opts: EntityOptions) => { + return listAllEntities(context, user, [ENTITY_TYPE_PLAYBOOK], opts); }; export const availableComponents = () => { diff --git a/opencti-platform/opencti-graphql/src/rules/containerWithRefsBuilder.ts b/opencti-platform/opencti-graphql/src/rules/containerWithRefsBuilder.ts index 17fe655f7050..7656a6765db6 100644 --- a/opencti-platform/opencti-graphql/src/rules/containerWithRefsBuilder.ts +++ b/opencti-platform/opencti-graphql/src/rules/containerWithRefsBuilder.ts @@ -150,7 +150,7 @@ const buildContainerRefsRule = (ruleDefinition: RuleDefinition, containerType: s filters: [{ key: [`${RULE_PREFIX}*.dependencies`], values: removedIds, operator: FilterOperator.Wildcard }], filterGroups: [], }; - const args = { fromId: report.extensions[STIX_EXT_OCTI].id, filters, indices: READ_DATA_INDICES }; + const args = { fromId: report.extensions[STIX_EXT_OCTI].id, filters, noFiltersChecking: true, indices: READ_DATA_INDICES }; const targets = await listAllRelations(context, RULE_MANAGER_USER, RELATION_OBJECT, args); deletedTargets.push(...targets); } diff --git a/opencti-platform/opencti-graphql/src/utils/filtering/filtering-utils.ts b/opencti-platform/opencti-graphql/src/utils/filtering/filtering-utils.ts index a89de40300ad..955ee7ba031e 100644 --- a/opencti-platform/opencti-graphql/src/utils/filtering/filtering-utils.ts +++ b/opencti-platform/opencti-graphql/src/utils/filtering/filtering-utils.ts @@ -19,6 +19,7 @@ import { } from './filtering-constants'; import { STIX_SIGHTING_RELATIONSHIP } from '../../schema/stixSightingRelationship'; import { STIX_CORE_RELATIONSHIPS } from '../../schema/stixCoreRelationship'; +import { UnsupportedError } from '../../config/errors'; //---------------------------------------------------------------------------------------------------------------------- // Basic utility functions @@ -217,7 +218,7 @@ const getRelationsConvertedNames = (relationNames: string[]) => { * - convert relation refs key if any * @param filterGroup */ -export const checkAndConvertFilters = (filterGroup?: FilterGroup) => { +export const checkAndConvertFilters = (filterGroup?: FilterGroup | null) => { if (!filterGroup) { return undefined; } @@ -247,7 +248,7 @@ export const checkAndConvertFilters = (filterGroup?: FilterGroup) => { } }); if (incorrectKeys.length > 0) { - throw Error(`incorrect filter keys: ${incorrectKeys} not existing in any schema definition`); + throw UnsupportedError('incorrect filter keys not existing in any schema definition', { keys: incorrectKeys }); } } diff --git a/opencti-platform/opencti-graphql/tests/02-integration/02-resolvers/grouping-test.js b/opencti-platform/opencti-graphql/tests/02-integration/02-resolvers/grouping-test.js index e7ff577eb308..db932470eb0f 100644 --- a/opencti-platform/opencti-graphql/tests/02-integration/02-resolvers/grouping-test.js +++ b/opencti-platform/opencti-graphql/tests/02-integration/02-resolvers/grouping-test.js @@ -293,10 +293,10 @@ describe('Grouping resolver standard behavior', () => { values: ['investigation'], }], filterGroups: [], - } + }, + noFiltersChecking: true } ); - await Promise.all(investigations.map(({ id }) => deleteElementById(testContext, ADMIN_USER, id, ENTITY_TYPE_WORKSPACE))); }); diff --git a/opencti-platform/opencti-graphql/tests/02-integration/02-resolvers/report-test.js b/opencti-platform/opencti-graphql/tests/02-integration/02-resolvers/report-test.js index 32d7756368a5..a366c219218a 100644 --- a/opencti-platform/opencti-graphql/tests/02-integration/02-resolvers/report-test.js +++ b/opencti-platform/opencti-graphql/tests/02-integration/02-resolvers/report-test.js @@ -468,9 +468,9 @@ describe('Report resolver standard behavior', () => { }], filterGroups: [], }, + noFiltersChecking: true }, ); - await Promise.all(investigations.map(({ id }) => deleteElementById(testContext, ADMIN_USER, id, ENTITY_TYPE_WORKSPACE))); });