[backend] Organization sharing behavior change for upsert and enrichment

opencti-platform/opencti-graphql/src/database/middleware.js

@@ -122,7 +122,7 @@ import {
   RELATION_OBJECT_MARKING,
   STIX_REF_RELATIONSHIP_TYPES
 } from '../schema/stixRefRelationship';
-import { ENTITY_TYPE_STATUS, ENTITY_TYPE_USER, isDatedInternalObject } from '../schema/internalObject';
+import { ENTITY_TYPE_SETTINGS, ENTITY_TYPE_STATUS, ENTITY_TYPE_USER, isDatedInternalObject } from '../schema/internalObject';
 import { isStixCoreObject, isStixObject } from '../schema/stixCoreObject';
 import { isBasicRelationship, isStixRelationshipExceptRef } from '../schema/stixRelationship';
 import {
@@ -183,7 +183,7 @@ import {
   storeLoadById
 } from './middleware-loader';
 import { checkRelationConsistency, isRelationConsistent } from '../utils/modelConsistency';
-import { getEntitiesListFromCache } from './cache';
+import { getEntitiesListFromCache, getEntityFromCache } from './cache';
 import { ACTION_TYPE_SHARE, ACTION_TYPE_UNSHARE, createListTask } from '../domain/backgroundTask-common';
 import { ENTITY_TYPE_VOCABULARY, vocabularyDefinitions } from '../modules/vocabulary/vocabulary-types';
 import { getVocabulariesCategories, getVocabularyCategoryForField, isEntityFieldAnOpenVocabulary, updateElasticVocabularyValue } from '../modules/vocabulary/vocabulary-utils';
@@ -1686,6 +1686,7 @@ const updateAttributeRaw = async (context, user, instance, inputs, opts = {}) =>
 export const updateAttributeMetaResolved = async (context, user, initial, inputs, opts = {}) => {
   const { locks = [], impactStandardId = true } = opts;
   const updates = Array.isArray(inputs) ? inputs : [inputs];
+  const settings = await getEntityFromCache(context, SYSTEM_USER, ENTITY_TYPE_SETTINGS);
   // Region - Pre-Check
   const elementsByKey = R.groupBy((e) => e.key, updates);
   const multiOperationKeys = Object.values(elementsByKey).filter((n) => n.length > 1);
@@ -1871,7 +1872,9 @@ export const updateAttributeMetaResolved = async (context, user, initial, inputs
         }
       } else {
         // Special access check for RELATION_GRANTED_TO meta
-        if (relType === RELATION_GRANTED_TO && !isUserHasCapability(user, KNOWLEDGE_ORGANIZATION_RESTRICT)) {
+        // If not supported, update must be rejected
+        const isUserCanManipulateGrantedRefs = isUserHasCapability(user, KNOWLEDGE_ORGANIZATION_RESTRICT) && isNotEmptyField(settings.enterprise_edition);
+        if (relType === RELATION_GRANTED_TO && !isUserCanManipulateGrantedRefs) {
           throw ForbiddenAccess();
         }
         let { value: refs, operation = UPDATE_OPERATION_REPLACE } = meta[metaIndex];
@@ -2350,6 +2353,7 @@ const buildRelationDeduplicationFilters = (input) => {
 
 const upsertElement = async (context, user, element, type, basePatch, opts = {}) => {
   // -- Independent update
+  const settings = await getEntityFromCache(context, SYSTEM_USER, ENTITY_TYPE_SETTINGS);
   const updatePatch = { ...basePatch };
   // Handle attributes updates
   if (isNotEmptyField(basePatch.stix_id) || isNotEmptyField(basePatch.x_opencti_stix_ids)) {
@@ -2376,7 +2380,6 @@ const upsertElement = async (context, user, element, type, basePatch, opts = {})
       updatePatch.number_observed = element.number_observed + updatePatch.number_observed;
     }
   }
-
   if (type === ENTITY_TYPE_INDICATOR) {
     if (updatePatch.decay_applied_rule && updatePatch.decay_base_score === element.decay_base_score) {
       logApp.debug('UPSERT INDICATOR -- no decay reset because no score change', { element, basePatch });
@@ -2389,7 +2392,6 @@ const upsertElement = async (context, user, element, type, basePatch, opts = {})
       logApp.debug('UPSERT INDICATOR -- Decay is reset', { element, basePatch });
     }
   }
-
   // Upsert relations with times extensions
   if (isStixCoreRelationship(type)) {
     const { date: cStartTime } = computeExtendedDateValues(updatePatch.start_time, element.start_time, ALIGN_OLDEST);
@@ -2416,13 +2418,11 @@ const upsertElement = async (context, user, element, type, basePatch, opts = {})
     const fileImpact = { key: 'x_opencti_files', value: [...(element.x_opencti_files ?? []), convertedFile] };
     inputs.push(fileImpact);
   }
-
   // region confidence control / upsert
   const { confidenceLevelToApply, isConfidenceMatch } = controlUpsertInputWithUserConfidence(user, updatePatch, element);
   updatePatch.confidence = confidenceLevelToApply;
   // note that if the existing data has no confidence (null) it will still be updated below, even if isConfidenceMatch = false
   // endregion
-
   // -- Upsert attributes
   const attributes = Array.from(schemaAttributesDefinition.getAttributes(type).values());
   for (let attrIndex = 0; attrIndex < attributes.length; attrIndex += 1) {
@@ -2455,19 +2455,22 @@ const upsertElement = async (context, user, element, type, basePatch, opts = {})
     if (isInputAvailable) {
       const patchInputData = updatePatch[inputField];
       const isInputWithData = isNotEmptyField(patchInputData);
-      const isUpsertSynchro = (context.synchronizedUpsert || inputField === INPUT_GRANTED_REFS); // Granted Refs are always fully sync
+      const isUpsertSynchro = context.synchronizedUpsert;
       if (relDef.multiple) {
         const currentData = element[relDef.databaseName] ?? [];
         const isCurrentWithData = isNotEmptyField(currentData);
         const targetData = (patchInputData ?? []).map((n) => n.internal_id);
+        // Specific case for organization restriction, has EE must be activated.
+        // If not supported, upsert of organization is not applied
+        const isUserCanManipulateGrantedRefs = isUserHasCapability(user, KNOWLEDGE_ORGANIZATION_RESTRICT) && isNotEmptyField(settings.enterprise_edition);
+        const allowedOperation = relDef.databaseName !== RELATION_GRANTED_TO || (relDef.databaseName === RELATION_GRANTED_TO && isUserCanManipulateGrantedRefs);
         // If expected data is different from current data
-        if (R.symmetricDifference(currentData, targetData).length > 0) {
+        if (allowedOperation && R.symmetricDifference(currentData, targetData).length > 0) {
           const diffTargets = (patchInputData ?? []).filter((target) => !currentData.includes(target.internal_id));
           // In full synchro, just replace everything
           if (isUpsertSynchro) {
             inputs.push({ key: inputField, value: patchInputData ?? [], operation: UPDATE_OPERATION_REPLACE });
-          } else if (
-            (isCurrentWithData && isInputWithData && diffTargets.length > 0 && isConfidenceMatch)
+          } else if ((isCurrentWithData && isInputWithData && diffTargets.length > 0 && isConfidenceMatch)
             || (isInputWithData && !isCurrentWithData)
           ) {
             // If data is provided, different from existing data, and of higher confidence
@@ -2674,6 +2677,7 @@ export const createRelationRaw = async (context, user, rawInput, opts = {}) => {
   const entitySetting = await getEntitySettingFromCache(context, relationshipType);
   const filledInput = fillDefaultValues(user, input, entitySetting);
   await validateEntityAndRelationCreation(context, user, filledInput, relationshipType, entitySetting, opts);
+
   // We need to check existing dependencies
   const resolvedInput = await inputResolveRefs(context, user, filledInput, relationshipType, entitySetting);
   const { from, to } = resolvedInput;
@@ -2687,6 +2691,7 @@ export const createRelationRaw = async (context, user, rawInput, opts = {}) => {
   if (!validateUserAccessOperation(user, from, 'edit') || !validateUserAccessOperation(user, to, 'edit')) {
     throw ForbiddenAccess();
   }
+
   // Check consistency
   await checkRelationConsistency(context, user, relationshipType, from, to);
   // In some case from and to can be resolved to the same element (because of automatic merging)
@@ -2708,6 +2713,7 @@ export const createRelationRaw = async (context, user, rawInput, opts = {}) => {
       throw UnsupportedError('Cant add another relation on single ref', errorData);
     }
   }
+
   // Build lock ids
   const inputIds = getInputIds(relationshipType, resolvedInput, fromRule);
   if (isImpactedTypeAndSide(relationshipType, ROLE_FROM)) inputIds.push(from.internal_id);
@@ -3183,7 +3189,7 @@ export const createEntity = async (context, user, input, type, opts = {}) => {
   const data = await createEntityRaw(context, user, input, type, opts);
   // In case of creation, start an enrichment
   if (data.isCreation) {
-    await createEntityAutoEnrichment(context, user, data.element.standard_id, type);
+    await createEntityAutoEnrichment(context, user, data.element, type);
   }
   return isCompleteResult ? data : data.element;
 };

opencti-platform/opencti-graphql/src/domain/enrichment.js

@@ -5,15 +5,17 @@
 import { connectorsEnrichment } from '../database/repository';
 import { ENTITY_TYPE_CONNECTOR } from '../schema/internalObject';
 import { getEntitiesListFromCache } from '../database/cache';
+import { CONNECTOR_INTERNAL_ENRICHMENT } from '../schema/general';
 
-export const createEntityAutoEnrichment = async (context, user, stixCoreObjectId, scope) => {
+export const createEntityAutoEnrichment = async (context, user, element, scope) => {
+  const elementStandardId = element.standard_id;
   // Get the list of compatible connectors
   const connectors = await getEntitiesListFromCache(context, user, ENTITY_TYPE_CONNECTOR);
   const targetConnectors = connectorsEnrichment(connectors, scope, true, true);
   // Create a work for each connector
   const workList = await Promise.all(
     map((connector) => {
-      return createWork(context, user, connector, `Enrichment (${stixCoreObjectId})`, stixCoreObjectId).then((work) => {
+      return createWork(context, user, connector, `Enrichment (${elementStandardId})`, elementStandardId).then((work) => {
         return { connector, work };
       });
     }, targetConnectors)
@@ -25,10 +27,12 @@
       const message = {
         internal: {
           work_id: work.id, // Related action for history
-          applicant_id: null, // User asking for the import
+          applicant_id: null, // No specific user asking for the import
         },
         event: {
-          entity_id: stixCoreObjectId,
+          event_type: CONNECTOR_INTERNAL_ENRICHMENT,
+          entity_id: elementStandardId,
+          entity_type: element.entity_type,
         },
       };
       return pushToConnector(connector.internal_id, message);

opencti-platform/opencti-graphql/src/domain/stix.js

@@ -16,6 +16,7 @@
   ABSTRACT_STIX_DOMAIN_OBJECT,
   ABSTRACT_STIX_OBJECT,
   ABSTRACT_STIX_RELATIONSHIP,
+  CONNECTOR_INTERNAL_EXPORT_FILE,
   INPUT_GRANTED_REFS
 } from '../schema/general';
 import { UPDATE_OPERATION_ADD, UPDATE_OPERATION_REMOVE } from '../database/utils';
@@ -74,6 +75,7 @@
       return {
         internal,
         event: {
+          event_type: CONNECTOR_INTERNAL_EXPORT_FILE,
           export_scope: 'selection', // query or selection or single
           file_name: fileName, // Export expected file name
           selected_ids: selectedIds, // ids that are both selected via checkboxes and respect the filtering
@@ -135,6 +137,7 @@
         applicant_id: user.id, // User asking for the import
       },
       event: {
+        event_type: CONNECTOR_INTERNAL_EXPORT_FILE,
         file_name: fileName, // Export expected file name
         ...baseEvent
       },

opencti-platform/opencti-graphql/src/domain/stixCoreObject.js

@@ -12,9 +12,10 @@
   ABSTRACT_STIX_CORE_OBJECT,
   ABSTRACT_STIX_DOMAIN_OBJECT,
   buildRefRelationKey,
+  CONNECTOR_INTERNAL_ENRICHMENT,
   ENTITY_TYPE_CONTAINER,
   INPUT_EXTERNAL_REFS,
-  REL_INDEX_PREFIX,
+  REL_INDEX_PREFIX
 } from '../schema/general';
 import { RELATION_CREATED_BY, RELATION_EXTERNAL_REFERENCE, RELATION_OBJECT, RELATION_OBJECT_MARKING } from '../schema/stixRefRelationship';
 import {
@@ -190,10 +191,12 @@
   const message = {
     internal: {
       work_id: work.id, // Related action for history
-      applicant_id: user.id, // User asking for the import
+      applicant_id: null, // No specific user asking for the import
     },
     event: {
+      event_type: CONNECTOR_INTERNAL_ENRICHMENT,
       entity_id: element.standard_id,
+      entity_type: element.entity_type,
     },
   };
   await pushToConnector(connector.internal_id, message);
@@ -444,6 +447,9 @@
   }
   // Get the context
   const baseDocument = await documentFindById(context, user, fileId);
+  if (!baseDocument) {
+    throw UnsupportedError('File removed or inaccessible', { fileId });
+  }
   const entityId = baseDocument.metaData.entity_id;
   const externalReferenceId = baseDocument.metaData.external_reference_id;
   const previous = await storeLoadByIdWithRefs(context, user, entityId);

opencti-platform/opencti-graphql/src/graphql/sseMiddleware.js

@@ -24,7 +24,15 @@
   READ_INDEX_STIX_SIGHTING_RELATIONSHIPS,
   READ_STIX_INDICES,
 } from '../database/utils';
-import { BYPASS, computeUserMemberAccessIds, executionContext, isUserCanAccessStixElement, isUserHasCapability, SYSTEM_USER } from '../utils/access';
+import {
+  BYPASS,
+  computeUserMemberAccessIds,
+  executionContext,
+  isUserCanAccessStixElement,
+  isUserHasCapability,
+  KNOWLEDGE_ORGANIZATION_RESTRICT,
+  SYSTEM_USER
+} from '../utils/access';
 import { FROM_START_STR, utcDate } from '../utils/format';
 import { stixRefsExtractor } from '../schema/stixEmbeddedRelationship';
 import { ABSTRACT_STIX_CORE_RELATIONSHIP, buildRefRelationKey, ENTITY_TYPE_CONTAINER, STIX_TYPE_RELATION, STIX_TYPE_SIGHTING } from '../schema/general';
@@ -237,7 +245,7 @@
       },
       setLastEventId: (id) => { lastEventId = id; },
       connected: () => !req.finished,
-      sendEvent: (eventId, topic, data) => {
+      sendEvent: (eventId, topic, event) => {
         if (req.finished) {
           // Write on an already terminated response
           return;
@@ -250,9 +258,16 @@
         if (topic) {
           message += `event: ${topic}\n`;
         }
-        if (data) {
+        if (event) {
           message += 'data: ';
-          message += JSON.stringify(data);
+          const isDataTopic = eventId && topic !== 'heartbeat';
+          if (isDataTopic && req.user && !isUserHasCapability(req.user, KNOWLEDGE_ORGANIZATION_RESTRICT)) {
+            const filtered = { ...event };
+            delete filtered.data.extensions[STIX_EXT_OCTI].granted_refs;
+            message += JSON.stringify(filtered);
+          } else {
+            message += JSON.stringify(event);
+          }
           message += '\n';
         }
         message += '\n';

opencti-platform/opencti-graphql/src/manager/activityListener.ts

@@ -194,7 +194,7 @@
         }
         if (action.event_scope === 'enrich') {
           const { entity_name, connector_name } = action.context_data;
-          const message = `asks for \`${connector_name}\` enrichment in \`${entity_name}\``;
+          const message = `asks for \`${entity_name}\` enrichment with connector \`${connector_name}\``;
           await activityLogger(action, message);
         }
       }

opencti-platform/opencti-graphql/src/modules/administrativeArea/administrativeArea.graphql

@@ -16,7 +16,7 @@ type AdministrativeArea implements BasicObject & StixCoreObject & StixDomainObje
     createdBy: Identity
     numberOfConnectedElement: Int!
     objectMarking: [MarkingDefinition!]
-    objectOrganization: [Organization!] @auth(for: [KNOWLEDGE_KNUPDATE_KNORGARESTRICT])
+    objectOrganization: [Organization!]
     objectLabel: [Label!]
     externalReferences(first: Int): ExternalReferenceConnection
     containersNumber: Number

opencti-platform/opencti-graphql/src/modules/attributes/internalObject-registrationAttributes.ts

@@ -98,6 +98,7 @@ const HistoryDefinition: AttributeDefinition[] = [
       { name: 'message', label: 'Message', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: false, upsert: true, isFilterable: true },
       { name: 'commit', label: 'Commit', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: false, upsert: true, isFilterable: true },
       { name: 'element_id', label: 'Element ID', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: false, upsert: true, isFilterable: true },
+      { name: 'entity_id', label: 'Entity ID', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: false, upsert: true, isFilterable: true },
       { name: 'entity_type', label: 'Entity type', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: false, upsert: true, isFilterable: true },
       { name: 'path', label: 'Path', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: false, upsert: true, isFilterable: true },
       { name: 'format', label: 'Format', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: false, upsert: true, isFilterable: true },
@@ -114,6 +115,7 @@ const HistoryDefinition: AttributeDefinition[] = [
       { name: 'connector_name', label: 'Connector name', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: false, upsert: true, isFilterable: true },
       { name: 'marking_definition_ids', label: 'Marking definition', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: true, upsert: true, isFilterable: false },
       { name: 'object_marking_refs_ids', label: 'Marking definition', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: true, upsert: true, isFilterable: false },
+      { name: 'connector_id', label: 'Source connector', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: false, upsert: true, isFilterable: false },
       { name: 'from_id', label: 'Source entity', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: false, upsert: true, isFilterable: false },
       { name: 'to_id', label: 'Target entity', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: false, upsert: true, isFilterable: false },
       { name: 'created_by_id', label: 'Created by', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: false, upsert: true, isFilterable: false },

opencti-platform/opencti-graphql/src/modules/case/case-incident/case-incident.graphql

@@ -16,7 +16,7 @@ type CaseIncident implements BasicObject & StixObject & StixCoreObject & StixDom
   createdBy: Identity
   numberOfConnectedElement: Int!
   objectMarking: [MarkingDefinition!]
-  objectOrganization: [Organization!] @auth(for: [KNOWLEDGE_KNUPDATE_KNORGARESTRICT])
+  objectOrganization: [Organization!]
   objectLabel: [Label!]
   externalReferences(first: Int): ExternalReferenceConnection
   containersNumber: Number

opencti-platform/opencti-graphql/src/modules/case/case-rfi/case-rfi.graphql

@@ -16,7 +16,7 @@ type CaseRfi implements BasicObject & StixObject & StixCoreObject & StixDomainOb
   createdBy: Identity
   numberOfConnectedElement: Int!
   objectMarking: [MarkingDefinition!]
-  objectOrganization: [Organization!] @auth(for: [KNOWLEDGE_KNUPDATE_KNORGARESTRICT])
+  objectOrganization: [Organization!]
   objectLabel: [Label!]
   externalReferences(first: Int): ExternalReferenceConnection
   containersNumber: Number

opencti-platform/opencti-graphql/src/modules/case/case-rft/case-rft.graphql

@@ -16,7 +16,7 @@ type CaseRft implements BasicObject & StixObject & StixCoreObject & StixDomainOb
   createdBy: Identity
   numberOfConnectedElement: Int!
   objectMarking: [MarkingDefinition!]
-  objectOrganization: [Organization!] @auth(for: [KNOWLEDGE_KNUPDATE_KNORGARESTRICT])
+  objectOrganization: [Organization!]
   objectLabel: [Label!]
   externalReferences(first: Int): ExternalReferenceConnection
   containersNumber: Number

opencti-platform/opencti-graphql/src/modules/case/case.graphql

@@ -16,7 +16,7 @@ interface Case implements BasicObject & StixObject & StixCoreObject & StixDomain
   createdBy: Identity
   numberOfConnectedElement: Int!
   objectMarking: [MarkingDefinition!]
-  objectOrganization: [Organization!] @auth(for: [KNOWLEDGE_KNUPDATE_KNORGARESTRICT])
+  objectOrganization: [Organization!]
   objectLabel: [Label!]
   externalReferences(first: Int): ExternalReferenceConnection
   containersNumber: Number