[backend] Adapt upsert indicator score with decay (#2859)

Co-authored-by: Adrien Servel <[email protected]>
Co-authored-by: Julien Richard <[email protected]>

opencti-platform/opencti-front/lang/front/de.json

@@ -2619,10 +2619,11 @@
   "Platform connected": "Verbundene Plattform",
   "Get OpenBAS now": "OpenBAS jetzt holen",
   "Platform under construction, subscribe to update!": "Plattform im Aufbau, Update abonnieren!",
-  "Current stable score": "Aktueller stabiler Stand",
+  "Initial score": "Anfangspunktzahl",
+  "Current stable score": "Aktuelle stabile Punktzahl",
   "Base score": "Basis Punktestand",
-  "Revoke score": "Score widerrufen",
-  "Stability threshold": "Stabilitätsschwelle",
+  "Revoke score": "Widerrufspunktzahl",
+  "Stable score": "Stabile Punktestand",
   "Score:": "Punktzahl:",
   "Days": "Tage",
   "Indicator observable types": "Indikator beobachtbare Typen",

opencti-platform/opencti-front/lang/front/en.json

@@ -2636,10 +2636,11 @@
   "Platform connected": "Platform connected",
   "Get OpenBAS now": "Get OpenBAS now",
   "Platform under construction, subscribe to update!": "Platform under construction, subscribe to update!",
+  "Initial score": "Initial score",
   "Current stable score": "Current stable score",
   "Base score": "Base score",
   "Revoke score": "Revoke score",
-  "Stability threshold": "Stability threshold",
+  "Stable score": "Stable score",
   "Score:": "Score:",
   "Days": "Days",
   "Indicator observable types": "Indicator observable types",

opencti-platform/opencti-front/lang/front/es.json

@@ -2620,10 +2620,11 @@
   "Platform connected": "Plataforma conectada",
   "Get OpenBAS now": "Obtener OpenBAS ahora",
   "Platform under construction, subscribe to update!": "Plataforma en construcción, ¡suscríbete para estar al día!",
+  "Initial score": "Puntuación inicial",
   "Current stable score": "Puntuación estable actual",
   "Base score": "Puntuación base",
   "Revoke score": "Puntuación de revocación",
-  "Stability threshold": "Umbral de estabilidad",
+  "Stable score": "Puntuación estable",
   "Score:": "Puntuación:",
   "Days": "Días",
   "Indicator observable types": "Indicador tipos observables",

opencti-platform/opencti-front/lang/front/fr.json

@@ -2629,10 +2629,11 @@
   "Platform connected": "Plate-forme connectée",
   "Get OpenBAS now": "Obtenir OpenBAS maintenant",
   "Platform under construction, subscribe to update!": "Plateforme en cours de construction, abonnez-vous pour être mis à jour !",
+  "Initial score": "Score initial",
   "Current stable score": "Score stable actuel",
   "Base score": "Score de base",
   "Revoke score": "Score de révocation",
-  "Stability threshold": "Seuil de stabilité",
+  "Stable score": "Score stable",
   "Score:": "Score :",
   "Days": "Jours",
   "Indicator observable types": "Types d'observables des indicateurs",

opencti-platform/opencti-front/lang/front/ja.json

@@ -2630,10 +2630,11 @@
   "Platform connected": "プラットフォーム接続",
   "Get OpenBAS now": "今すぐOpenBASを入手する",
   "Platform under construction, subscribe to update!": "プラットフォームは構築中です！",
+  "Initial score": "初期スコア",
   "Current stable score": "現在の安定スコア",
   "Base score": "基本スコア",
   "Revoke score": "失効スコア",
-  "Stability threshold": "安定閾値",
+  "Stable score": "安定したスコア",
   "Score:": "スコア",
   "Days": "日数",
   "Indicator observable types": "指標タイプ",

opencti-platform/opencti-front/lang/front/zh.json

@@ -2630,10 +2630,11 @@
   "Platform connected": "连接的平台",
   "Get OpenBAS now": "立即获取 OpenBAS",
   "Platform under construction, subscribe to update!": "平台正在建设中，请订阅更新！",
+  "Initial score": "初始分数",
   "Current stable score": "当前稳定分数",
   "Base score": "基础分数",
   "Revoke score": "撤销得分",
-  "Stability threshold": "稳定性阈值",
+  "Stable score": "成绩稳定",
   "Score:": "得分：",
   "Days": "天数",
   "Indicator observable types": "指标观测类型",

opencti-platform/opencti-front/src/private/components/observations/indicators/DecayDialog.tsx

@@ -44,17 +44,8 @@ const DecayDialogContent : FunctionComponent<DecayDialogContentProps> = ({ indic
     return mhd(history.updated_at);
   };
 
-  const getDisplayFor = (history: DecayHistory) => {
-    if (history.updated_at < indicator.decay_base_score_date) {
-      // Anything before base score reset is just "score"
-      return {
-        label: t_i18n('Score'),
-        style: { color: theme.palette.text.primary },
-        score: history.score,
-        updated_at: getDateAsTextFor(history),
-      };
-    }
-    if (history.score === indicator.x_opencti_score) {
+  const getDisplayForHistory = (history: DecayHistory, index: number, currentScoreIndex: number) => {
+    if (index === currentScoreIndex) {
       return {
         label: t_i18n('Current stable score'),
         style: {
@@ -64,14 +55,33 @@ const DecayDialogContent : FunctionComponent<DecayDialogContentProps> = ({ indic
         score: history.score,
         updated_at: getDateAsTextFor(history),
       };
-    } if (history.score === indicator.decay_base_score) {
+    }
+    if (index === 0) {
       return {
-        label: t_i18n('Base score'),
+        label: t_i18n('Initial score'),
         style: { color: theme.palette.text.primary },
         score: history.score,
         updated_at: getDateAsTextFor(history),
       };
-    } if (history.score === indicator.decay_applied_rule?.decay_revoke_score) {
+    }
+    if (history.score === indicator.decay_applied_rule?.decay_revoke_score) {
+      return {
+        label: t_i18n('Revoke score'),
+        style: { color: theme.palette.secondary.main },
+        score: history.score,
+        updated_at: getDateAsTextFor(history),
+      };
+    }
+    return {
+      label: t_i18n('Stable score'),
+      style: { color: theme.palette.text.primary },
+      score: history.score,
+      updated_at: getDateAsTextFor(history),
+    };
+  };
+
+  const getDisplayForUpcomingUpdates = (history: DecayHistory) => {
+    if (history.score === indicator.decay_applied_rule?.decay_revoke_score) {
       return {
         label: t_i18n('Revoke score'),
         style: { color: theme.palette.secondary.main },
@@ -80,20 +90,21 @@ const DecayDialogContent : FunctionComponent<DecayDialogContentProps> = ({ indic
       };
     }
     return {
-      label: t_i18n('Stability threshold'),
+      label: t_i18n('Stable score'),
       style: { color: theme.palette.text.primary },
       score: history.score,
       updated_at: getDateAsTextFor(history),
     };
   };
 
   const labelledHistoryList: LabelledDecayHistory[] = [];
-  decayHistory.forEach((history) => (
-    labelledHistoryList.push(getDisplayFor(history))
+  const currentScoreIndex = decayHistory.findLastIndex((history) => history.score === indicator.x_opencti_score);
+  decayHistory.forEach((history, index) => (
+    labelledHistoryList.push(getDisplayForHistory(history, index, currentScoreIndex))
   ));
 
   decayLivePoints.forEach((history) => (
-    labelledHistoryList.push(getDisplayFor(history))
+    labelledHistoryList.push(getDisplayForUpcomingUpdates(history))
   ));
 
   labelledHistoryList.sort((a, b) => {
@@ -117,7 +128,7 @@ const DecayDialogContent : FunctionComponent<DecayDialogContentProps> = ({ indic
         spacing={3}
         style={{ borderColor: 'white', borderWidth: 1 }}
       >
-        <Grid item={true} xs={6}>
+        <Grid item={true} xs={7}>
           <DecayChart
             currentScore={indicator.x_opencti_score || 0}
             revokeScore={indicator.decay_applied_rule?.decay_revoke_score || 0}
@@ -126,7 +137,7 @@ const DecayDialogContent : FunctionComponent<DecayDialogContentProps> = ({ indic
             decayLiveScore={indicator.decayLiveDetails?.live_score}
           />
         </Grid>
-        <Grid item={true} xs={6}>
+        <Grid item={true} xs={5}>
           <TableContainer component={Paper}>
             <Table sx={{ maxHeight: 440 }} size="small" aria-label="lifecycle history">
               <TableHead>

opencti-platform/opencti-front/src/private/components/settings/decay/DecayChart.tsx

@@ -53,15 +53,16 @@ const DecayChart : FunctionComponent<DecayChartProps> = ({ currentScore, decayCu
   const graphLinesAnnotations = [];
   // Horizontal lines that shows reaction points
   if (reactionPoints) {
-    reactionPoints.forEach((reactionPoint) => {
+    const currentScoreIndex = reactionPoints.findLastIndex((reactionPoint) => reactionPoint === currentScore);
+    reactionPoints.forEach((reactionPoint, index) => {
       const lineReactionValue = {
         y: reactionPoint,
-        borderColor: reactionPoint === currentScore ? scoreColor : reactionPointColor,
+        borderColor: index === currentScoreIndex ? scoreColor : reactionPointColor,
         label: {
-          borderColor: reactionPoint === currentScore ? scoreColor : reactionPointColor,
+          borderColor: index === currentScoreIndex ? scoreColor : reactionPointColor,
           offsetY: 0,
           style: {
-            color: reactionPoint === currentScore ? scoreColor : chartInfoTextColor,
+            color: index === currentScoreIndex ? scoreColor : chartInfoTextColor,
             background: chartLabelBackgroundColor,
           },
           text: `${reactionPoint}`,
@@ -104,7 +105,7 @@ const DecayChart : FunctionComponent<DecayChartProps> = ({ currentScore, decayCu
     });
 
     // circle on the curve that show the current stable score
-    const currentScoreData = decayCurvePoint.find((point) => point.score === currentScore);
+    const currentScoreData = decayCurvePoint.findLast((point) => point.score === currentScore);
     if (currentScoreData !== undefined) {
       pointAnnotations.push({
         x: convertTimeForChart(currentScoreData.updated_at),

opencti-platform/opencti-graphql/src/database/middleware.js

@@ -1632,7 +1632,7 @@ const updateAttributeRaw = async (context, user, instance, inputs, opts = {}) =>
     if (ins) { // If update will really produce a data change
       impactedInputs.push(ins);
       // region Compute the update to push in the stream
-      if (!input.key.startsWith('i_') && input.key !== 'x_opencti_graph_data') {
+      if (!input.key.startsWith('i_') && input.key !== 'x_opencti_graph_data' && !input.key.startsWith('decay_')) {
         const previous = getPreviousInstanceValue(input.key, instance);
         if (input.operation === UPDATE_OPERATION_ADD || input.operation === UPDATE_OPERATION_REMOVE) {
           // Check symmetric difference for add and remove
@@ -2380,14 +2380,20 @@ const upsertElement = async (context, user, element, type, basePatch, opts = {})
     }
   }
   if (type === ENTITY_TYPE_INDICATOR) {
+    // Do not compute decay again when base score does not change
     if (updatePatch.decay_applied_rule && updatePatch.decay_base_score === element.decay_base_score) {
       logApp.debug('UPSERT INDICATOR -- no decay reset because no score change', { element, basePatch });
-      // Do not compute decay again when base score does not change
+      // don't reset score, valid_from & valid_until
+      updatePatch.x_opencti_score = element.x_opencti_score; // don't change the score
+      updatePatch.valid_from = element.valid_from;
+      updatePatch.valid_until = element.valid_until;
+      // don't reset decay attributes
       updatePatch.decay_base_score_date = element.decay_base_score_date;
       updatePatch.decay_applied_rule = element.decay_applied_rule;
-      updatePatch.decay_history = [];
+      updatePatch.decay_history = []; // History is multiple, forcing to empty array will prevent any modification
       updatePatch.decay_next_reaction_date = element.decay_next_reaction_date;
     } else {
+      // As base_score as change, decay will be reset by upsert
       logApp.debug('UPSERT INDICATOR -- Decay is reset', { element, basePatch });
     }
   }

opencti-platform/opencti-graphql/src/manager/indicatorDecayManager.ts

@@ -1,6 +1,6 @@
 import { type ManagerDefinition, registerManager } from './managerModule';
 import conf, { booleanConf, logApp } from '../config/conf';
-import { executionContext, SYSTEM_USER } from '../utils/access';
+import { DECAY_MANAGER_USER, executionContext } from '../utils/access';
 import { findIndicatorsForDecay, updateIndicatorDecayScore } from '../modules/indicator/indicator-domain';
 
 const INDICATOR_DECAY_MANAGER_ENABLED = booleanConf('indicator_decay_manager:enabled', true);
@@ -15,12 +15,12 @@ const BATCH_SIZE = conf.get('indicator_decay_manager:batch_size') || 10000;
  */
 export const indicatorDecayHandler = async () => {
   const context = executionContext('indicator_decay_manager');
-  const indicatorsToUpdate = await findIndicatorsForDecay(context, SYSTEM_USER, BATCH_SIZE);
+  const indicatorsToUpdate = await findIndicatorsForDecay(context, DECAY_MANAGER_USER, BATCH_SIZE);
   let errorCount = 0;
   for (let i = 0; i < indicatorsToUpdate.length; i += 1) {
     try {
       const indicator = indicatorsToUpdate[i];
-      await updateIndicatorDecayScore(context, SYSTEM_USER, indicator);
+      await updateIndicatorDecayScore(context, DECAY_MANAGER_USER, indicator);
     } catch (e) {
       logApp.warn(e, `[OPENCTI-MODULE] Error when processing decay for ${indicatorsToUpdate[i].id}, skipping.`);
       errorCount += 1;

opencti-platform/opencti-graphql/src/modules/decayRule/decayRule-domain.ts

@@ -29,6 +29,7 @@ const DECAY_FACTOR: number = 3.0;
 export interface DecayChartData {
   live_score_serie: DecayHistory[]
 }
+
 export interface DecayModel {
   decay_lifetime: number // in days
   decay_pound: number // can be changed in other model when feature is ready.
@@ -171,7 +172,7 @@ export const countAppliedIndicators = async (context: AuthContext, user: AuthUse
 /**
  * Compute all time scores needed to draw the chart from base score to 0.
  */
-export const computeScoreList = (maxScore:number): number[] => {
+export const computeScoreList = (maxScore: number): number[] => {
   const scoreArray: number[] = [];
   for (let i = maxScore; i >= 0; i -= 1) {
     scoreArray.push(i);
@@ -186,6 +187,9 @@ export const computeScoreList = (maxScore:number): number[] => {
  * @param model decay configuration to use.
  */
 export const computeTimeFromExpectedScore = (initialScore: number, score: number, model: DecayModel) => {
+  if (initialScore === 0) { // Can't divide by 0 when the initial score is 0
+    return 0;
+  }
   if (model.decay_pound && model.decay_lifetime) {
     return (Math.E ** (Math.log(1 - (score / initialScore)) * (DECAY_FACTOR * model.decay_pound))) * model.decay_lifetime;
   }
@@ -374,7 +378,7 @@ export const computeScoreFromExpectedTime = (initialScore: number, daysFromStart
   return initialScore * (1 - ((daysFromStart / rule.decay_lifetime) ** (1 / (DECAY_FACTOR * rule.decay_pound))));
 };
 
-export const computeDecayPointReactionDate = (initialScore: number, stableScore: number, model: DecayModel, startDate: Moment, decayPoint: number) => {
+export const computeDecayPointReactionDate = (initialScore: number, model: DecayModel, startDate: Moment, decayPoint: number) => {
   const daysDelay = computeTimeFromExpectedScore(initialScore, decayPoint, model);
   const duration = moment.duration(daysDelay, 'days');
   return moment(startDate).add(duration.asMilliseconds(), 'ms').toDate();
@@ -383,7 +387,7 @@ export const computeDecayPointReactionDate = (initialScore: number, stableScore:
 export const computeNextScoreReactionDate = (initialScore: number, stableScore: number, model: DecayModel, startDate: Moment) => {
   if (model.decay_points && model.decay_points.length > 0) {
     const nextKeyPoint = model.decay_points.find((p) => p < stableScore) || model.decay_revoke_score;
-    return computeDecayPointReactionDate(initialScore, stableScore, model, startDate, nextKeyPoint);
+    return computeDecayPointReactionDate(initialScore, model, startDate, nextKeyPoint);
   }
   return null;
 };

opencti-platform/opencti-graphql/src/modules/indicator/indicator-domain.ts

@@ -255,7 +255,7 @@ export const addIndicator = async (context: AuthContext, user: AuthUser, indicat
       updated_at: validFrom.toDate(),
       score: indicatorBaseScore,
     });
-    const revokeDate = computeDecayPointReactionDate(indicatorBaseScore, indicatorBaseScore, decayRule, validFrom, decayRule.decay_revoke_score);
+    const revokeDate = computeDecayPointReactionDate(indicatorBaseScore, decayRule, validFrom, decayRule.decay_revoke_score);
     finalIndicatorToCreate = {
       ...indicatorToCreate,
       decay_next_reaction_date: nextScoreReactionDate,
@@ -321,7 +321,7 @@ export const indicatorEditField = async (context: AuthContext, user: AuthUser, i
       if (nextScoreReactionDate) {
         finalInput.push({ key: 'decay_next_reaction_date', value: [nextScoreReactionDate.toISOString()] });
       }
-      const newValidUntilDate = computeDecayPointReactionDate(newScore, newScore, model, updateDate, model.decay_revoke_score);
+      const newValidUntilDate = computeDecayPointReactionDate(newScore, model, updateDate, model.decay_revoke_score);
       finalInput.push({ key: 'valid_until', value: [newValidUntilDate.toISOString()] });
     }
   }

opencti-platform/opencti-graphql/src/modules/indicator/indicator.ts

@@ -49,7 +49,7 @@ const INDICATOR_DEFINITION: ModuleDefinition<StoreEntityIndicator, StixIndicator
       multiple: false,
       upsert: true,
       label: 'Decay base score',
-      isFilterable: true,
+      isFilterable: false,
       precision: 'integer',
     },
     {

opencti-platform/opencti-graphql/src/utils/access.ts

@@ -33,6 +33,7 @@ export const ROLE_ADMINISTRATOR = 'Administrator';
 const RETENTION_MANAGER_USER_UUID = '82ed2c6c-eb27-498e-b904-4f2abc04e05f';
 export const RULE_MANAGER_USER_UUID = 'f9d7b43f-b208-4c56-8637-375a1ce84943';
 export const AUTOMATION_MANAGER_USER_UUID = 'c49fe040-2dad-412d-af07-ce639204ad55';
+export const DECAY_MANAGER_USER_UUID = '7f176d74-9084-4d23-8138-22ac78549547';
 export const REDACTED_USER_UUID = '31afac4e-6b99-44a0-b91b-e04738d31461';
 
 export const MEMBER_ACCESS_ALL = 'ALL';
@@ -180,6 +181,37 @@ export const AUTOMATION_MANAGER_USER: AuthUser = {
   },
 };
 
+export const DECAY_MANAGER_USER: AuthUser = {
+  entity_type: 'User',
+  id: DECAY_MANAGER_USER_UUID,
+  internal_id: DECAY_MANAGER_USER_UUID,
+  individual_id: undefined,
+  name: 'DECAY MANAGER',
+  user_email: 'DECAY MANAGER',
+  inside_platform_organization: true,
+  origin: { user_id: DECAY_MANAGER_USER_UUID, socket: 'internal' },
+  roles: [ADMINISTRATOR_ROLE],
+  groups: [],
+  capabilities: [{ name: BYPASS }],
+  organizations: [],
+  allowed_organizations: [],
+  allowed_marking: [],
+  default_marking: [],
+  all_marking: [],
+  api_token: '',
+  account_lock_after_date: undefined,
+  account_status: ACCOUNT_STATUS_ACTIVE,
+  administrated_organizations: [],
+  effective_confidence_level: {
+    max_confidence: 100,
+    overrides: [],
+  },
+  user_confidence_level: {
+    max_confidence: 100,
+    overrides: [],
+  },
+};
+
 export const REDACTED_USER: AuthUser = {
   administrated_organizations: [],
   entity_type: 'User',
@@ -241,6 +273,7 @@ export const INTERNAL_USERS = {
   [RETENTION_MANAGER_USER.id]: RETENTION_MANAGER_USER,
   [RULE_MANAGER_USER.id]: RULE_MANAGER_USER,
   [AUTOMATION_MANAGER_USER.id]: AUTOMATION_MANAGER_USER,
+  [DECAY_MANAGER_USER.id]: DECAY_MANAGER_USER,
   [REDACTED_USER.id]: REDACTED_USER
 };