[backend] migration to delete files duplicates (#8235)

opencti-platform/opencti-graphql/src/migrations/1726041032380-delete-files-duplicates.js

@@ -0,0 +1,34 @@
+import { uniq } from 'ramda';
+import { logApp } from '../config/conf';
+import { listAllEntities } from '../database/middleware-loader';
+import { ENTITY_TYPE_INTERNAL_FILE } from '../schema/internalObject';
+import { ADMIN_USER, testContext } from '../../tests/utils/testQuery';
+import { elDeleteInstances, elFindByIds } from '../database/engine';
+
+const message = '[MIGRATION] Delete potential files duplicates after index rollover';
+
+export const up = async (next) => {
+  logApp.info(`${message} > started`);
+  const context = testContext;
+  const files = await listAllEntities(context, ADMIN_USER, [ENTITY_TYPE_INTERNAL_FILE], {});
+  const filesToStudy = files.filter((f) => f._index !== 'opencti_internal_objects-000001'); // files in index 2 and more
+  const hitsToDelete = [];
+  for (let i = 0; i < filesToStudy.length; i += 1) {
+    const fileToStudy = filesToStudy[i];
+    const hits = await elFindByIds(context, ADMIN_USER, fileToStudy.id, {});
+    if (hits.length > 1) { // if a duplicate exists
+      const maximumLastModified = hits.map((h) => h.lastModified).sort((a, b) => b.localeCompare(a))[0];
+      const olderHits = hits.filter((h) => h.lastModified !== maximumLastModified);
+      hitsToDelete.push(olderHits);
+    }
+  }
+  const finalHitsToDelete = uniq(hitsToDelete.map((h) => [h._index, h.internal_id]));
+  logApp.info(`Deleting ${finalHitsToDelete.length} files that have duplicates.`);
+  await elDeleteInstances(finalHitsToDelete);
+  logApp.info(`${message} > done`);
+  next();
+};
+
+export const down = async (next) => {
+  next();
+};