Comment out the filter logic to verify the replica count increase

crustio · Jul 8, 2024 · 32d9597 · 32d9597
1 parent 20ff586
commit 32d9597
Show file tree

Hide file tree

Showing 4 changed files with 61 additions and 59 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "crust-smanager",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "A storage manager integrated with Crust, IPFS and sWorker(storage inspector) of Crust protocol.",
   "main": "build/src/main.js",
   "repository": "https://github.com/crustio/crust-smanager.git",

diff --git a/src/config/config.schema.ts b/src/config/config.schema.ts
@@ -40,8 +40,8 @@ const schedulerConfig = Joi.object().keys({
   maxPendingTasks: Joi.number().min(1).default(32),
   minFileSize: Joi.number().min(0).default(0),
   maxFileSize: Joi.number().min(0).default(0),
-  minReplicas: Joi.number().min(0).default(40),
-  maxReplicas: Joi.number().min(0).default(100),
+  minReplicas: Joi.number().min(0).default(0),
+  maxReplicas: Joi.number().min(0).default(200),
 });
 
 const sealCoordinatorConfig = Joi.object().keys({

diff --git a/src/tasks/file-retry-task.ts b/src/tasks/file-retry-task.ts
@@ -24,7 +24,7 @@ async function handleRetry(context: AppContext) {
   await database.run(
     `update file_record set status = "failed"
     where status in (${toQuotedList(PendingStatus)})
-    and create_at < ?`,
+    and last_updated < ?`, // Should use last_updated instead of create_at, since replay old order will only update last_updated
     [maxCreateTime],
   );
 

diff --git a/src/tasks/pull-utils.ts b/src/tasks/pull-utils.ts
@@ -47,11 +47,11 @@ const MinLifeTime = Dayjs.duration({
 // TODO: add some tests
 export async function filterFile(
   record: FileRecord,
-  strategey: PullingStrategy,
-  lastBlockTime: BlockAndTime,
+  _strategey: PullingStrategy,
+  _lastBlockTime: BlockAndTime,
   context: AppContext,
 ): Promise<FilterFileResult> {
-  const config = context.config.scheduler;
+  //const config = context.config.scheduler;
   const groupInfo = context.groupInfo;
   try {
     const bn = cidToBigNumber(record.cid);
@@ -62,58 +62,60 @@ export async function filterFile(
     return 'invalidCID';
   }
 
-  const maxReplicas = strategey === 'newFilesWeight' ? 300 : 160;
-  if (!probabilityFilter(context, maxReplicas)) {
-    return 'pfSkipped';
-  }
-  const fileSizeInMb = bytesToMb(record.size);
-  // check min file size limit
-  if (config.minFileSize > 0 && fileSizeInMb < config.minFileSize) {
-    return 'sizeTooSmall';
-  }
-  if (config.maxFileSize > 0 && fileSizeInMb > config.maxFileSize) {
-    return 'sizeTooLarge';
-  }
-  if (
-    strategey === 'dbFilesWeight' &&
-    config.minReplicas > 0 &&
-    record.replicas < config.minReplicas
-  ) {
-    return 'replicasNotEnough';
-  }
-  if (config.maxReplicas > 0 && record.replicas >= config.maxReplicas) {
-    return 'tooManyReplicas';
-  }
-  if (record.indexer === 'dbScan') {
-    // file record has no valid expire_at information
-    if (record.expire_at === 0) {
-      // check how long the file was indexed
-      const createAt = Dayjs.unix(record.create_at);
-      if (
-        Dayjs.duration(Dayjs().diff(createAt)).asSeconds() >
-        MaxNoReplicaDuration.asSeconds()
-      ) {
-        return 'invalidNoReplica';
-      }
-      return 'pendingForReplica';
-    }
-    const expireAt = estimateTimeAtBlock(record.expire_at, lastBlockTime);
-    if (
-      Dayjs.duration(expireAt.diff(Dayjs())).asSeconds() <
-      MinLifeTime.asSeconds()
-    ) {
-      return 'lifeTimeTooShort';
-    }
-  }
-  const sealCoordinator = context.sealCoordinator;
-  if (sealCoordinator != null) {
-    const shouldSeal = await sealCoordinator.markSeal(record.cid);
-    if (shouldSeal.seal && shouldSeal.reason === 'ok') {
-      return 'good';
-    }
-    logger.info(`seal for file "${record.cid}" skipped by seal coordinator`);
-    return 'nodeSkipped';
-  }
+  // 2024/07/08 - Comment out the following filter logic to verify the replica count increase in the whole mainnet
+
+  // const maxReplicas = strategey === 'newFilesWeight' ? 300 : 160;
+  // if (!probabilityFilter(context, maxReplicas)) {
+  //   return 'pfSkipped';
+  // }
+  // const fileSizeInMb = bytesToMb(record.size);
+  // // check min file size limit
+  // if (config.minFileSize > 0 && fileSizeInMb < config.minFileSize) {
+  //   return 'sizeTooSmall';
+  // }
+  // if (config.maxFileSize > 0 && fileSizeInMb > config.maxFileSize) {
+  //   return 'sizeTooLarge';
+  // }
+  // if (
+  //   strategey === 'dbFilesWeight' &&
+  //   config.minReplicas > 0 &&
+  //   record.replicas < config.minReplicas
+  // ) {
+  //   return 'replicasNotEnough';
+  // }
+  // if (config.maxReplicas > 0 && record.replicas >= config.maxReplicas) {
+  //   return 'tooManyReplicas';
+  // }
+  // if (record.indexer === 'dbScan') {
+  //   // file record has no valid expire_at information
+  //   if (record.expire_at === 0) {
+  //     // check how long the file was indexed
+  //     const createAt = Dayjs.unix(record.create_at);
+  //     if (
+  //       Dayjs.duration(Dayjs().diff(createAt)).asSeconds() >
+  //       MaxNoReplicaDuration.asSeconds()
+  //     ) {
+  //       return 'invalidNoReplica';
+  //     }
+  //     return 'pendingForReplica';
+  //   }
+  //   const expireAt = estimateTimeAtBlock(record.expire_at, lastBlockTime);
+  //   if (
+  //     Dayjs.duration(expireAt.diff(Dayjs())).asSeconds() <
+  //     MinLifeTime.asSeconds()
+  //   ) {
+  //     return 'lifeTimeTooShort';
+  //   }
+  // }
+  // const sealCoordinator = context.sealCoordinator;
+  // if (sealCoordinator != null) {
+  //   const shouldSeal = await sealCoordinator.markSeal(record.cid);
+  //   if (shouldSeal.seal && shouldSeal.reason === 'ok') {
+  //     return 'good';
+  //   }
+  //   logger.info(`seal for file "${record.cid}" skipped by seal coordinator`);
+  //   return 'nodeSkipped';
+  // }
 
   return 'good';
 }