From 32d959722bbcaa44bfeb6cec5c90a6db730714e6 Mon Sep 17 00:00:00 2001 From: BrianWu Date: Mon, 8 Jul 2024 14:52:51 +0800 Subject: [PATCH] Comment out the filter logic to verify the replica count increase --- package.json | 2 +- src/config/config.schema.ts | 4 +- src/tasks/file-retry-task.ts | 2 +- src/tasks/pull-utils.ts | 112 ++++++++++++++++++----------------- 4 files changed, 61 insertions(+), 59 deletions(-) diff --git a/package.json b/package.json index 94c696f..47e73d9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "crust-smanager", - "version": "2.0.0", + "version": "2.0.1", "description": "A storage manager integrated with Crust, IPFS and sWorker(storage inspector) of Crust protocol.", "main": "build/src/main.js", "repository": "https://github.com/crustio/crust-smanager.git", diff --git a/src/config/config.schema.ts b/src/config/config.schema.ts index 98b37ff..baa10d9 100644 --- a/src/config/config.schema.ts +++ b/src/config/config.schema.ts @@ -40,8 +40,8 @@ const schedulerConfig = Joi.object().keys({ maxPendingTasks: Joi.number().min(1).default(32), minFileSize: Joi.number().min(0).default(0), maxFileSize: Joi.number().min(0).default(0), - minReplicas: Joi.number().min(0).default(40), - maxReplicas: Joi.number().min(0).default(100), + minReplicas: Joi.number().min(0).default(0), + maxReplicas: Joi.number().min(0).default(200), }); const sealCoordinatorConfig = Joi.object().keys({ diff --git a/src/tasks/file-retry-task.ts b/src/tasks/file-retry-task.ts index 8cf0df0..b8dbe6c 100644 --- a/src/tasks/file-retry-task.ts +++ b/src/tasks/file-retry-task.ts @@ -24,7 +24,7 @@ async function handleRetry(context: AppContext) { await database.run( `update file_record set status = "failed" where status in (${toQuotedList(PendingStatus)}) - and create_at < ?`, + and last_updated < ?`, // Should use last_updated instead of create_at, since replay old order will only update last_updated [maxCreateTime], ); diff --git a/src/tasks/pull-utils.ts b/src/tasks/pull-utils.ts index 70eaa9f..6165c9e 100644 --- a/src/tasks/pull-utils.ts +++ b/src/tasks/pull-utils.ts @@ -47,11 +47,11 @@ const MinLifeTime = Dayjs.duration({ // TODO: add some tests export async function filterFile( record: FileRecord, - strategey: PullingStrategy, - lastBlockTime: BlockAndTime, + _strategey: PullingStrategy, + _lastBlockTime: BlockAndTime, context: AppContext, ): Promise { - const config = context.config.scheduler; + //const config = context.config.scheduler; const groupInfo = context.groupInfo; try { const bn = cidToBigNumber(record.cid); @@ -62,58 +62,60 @@ export async function filterFile( return 'invalidCID'; } - const maxReplicas = strategey === 'newFilesWeight' ? 300 : 160; - if (!probabilityFilter(context, maxReplicas)) { - return 'pfSkipped'; - } - const fileSizeInMb = bytesToMb(record.size); - // check min file size limit - if (config.minFileSize > 0 && fileSizeInMb < config.minFileSize) { - return 'sizeTooSmall'; - } - if (config.maxFileSize > 0 && fileSizeInMb > config.maxFileSize) { - return 'sizeTooLarge'; - } - if ( - strategey === 'dbFilesWeight' && - config.minReplicas > 0 && - record.replicas < config.minReplicas - ) { - return 'replicasNotEnough'; - } - if (config.maxReplicas > 0 && record.replicas >= config.maxReplicas) { - return 'tooManyReplicas'; - } - if (record.indexer === 'dbScan') { - // file record has no valid expire_at information - if (record.expire_at === 0) { - // check how long the file was indexed - const createAt = Dayjs.unix(record.create_at); - if ( - Dayjs.duration(Dayjs().diff(createAt)).asSeconds() > - MaxNoReplicaDuration.asSeconds() - ) { - return 'invalidNoReplica'; - } - return 'pendingForReplica'; - } - const expireAt = estimateTimeAtBlock(record.expire_at, lastBlockTime); - if ( - Dayjs.duration(expireAt.diff(Dayjs())).asSeconds() < - MinLifeTime.asSeconds() - ) { - return 'lifeTimeTooShort'; - } - } - const sealCoordinator = context.sealCoordinator; - if (sealCoordinator != null) { - const shouldSeal = await sealCoordinator.markSeal(record.cid); - if (shouldSeal.seal && shouldSeal.reason === 'ok') { - return 'good'; - } - logger.info(`seal for file "${record.cid}" skipped by seal coordinator`); - return 'nodeSkipped'; - } + // 2024/07/08 - Comment out the following filter logic to verify the replica count increase in the whole mainnet + + // const maxReplicas = strategey === 'newFilesWeight' ? 300 : 160; + // if (!probabilityFilter(context, maxReplicas)) { + // return 'pfSkipped'; + // } + // const fileSizeInMb = bytesToMb(record.size); + // // check min file size limit + // if (config.minFileSize > 0 && fileSizeInMb < config.minFileSize) { + // return 'sizeTooSmall'; + // } + // if (config.maxFileSize > 0 && fileSizeInMb > config.maxFileSize) { + // return 'sizeTooLarge'; + // } + // if ( + // strategey === 'dbFilesWeight' && + // config.minReplicas > 0 && + // record.replicas < config.minReplicas + // ) { + // return 'replicasNotEnough'; + // } + // if (config.maxReplicas > 0 && record.replicas >= config.maxReplicas) { + // return 'tooManyReplicas'; + // } + // if (record.indexer === 'dbScan') { + // // file record has no valid expire_at information + // if (record.expire_at === 0) { + // // check how long the file was indexed + // const createAt = Dayjs.unix(record.create_at); + // if ( + // Dayjs.duration(Dayjs().diff(createAt)).asSeconds() > + // MaxNoReplicaDuration.asSeconds() + // ) { + // return 'invalidNoReplica'; + // } + // return 'pendingForReplica'; + // } + // const expireAt = estimateTimeAtBlock(record.expire_at, lastBlockTime); + // if ( + // Dayjs.duration(expireAt.diff(Dayjs())).asSeconds() < + // MinLifeTime.asSeconds() + // ) { + // return 'lifeTimeTooShort'; + // } + // } + // const sealCoordinator = context.sealCoordinator; + // if (sealCoordinator != null) { + // const shouldSeal = await sealCoordinator.markSeal(record.cid); + // if (shouldSeal.seal && shouldSeal.reason === 'ok') { + // return 'good'; + // } + // logger.info(`seal for file "${record.cid}" skipped by seal coordinator`); + // return 'nodeSkipped'; + // } return 'good'; }