1. Connect to crust gateway peers explicitly to improve the ipfs file…

… pin success rate

2. The connect need to be done periodically because IPFS may drop the connection if it's idle for sometime
3. Periodically download peers list from cloud to update the local cache to keep it updated automatically without software upgrade

package.json

@@ -1,6 +1,6 @@
 {
   "name": "crust-smanager",
-  "version": "2.0.3",
+  "version": "2.0.4",
   "description": "A storage manager integrated with Crust, IPFS and sWorker(storage inspector) of Crust protocol.",
   "main": "build/src/main.js",
   "repository": "https://github.com/crustio/crust-smanager.git",

src/ipfs/index.ts

@@ -97,4 +97,12 @@ export default class IpfsApi {
   async repoGC(to: number): Promise<void> {
     await this.ipfs.repo.gc({ timeout: to });
   }
+
+  /**
+   * ipfs swarm connect
+   * @param peerAddress peer address
+   */
+  async connectPeer(peerAddress: string): Promise<void> {
+    await this.ipfs.swarm.connect(peerAddress);
+  }
 }

src/tasks/index.ts

@@ -11,6 +11,8 @@ import { createPullSchedulerTask } from './pull-scheduler-task';
 import { createSealCleanupTask } from './seal-cleanup-task';
 import { createSealStatuUpdater } from './seal-status-updater-task';
 import { createTelemetryReportTask } from './telemetry-task';
+import { createIpfsUpdatePeersListTask } from './ipfs-update-peers-list-task';
+import { createIpfsConnectPeersTask } from './ipfs-connect-peers-task';
 
 /**
  * create simpile tasks which only handle start/stop
@@ -29,6 +31,8 @@ export async function createSimpleTasks(
     createGroupInfoUpdateTask,
     createNodeInfoUpdateTask,
     createSealCleanupTask,
+    createIpfsUpdatePeersListTask,
+    createIpfsConnectPeersTask
   ];
   return Bluebird.mapSeries(tasks, (t) => {
     return t(context, logger);

src/tasks/ipfs-connect-peers-task.ts

@@ -0,0 +1,40 @@
+import { Logger } from 'winston';
+import { AppContext } from '../types/context';
+import { SimpleTask } from '../types/tasks';
+import { makeIntervalTask } from './task-utils';
+import { CrustGWPeers, DefaultCrustGWPeers } from './ipfs-update-peers-list-task';
+
+
+/**
+ * task to connect to dedicated crust gateway peers periodly to improve the ipfs file pin success rate
+ */
+async function handleIpfsConnectPeers(context: AppContext, logger: Logger): Promise<void> {
+
+  const peersToConnect = CrustGWPeers.size > 0 ? Array.from(CrustGWPeers): DefaultCrustGWPeers;
+
+  let successCount = 0;
+  for (const peer of peersToConnect) {
+    try {
+      await context.ipfsApi.connectPeer(peer);
+      successCount++;
+    } catch (error) {
+      logger.error(`Failed to connect to peer: '${peer}'. Error: ${error}`);
+    }
+  }
+  logger.info(`Connect to ${successCount} dedicated crust gateway peers.`);
+}
+
+export async function createIpfsConnectPeersTask(
+  context: AppContext,
+  loggerParent: Logger,
+): Promise<SimpleTask> {
+  const ipfsInterval = 3 * 60 * 1000; // Connect to dedicated peers every 3 minutes
+  return makeIntervalTask(
+    30 * 1000,
+    ipfsInterval,
+    'ipfs-connect-peers',
+    context,
+    loggerParent,
+    handleIpfsConnectPeers,
+  );
+}

src/tasks/ipfs-update-peers-list-task.ts

@@ -0,0 +1,70 @@
+import { Logger } from 'winston';
+import { AppContext } from '../types/context';
+import { SimpleTask } from '../types/tasks';
+import { makeIntervalTask } from './task-utils';
+import axios from 'axios';
+import https from 'https';
+
+export const CrustGWPeers = new Set<string>();
+
+export const DefaultCrustGWPeers: string[] = [
+  '/ip4/60.188.112.180/tcp/14001/p2p/12D3KooWP4PpTGcVQsiSCRS6E7X31j6ivWNLdfSMdgErQXXv3WoL',
+  '/ip4/60.188.112.178/tcp/14001/p2p/12D3KooWMcAHcs97R49PLZjGUKDbP1fr9iijeepod8fkktHTLCgN',
+  '/ip4/45.77.251.250/tcp/4001/p2p/12D3KooWAEw4RBa1oRyqMXUCzyAbwgvDduRFvi1KzaahSb5Pk61q',
+  '/ip4/103.73.242.131/tcp/14001/p2p/12D3KooWDbPeYWubpE2tyDs3Gp7UgNGmHRgMvyALRNCeL3s77fgv'];
+
+const PeersListURL = 'https://gw-peer-address.crust.network/crust-gw-peer-address';
+
+/**
+ * task to download and refresh the peers cache list periodly
+ */
+async function handleUpdatePeersList(_context: AppContext, logger: Logger): Promise<void> {
+  try {
+    const agent = new https.Agent({
+      rejectUnauthorized: false, // Ignore the SSL certificate
+    });
+    const response = await axios.get(PeersListURL, {
+        responseType: 'text',
+        timeout: 10 * 1000,
+        httpsAgent: agent
+      });
+
+    if (response.status !== 200) {
+      logger.error(`Download gateway peers list failed. Status Code: ${response.status}`);
+      return;
+    }
+
+    // Split the data, each line represents one peer address
+    const lines = response.data.split('\n').filter(line => line.trim() != '');
+
+    logger.info(`Update the gateway peers list with ${lines.length} peers`);
+    CrustGWPeers.clear();
+    lines.forEach((line) => {
+      CrustGWPeers.add(line);
+      logger.debug(line);
+    });
+
+  } catch(error) {
+    logger.error(`Download the gateway peers list failed: ${error}`);
+    // Request failed, keep the existing peers list, or use the default list if empty
+    if (CrustGWPeers.size === 0) {
+      logger.info('Use the default gateway peers list');
+      DefaultCrustGWPeers.forEach((peer) => CrustGWPeers.add(peer));
+    }
+  }
+}
+
+export async function createIpfsUpdatePeersListTask(
+  context: AppContext,
+  loggerParent: Logger,
+): Promise<SimpleTask> {
+  const ipfsInterval = 60 * 60 * 1000; // Download and refresh the peers cache list every hour
+  return makeIntervalTask(
+    15 * 1000,
+    ipfsInterval,
+    'ipfs-update-peers-list',
+    context,
+    loggerParent,
+    handleUpdatePeersList,
+  );
+}