Create a web3 http provider that can handle massive JSON blobs without crashing due to string or buffer length issues

packages/core/lib/command-utils.js

@@ -11,6 +11,7 @@ const {
 } = require("./commands/commands");
 const Web3 = require("web3");
 const TruffleError = require("@truffle/error");
+const { StreamingWeb3HttpProvider } = require("@truffle/stream-provider");
 
 const defaultHost = "127.0.0.1";
 const managedGanacheDefaultPort = 9545;
@@ -397,7 +398,7 @@ const deriveConfigEnvironment = function (detectedConfig, network, url) {
     configuredNetwork = {
       network_id: customConfig.network_id || defaultNetworkId,
       provider: function () {
-        return new Web3.providers.HttpProvider(configuredNetworkUrl, {
+        return new StreamingWeb3HttpProvider(configuredNetworkUrl, {
           keepAlive: false
         });
       },

packages/core/package.json

@@ -54,6 +54,7 @@
     "@truffle/source-fetcher": "^1.0.30",
     "@truffle/spinners": "^0.2.3",
     "@truffle/test": "^0.1.15",
+    "@truffle/stream-provider": "^0.0.1",
     "@truffle/workflow-compile": "^4.0.54",
     "JSONStream": "^1.3.5",
     "address": "^1.1.2",

packages/environment/environment.js

@@ -1,11 +1,11 @@
-const Web3 = require("web3");
 const { createInterfaceAdapter } = require("@truffle/interface-adapter");
 const expect = require("@truffle/expect");
 const TruffleError = require("@truffle/error");
 const { Resolver } = require("@truffle/resolver");
 const Artifactor = require("@truffle/artifactor");
 const Ganache = require("ganache");
 const Provider = require("@truffle/provider");
+const { StreamingWeb3HttpProvider } = require("@truffle/stream-provider");
 
 const Environment = {
   // It's important config is a Config object and not a vanilla object
@@ -77,7 +77,7 @@ const Environment = {
       ...config.networks[network],
       network_id: ganacheOptions.network_id,
       provider: function () {
-        return new Web3.providers.HttpProvider(url, { keepAlive: false });
+        return new StreamingWeb3HttpProvider(url, { keepAlive: false });
       }
     };
 

packages/environment/package.json

@@ -27,6 +27,7 @@
     "@truffle/interface-adapter": "^0.5.30",
     "@truffle/provider": "^0.3.6",
     "@truffle/resolver": "^9.0.36",
+    "@truffle/stream-provider": "^0.0.1",
     "chalk": "^4.1.2",
     "ganache": "7.7.7",
     "node-ipc": "9.2.1",

packages/provider/index.js

@@ -3,6 +3,7 @@ const Web3 = require("web3");
 const { createInterfaceAdapter } = require("@truffle/interface-adapter");
 const wrapper = require("./wrapper");
 const DEFAULT_NETWORK_CHECK_TIMEOUT = 5000;
+const { StreamingWeb3HttpProvider } = require("@truffle/stream-provider");
 
 module.exports = {
   wrap: function (provider, options) {
@@ -25,7 +26,7 @@ module.exports = {
         options.url || "ws://" + options.host + ":" + options.port
       );
     } else {
-      provider = new Web3.providers.HttpProvider(
+      provider = new StreamingWeb3HttpProvider(
         options.url || `http://${options.host}:${options.port}`,
         { keepAlive: false }
       );
@@ -49,17 +50,18 @@ module.exports = {
     return new Promise((resolve, reject) => {
       const noResponseFromNetworkCall = setTimeout(() => {
         let errorMessage =
-          "There was a timeout while attempting to connect to the network at " + host +
+          "There was a timeout while attempting to connect to the network at " +
+          host +
           ".\n       Check to see that your provider is valid." +
           "\n       If you have a slow internet connection, try configuring a longer " +
           "timeout in your Truffle config. Use the " +
           "networks[networkName].networkCheckTimeout property to do this.";
 
-          if (network === "dashboard") {
-            errorMessage +=
-              "\n       Also make sure that your Truffle Dashboard browser " +
-              "tab is open and connected to MetaMask.";
-          }
+        if (network === "dashboard") {
+          errorMessage +=
+            "\n       Also make sure that your Truffle Dashboard browser " +
+            "tab is open and connected to MetaMask.";
+        }
 
         throw new Error(errorMessage);
       }, networkCheckTimeout);
@@ -75,7 +77,9 @@ module.exports = {
           } catch (error) {
             console.log(
               "> Something went wrong while attempting to connect to the " +
-                "network at " + host + ". Check your network configuration."
+                "network at " +
+                host +
+                ". Check your network configuration."
             );
             clearTimeout(noResponseFromNetworkCall);
             clearTimeout(networkCheck);
@@ -86,5 +90,5 @@ module.exports = {
         }, networkCheckDelay);
       })();
     });
-  },
+  }
 };

packages/provider/package.json

@@ -22,6 +22,7 @@
   "dependencies": {
     "@truffle/error": "^0.2.0",
     "@truffle/interface-adapter": "^0.5.30",
+    "@truffle/stream-provider": "^0.0.1",
     "debug": "^4.3.1",
     "web3": "1.8.2"
   },

packages/stream-provider/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+dist

packages/stream-provider/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2021 ConsenSys, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/stream-provider/README.md

@@ -0,0 +1,6 @@
+# @truffle/stream-provider
+
+A Web3.providers.HttpProvider that can handle JSON blobs greater than 1 GB.
+
+Large blob handling is currently only used for `debug_traceTransaction`
+results.

packages/stream-provider/lib/index.ts

@@ -0,0 +1,102 @@
+import axios, { AxiosRequestConfig } from "axios";
+import type { JsonRpcResponse } from "web3-core-helpers";
+import { errors } from "web3-core-helpers";
+import _Web3HttpProvider, { HttpProvider } from "web3-providers-http";
+const JSONStream = require("JSONStream");
+
+// they export types, but in the wrong place
+const Web3HttpProvider = _Web3HttpProvider as any as typeof HttpProvider;
+
+export class StreamingWeb3HttpProvider extends Web3HttpProvider {
+  /**
+   * Should be used to make async request
+   *
+   * @method send
+   * @param {Object} payload
+   * @param {Function} callback triggered on end with (err, result)
+   */
+  send(
+    payload: any,
+    callback: (error: Error | null, result: JsonRpcResponse | undefined) => void
+  ) {
+    if (
+      typeof payload === "object" &&
+      payload.method === "debug_traceTransaction"
+    ) {
+      const requestOptions: AxiosRequestConfig = {
+        method: "post",
+        url: this.host,
+        responseType: "stream",
+        data: payload,
+        timeout: this.timeout,
+        withCredentials: this.withCredentials,
+        headers: this.headers
+          ? this.headers.reduce((acc, header) => {
+              acc[header.name] = header.value;
+              return acc;
+            }, {} as Record<string, string>)
+          : undefined
+      };
+      // transitional.clarifyTimeoutError is required so we can detect and emit
+      // a timeout error the way web3 already does
+      (requestOptions as any).transitional = {
+        clarifyTimeoutError: true
+      };
+      const agents = {
+        httpsAgent: (this as any).httpsAgent,
+        httpAgent: (this as any).httpAgent,
+        baseUrl: (this as any).baseUrl
+      };
+      if (this.agent) {
+        agents.httpsAgent = this.agent.https;
+        agents.httpAgent = this.agent.http;
+        agents.baseUrl = this.agent.baseUrl;
+      }
+      requestOptions.httpAgent = agents.httpAgent;
+      requestOptions.httpsAgent = agents.httpsAgent;
+      requestOptions.baseURL = agents.baseUrl;
+
+      axios(requestOptions)
+        .then(async response => {
+          let error = null;
+          let result: any = {};
+          const stream = response.data.pipe(
+            JSONStream.parse([{ emitKey: true }])
+          );
+          try {
+            result = await new Promise((resolve, reject) => {
+              let result: any = {};
+              stream.on("data", (data: any) => {
+                const { key, value } = data;
+                result[key] = value;
+              });
+              stream.on("error", (error: Error) => {
+                reject(error);
+              });
+              stream.on("end", () => {
+                resolve(result);
+              });
+            });
+          } catch (e) {
+            error = errors.InvalidResponse(e);
+          }
+          this.connected = true;
+          // process.nextTick so an exception thrown in the callback doesn't
+          // bubble back up to here
+          process.nextTick(callback, error, result);
+        })
+        .catch(error => {
+          this.connected = false;
+          if (error.code === "ETIMEDOUT") {
+            // web3 passes timeout as a number to ConnectionTimeout, despite the
+            // type requiring a string
+            callback(errors.ConnectionTimeout(this.timeout as any), undefined);
+          } else {
+            callback(errors.InvalidConnection(this.host), undefined);
+          }
+        });
+    } else {
+      return super.send(payload, callback);
+    }
+  }
+}

packages/stream-provider/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@truffle/stream-provider",
+  "description": "Enables streaming results into memory from debug_traceTransaction",
+  "license": "MIT",
+  "author": "David Murdoch <[email protected]>",
+  "homepage": "https://github.com/trufflesuite/truffle/tree/master/packages/stream-provider#readme",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/trufflesuite/truffle.git",
+    "directory": "packages/stream-provider"
+  },
+  "bugs": {
+    "url": "https://github.com/trufflesuite/truffle/issues"
+  },
+  "version": "0.0.1",
+  "main": "dist/index.js",
+  "files": [
+    "dist"
+  ],
+  "directories": {
+    "lib": "lib"
+  },
+  "scripts": {
+    "build": "tsc",
+    "prepare": "yarn build",
+    "test:ts": "NODE_OPTIONS=--max-old-space-size=4096 mocha -r ts-node/register test/*.ts",
+    "test": "yarn test:ts"
+  },
+  "types": "dist/index.d.ts",
+  "dependencies": {
+    "web3-providers-http": "^1.6.0",
+    "web3-core-helpers": "1.5.3",
+    "JSONStream": "1.3.5",
+    "axios": "^0.21.1"
+  },
+  "devDependencies": {
+    "mocha": "8.1.2"
+  },
+  "keywords": [
+    "ethereum",
+    "etherscan",
+    "ipfs",
+    "solidity",
+    "web3"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
+}

packages/stream-provider/test/stream.ts

@@ -0,0 +1,65 @@
+import assert from "assert";
+import http, { Server, ServerResponse } from "http";
+import { StreamingWeb3HttpProvider } from "../lib/";
+
+describe("test", () => {
+  const PORT = 9451;
+  let server: Server;
+  const SIZE = (1000000000 / 2) | 0;
+  beforeEach(() => {
+    server = http.createServer();
+    server
+      .on("request", (request, response: ServerResponse) => {
+        let body: Buffer[] = [];
+        request
+          .on("data", (chunk: Buffer) => {
+            body.push(chunk);
+          })
+          .on("end", () => {
+            response.writeHead(200, "OK", {
+              "content-type": "application/json"
+            });
+            const prefix = '{"id":1,"result":{"structLogs":["';
+            const postfix = '"]}}';
+            response.write(prefix);
+            // .5 gigs of utf-8 0s
+            const lots = Buffer.allocUnsafe(SIZE).fill(48);
+            response.write(lots);
+            response.write('","');
+            response.write(lots);
+            response.write('","');
+            response.write(lots);
+            response.write('","');
+            response.write(lots);
+            response.write(postfix);
+            response.end();
+          });
+      })
+      .listen(PORT);
+  });
+  afterEach(async () => {
+    server && server.close();
+  });
+  it("handles giant traces", async function () {
+    this.timeout(0);
+    const provider = new StreamingWeb3HttpProvider(`http://localhost:${PORT}`);
+    const { result } = await new Promise<any>((resolve, reject) => {
+      provider.send(
+        {
+          id: 1,
+          jsonrpc: "2.0",
+          method: "debug_traceTransaction",
+          params: ["0x1234"]
+        },
+        (err, result) => {
+          if (err) return void reject(err);
+          resolve(result);
+        }
+      );
+    });
+    assert.strictEqual(result.structLogs.length, 4);
+    result.structLogs.forEach((log: Buffer) => {
+      assert.strictEqual(log.length, SIZE);
+    });
+  });
+});

packages/stream-provider/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "declaration": true,
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "target": "es2016",
+    "downlevelIteration": true,
+    "noImplicitAny": true,
+    "moduleResolution": "node",
+    "sourceMap": true,
+    "outDir": "dist",
+    "baseUrl": ".",
+    "lib": [
+      "es2017"
+    ],
+    "paths": {},
+    "rootDir": "lib",
+    "types": [
+      "mocha",
+      "node"
+    ]
+  },
+  "include": [
+    "./lib/**/*.ts"
+  ]
+}