Feature/puppeteer timeout

Readme.md

@@ -284,7 +284,9 @@ Puppeteer (Headless Chrome) may fail due to sandboxing issues. To get around thi
 you may use:
 
 ```json
-"puppeteerArgs": ["--no-sandbox", "--disable-setuid-sandbox"]
+"puppeteer": { 
+  "args": ["--no-sandbox", "--disable-setuid-sandbox"]
+}
 ```
 
 Read more about [puppeteer troubleshooting](https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md).

index.js

@@ -20,15 +20,15 @@ const defaultOptions = {
   concurrency: 4,
   include: ["/"],
   userAgent: "ReactSnap",
-  // 4 params below will be refactored to one: `puppeteer: {}`
-  // https://github.com/stereobooster/react-snap/issues/120
-  headless: true,
   puppeteer: {
-    cache: true
+    args: [],
+    cache: true,
+    headless: true,
+    executablePath: undefined,
+    ignoreHTTPSErrors: false,
+    handleSIGINT: false,
+    timeout: 2 * 60 * 1000 // 2mins
   },
-  puppeteerArgs: [],
-  puppeteerExecutablePath: undefined,
-  puppeteerIgnoreHTTPSErrors: false,
   publicPath: "/",
   minifyCss: {},
   minifyHtml: {
@@ -82,10 +82,13 @@ const defaultOptions = {
  * @param {{source: ?string, destination: ?string, include: ?Array<string>, sourceMaps: ?boolean, skipThirdPartyRequests: ?boolean }} userOptions
  * @return {*}
  */
-const defaults = userOptions => {
+const defaults = (userOptions = {}) => {
+  const puppeteer = { ...defaultOptions.puppeteer, ...userOptions.puppeteer };
+
   const options = {
     ...defaultOptions,
-    ...userOptions
+    ...userOptions,
+    puppeteer
   };
   options.destination = options.destination || options.source;
 
@@ -514,8 +517,9 @@ const fixParcelChunksIssue = ({
 }) => {
   return page.evaluate(
     (basePath, http2PushManifest, inlineCss) => {
-      const localScripts = Array.from(document.scripts)
-        .filter(x => x.src && x.src.startsWith(basePath))
+      const localScripts = Array.from(document.scripts).filter(
+        x => x.src && x.src.startsWith(basePath)
+      );
 
       const mainRegexp = /main\.[\w]{8}\.js/;
       const mainScript = localScripts.find(x => mainRegexp.test(x.src));
@@ -840,7 +844,7 @@ const run = async (userOptions, { fs } = { fs: nativeFs }) => {
         );
         routePath = normalizePath(routePath);
         if (routePath !== newPath) {
-          console.log(newPath)
+          console.log(newPath);
           console.log(`💬  in browser redirect (${newPath})`);
           addToQueue(newRoute);
         }

package.json

@@ -17,7 +17,7 @@
     "html-minifier": "4.0.0",
     "minimalcss": "0.8.2",
     "mkdirp": "0.5.1",
-    "puppeteer": "^1.8.0",
+    "puppeteer": "^5.3.1",
     "serve-static": "1.14.1",
     "sourcemapped-stacktrace-node": "2.1.8"
   },

run.js

@@ -12,8 +12,8 @@ const {
 const publicUrl = process.env.PUBLIC_URL || homepage;
 
 const reactScriptsVersion = parseInt(
-  (devDependencies && devDependencies["react-scripts"])
-  || (dependencies && dependencies["react-scripts"])
+  (devDependencies && devDependencies["react-scripts"]) ||
+    (dependencies && dependencies["react-scripts"])
 );
 let fixWebpackChunksIssue;
 switch (reactScriptsVersion) {
@@ -26,13 +26,13 @@ switch (reactScriptsVersion) {
 }
 
 const parcel = Boolean(
-  (devDependencies && devDependencies["parcel-bundler"])
-  || (dependencies && dependencies["parcel-bundler"])
+  (devDependencies && devDependencies["parcel-bundler"]) ||
+    (dependencies && dependencies["parcel-bundler"])
 );
 
 if (parcel) {
   if (fixWebpackChunksIssue) {
-    console.log("Detected both Parcel and CRA. Fixing chunk names for CRA!")
+    console.log("Detected both Parcel and CRA. Fixing chunk names for CRA!");
   } else {
     fixWebpackChunksIssue = "Parcel";
   }
@@ -45,4 +45,4 @@ run({
 }).catch(error => {
   console.error(error);
   process.exit(1);
-});
+});

src/puppeteer_utils.js

@@ -108,14 +108,16 @@ const enableLogging = opt => {
 const getLinks = async opt => {
   const { page } = opt;
   const anchors = await page.evaluate(() =>
-    Array.from(document.querySelectorAll("a,link[rel='alternate']")).map(anchor => {
-      if (anchor.href.baseVal) {
-        const a = document.createElement("a");
-        a.href = anchor.href.baseVal;
-        return a.href;
+    Array.from(document.querySelectorAll("a,link[rel='alternate']")).map(
+      anchor => {
+        if (anchor.href.baseVal) {
+          const a = document.createElement("a");
+          a.href = anchor.href.baseVal;
+          return a.href;
+        }
+        return anchor.href;
       }
-      return anchor.href;
-    })
+    )
   );
 
   const iframes = await page.evaluate(() =>
@@ -184,7 +186,12 @@ const crawl = async opt => {
     // Port can be null, therefore we need the null check
     const isOnAppPort = port && port.toString() === options.port.toString();
 
-    if (hostname === "localhost" && isOnAppPort && !uniqueUrls.has(newUrl) && !streamClosed) {
+    if (
+      hostname === "localhost" &&
+      isOnAppPort &&
+      !uniqueUrls.has(newUrl) &&
+      !streamClosed
+    ) {
       uniqueUrls.add(newUrl);
       enqued++;
       queue.write(newUrl);
@@ -194,13 +201,7 @@ const crawl = async opt => {
     }
   };
 
-  const browser = await puppeteer.launch({
-    headless: options.headless,
-    args: options.puppeteerArgs,
-    executablePath: options.puppeteerExecutablePath,
-    ignoreHTTPSErrors: options.puppeteerIgnoreHTTPSErrors,
-    handleSIGINT: false
-  });
+  const browser = await puppeteer.launch(options.puppeteer);
 
   /**
    * @param {string} pageUrl
@@ -238,7 +239,10 @@ const crawl = async opt => {
         await page.setUserAgent(options.userAgent);
         const tracker = createTracker(page);
         try {
-          await page.goto(pageUrl, { waitUntil: "networkidle0" });
+          await page.goto(pageUrl, {
+            timeout: options.puppeteer.timeout,
+            waitUntil: "networkidle0"
+          });
         } catch (e) {
           e.message = augmentTimeoutError(e.message, tracker);
           throw e;

tests/__snapshots__/defaultOptions.test.js.snap

@@ -10,7 +10,6 @@ Object {
   "externalServer": false,
   "fixInsertRule": true,
   "fixWebpackChunksIssue": "CRA1",
-  "headless": true,
   "http2PushManifest": false,
   "ignoreForPreload": Array [
     "service-worker.js",
@@ -33,11 +32,14 @@ Object {
   "preloadImages": false,
   "publicPath": "/",
   "puppeteer": Object {
+    "args": Array [],
     "cache": true,
+    "executablePath": undefined,
+    "handleSIGINT": false,
+    "headless": true,
+    "ignoreHTTPSErrors": false,
+    "timeout": 120000,
   },
-  "puppeteerArgs": Array [],
-  "puppeteerExecutablePath": undefined,
-  "puppeteerIgnoreHTTPSErrors": false,
   "removeBlobs": true,
   "removeScriptTags": false,
   "removeStyleTags": false,

tests/run.test.js

@@ -10,7 +10,7 @@ const snapRun = (fs, options) =>
   run(
     {
       // for Travis CI
-      puppeteerArgs: ["--no-sandbox", "--disable-setuid-sandbox"],
+      puppeteer: { args: ["--no-sandbox", "--disable-setuid-sandbox"] },
       // sometimes web server from previous test have not enough time to shut down
       // as a result you get `Error: listen EADDRINUSE :::45678`
       // to prevent this we use random port