fix(http-crawler): avoid crashing when gotOptions.cache is on (#2686)

When the response is from cache, http-crawler will throw "TypeError:
Cannot convert undefined or null to object"

This PR fixes this issue. Related usage is added as test case as well.

packages/http-crawler/src/internals/http-crawler.ts

@@ -977,9 +977,9 @@ function addResponsePropertiesToStream(stream: GotRequest) {
 
     response.on('end', () => {
         // @ts-expect-error
-        Object.assign(stream.rawTrailers, response.rawTrailers);
+        if (stream.rawTrailers) Object.assign(stream.rawTrailers, response.rawTrailers);
         // @ts-expect-error
-        Object.assign(stream.trailers, response.trailers);
+        if (stream.trailers) Object.assign(stream.trailers, response.trailers);
 
         // @ts-expect-error
         stream.complete = response.complete;

test/core/crawlers/http_crawler.test.ts

@@ -374,3 +374,29 @@ test('should retry on 403 even with disallowed content-type', async () => {
     expect(succeeded).toHaveLength(1);
     expect(succeeded[0].retryCount).toBe(1);
 });
+
+test('should work with cacheable-request', async () => {
+    const isFromCache: Record<string, boolean> = {};
+    const cache = new Map();
+    const crawler = new HttpCrawler({
+        maxConcurrency: 1,
+        preNavigationHooks: [
+            async (_, gotOptions) => {
+                gotOptions.cache = cache;
+                gotOptions.headers = {
+                    ...gotOptions.headers,
+                    // to force cache
+                    'cache-control': 'max-stale',
+                };
+            },
+        ],
+        requestHandler: async ({ request, response }) => {
+            isFromCache[request.uniqueKey] = response.isFromCache;
+        },
+    });
+    await crawler.run([
+        { url, uniqueKey: 'first' },
+        { url, uniqueKey: 'second' },
+    ]);
+    expect(isFromCache).toEqual({ first: false, second: true });
+});