Fix problems with guide script

scripts/download_data.mjs

@@ -1,95 +1,63 @@
-import { fileURLToPath } from "node:url";
-import { mkdir, writeFile } from "node:fs/promises";
-import { gunzipSync } from "node:zlib";
-import * as path from "node:path";
-
-const dataFolder = fileURLToPath(import.meta.resolve("../data"));
-await mkdir(dataFolder, { recursive: true });
-
-const baseUrl = "https://guide-assets.appliedenergistics.org/";
-
-function describeVersion(versionFromIndex) {
-  if (versionFromIndex.development) {
-    return versionFromIndex.gameVersion + " [DEV]";
-  } else {
-    return versionFromIndex.gameVersion;
-  }
-}
-
-async function fetchJson(url) {
-  const response = await fetch(url);
-  if (!response.ok) {
-    throw new Error(`Failed request for ${url}: ${response}`);
-  }
-  return await response.json();
-}
-
-const versionIndex = await fetchJson(`${baseUrl}index.json`);
-
-// For dev purposes:
-// versionIndex.versions = versionIndex.versions.slice(0, 1);
-
-async function downloadVersion(versionInfo) {
-  const { url, development } = versionInfo;
-
-  console.log("Downloading %s", url);
-  const versionDetails = await fetchJson(url);
-  const { format, generated, modVersion, gameVersion, guideDataPath } =
-    versionDetails;
-
-  const versionSlug = development ? "development" : gameVersion;
-
-  const guideDataUrl = new URL(guideDataPath, url);
-  console.info("Downloading %s", guideDataUrl);
-  const guideDataResponse = await fetch(guideDataUrl);
-  if (!guideDataResponse.ok) {
-    throw guideDataResponse;
-  }
-
-  // Apply GZIP decompression
-  const unzippedGuideData = gunzipSync(await guideDataResponse.arrayBuffer());
-  const dataFilename = "data_" + versionSlug + ".json";
-  await writeFile(path.join(dataFolder, dataFilename), unzippedGuideData);
-
-  // Extract all pages:
-  const guideData = JSON.parse(unzippedGuideData.toString());
-
-  // URL which assets are relative to
-  const baseUrl = new URL(".", guideDataUrl);
-
-  // Return entry for version index, see Typescript Type GuideVersion
-  return {
-    baseUrl,
-    format,
-    gameVersion,
-    modVersion,
-    generated,
-    development,
-    slug: versionSlug,
-    dataFilename,
-    defaultNamespace: guideData.defaultNamespace,
-  };
-}
-
-const results = await Promise.allSettled(
-  versionIndex.versions.map(downloadVersion),
-);
-const downloadedVersions = [];
-
-for (let i = 0; i < results.length; i++) {
-  const result = results[i];
-  const version = describeVersion(versionIndex.versions[i]);
-  if (result.status === "fulfilled") {
-    console.info("[SUCCESS] Minecraft %s", version);
-    downloadedVersions.push(result.value);
-  } else {
-    console.group("[ERROR] Minecraft %s", version);
-    console.log("%o", result.reason);
-    console.groupEnd();
-  }
-}
-
-await writeFile(
-  path.join(dataFolder, "index.json"),
-  JSON.stringify(downloadedVersions, null, 2),
-);
+export default {
+  async scheduled(event, env, ctx) {
+    const bucket = env.GUIDE_ASSETS;
+    const listOptions = {
+      prefix: "",
+      delimiter: "/",
+    };
+
+    const versions = [];
+    async function processPrefixes(delimitedPrefixes) {
+      for (const prefix of delimitedPrefixes) {
+        const devVersion = prefix === "development/";
+
+        if (!prefix.match(/^minecraft-([^/]+)\/$/) && !devVersion) {
+          continue;
+        }
+        const indexFile = prefix + "index.json";
+
+        const indexResponse = await bucket.get(indexFile);
+        if (!indexResponse) {
+          console.warn("Index-File %s does not exist.", indexFile);
+          continue;
+        }
+
+        try {
+          const versionIndexContent = await indexResponse.json();
+          console.info("Content of %s: %o", indexFile, versionIndexContent);
+          versions.push({
+            format: versionIndexContent.format,
+            generated: versionIndexContent.generated,
+            gameVersion: versionIndexContent.gameVersion,
+            modVersion: versionIndexContent.modVersion,
+            // This is where the actual data lives in V1 guides
+            url: "https://guide-assets.appliedenergistics.org/" + indexFile,
+            development: devVersion,
+          });
+        } catch (e) {
+          console.error("Failed to process index file %s", indexFile);
+        }
+      }
+    }
+
+    // List directories in root
+    let response = await bucket.list(listOptions);
+    await processPrefixes(response.delimitedPrefixes);
+    while (response.truncated) {
+      response = await bucket.list({ ...listOptions, cursor: response.cursor });
+      await processPrefixes(response.delimitedPrefixes);
+    }
+
+    const indexJsonContent = JSON.stringify({
+      versions,
+    });
+    console.info("Overall version info: %s", indexJsonContent);
+    bucket.put("index.json", indexJsonContent, {
+      httpMetadata: {
+        contentType: "application/json",
+        // Allow browser to assume for 15m that the response is fresh
+        cacheControl: "public, max-age=900",
+      },
+    });
+  },
+};