export docs and related docs (#7)

i18n/en.json

@@ -1,6 +1,7 @@
 {
   "export": "Export",
-  "exporting": "Exporting",
+  "exported": "Exported {{ count }} {{ type }}",
+  "exporting": "Exporting {{ type }}...",
   "exportModalDescription": "You've selected {{ count }} {{ type }} for export",
   "exportModalRelatedDocumentDescription": "Include the following document types where applicable",
   "exportModalSettingsLabel": "Export Settings",

index.js

@@ -1,5 +1,9 @@
 const fs = require('fs');
 const path = require('path');
+const methods = require('./lib/methods');
+const apiRoutes = require('./lib/apiRoutes');
+const zip = require('./lib/formats/zip');
+const gzip = require('./lib/formats/gzip');
 
 module.exports = {
   bundle: {
@@ -16,56 +20,25 @@ module.exports = {
   },
 
   init(self) {
-    self.apos.asset.iconMap['apos-import-export-download-icon'] = 'Download';
-    self.apos.asset.iconMap['apos-import-export-upload-icon'] = 'Upload';
-  },
-
-  apiRoutes(self) {
-    return {
-      get: {
-        async related(req) {
-          if (!req.user) {
-            throw self.apos.error('forbidden');
-          }
-
-          const type = self.apos.launder.string(req.query.type);
-          if (!type) {
-            throw self.apos.error('invalid');
-          }
-
-          const { schema = [] } = self.apos.modules[type];
-
-          // Limit recursions in order to avoid the "Maximum call stack size exceeded" error
-          // if widgets or pieces are related to themselves.
-          const maxRecursions = 10;
-          let recursions = 0;
+    if (self.options.export !== false) {
+      self.apos.asset.iconMap['apos-import-export-download-icon'] = 'Download';
+    }
+    if (self.options.import !== false) {
+      self.apos.asset.iconMap['apos-import-export-upload-icon'] = 'Upload';
+    }
 
-          const relatedTypes = schema
-            .flatMap(searchRelationships)
-            .filter(Boolean);
+    self.exportFormats = {
+      zip,
+      gzip,
+      ...(self.options.exportFormats || {})
+    };
 
-          return [ ...new Set(relatedTypes) ];
+    self.enableBrowserData();
+  },
 
-          function searchRelationships(obj) {
-            const shouldRecurse = recursions <= maxRecursions;
+  methods,
 
-            if (obj.type === 'relationship') {
-              return obj.withType;
-            } else if (obj.type === 'array' || obj.type === 'object') {
-              recursions++;
-              return shouldRecurse && obj.schema.flatMap(searchRelationships);
-            } else if (obj.type === 'area') {
-              recursions++;
-              return Object.keys(obj.options.widgets).flatMap(widget => {
-                const { schema = [] } = self.apos.modules[`${widget}-widget`];
-                return shouldRecurse && schema.flatMap(searchRelationships);
-              });
-            }
-          }
-        }
-      }
-    };
-  }
+  apiRoutes
 };
 
 function getBundleModuleNames() {

lib/apiRoutes.js

@@ -0,0 +1,52 @@
+module.exports = self => {
+  if (self.options.export === false) {
+    return {};
+  }
+
+  return {
+    get: {
+      async related(req) {
+        if (!req.user) {
+          throw self.apos.error('forbidden');
+        }
+
+        const type = self.apos.launder.string(req.query.type);
+        if (!type) {
+          throw self.apos.error('invalid');
+        }
+
+        const { schema = [] } = self.apos.modules[type];
+
+        // Limit recursions in order to avoid the "Maximum call stack size exceeded" error
+        // if widgets or pieces are related to themselves.
+        const maxRecursions = 10;
+        let recursions = 0;
+
+        const relatedTypes = schema
+          .flatMap(searchRelationships)
+          .filter(Boolean);
+
+        return [ ...new Set(relatedTypes) ];
+
+        // TODO: do not include types that have the `relatedDocument = false`
+        // option in their module configuration
+        function searchRelationships(obj) {
+          const shouldRecurse = recursions <= maxRecursions;
+
+          if (obj.type === 'relationship') {
+            return obj.withType;
+          } else if (obj.type === 'array' || obj.type === 'object') {
+            recursions++;
+            return shouldRecurse && obj.schema.flatMap(searchRelationships);
+          } else if (obj.type === 'area') {
+            recursions++;
+            return Object.keys(obj.options.widgets).flatMap(widget => {
+              const { schema = [] } = self.apos.modules[`${widget}-widget`];
+              return shouldRecurse && schema.flatMap(searchRelationships);
+            });
+          }
+        }
+      }
+    }
+  };
+};

lib/formats/archiver.js

@@ -0,0 +1,36 @@
+const { createWriteStream } = require('node:fs');
+
+module.exports = {
+  compress
+};
+
+function compress(filepath, data, archive) {
+  return new Promise((resolve, reject) => {
+    const output = createWriteStream(filepath);
+
+    archive.on('warning', function(err) {
+      if (err.code === 'ENOENT') {
+        console.warn(err);
+      } else {
+        reject(err);
+      }
+    });
+    archive.on('error', reject);
+    archive.on('finish', resolve);
+
+    archive.pipe(output);
+
+    for (const filename in data) {
+      const content = data[filename];
+
+      if (content.endsWith('/')) {
+        archive.directory(content, filename);
+        continue;
+      }
+
+      archive.append(content, { name: filename });
+    }
+
+    archive.finalize();
+  });
+}

lib/formats/gzip.js

@@ -0,0 +1,12 @@
+const archiver = require('archiver');
+const { compress } = require('./archiver');
+
+module.exports = {
+  label: 'gzip',
+  extension: 'tar.gz',
+  output(filepath, data) {
+    const archive = archiver('tar', { gzip: true });
+
+    return compress(filepath, data, archive);
+  }
+};

lib/formats/zip.js

@@ -0,0 +1,11 @@
+const archiver = require('archiver');
+const { compress } = require('./archiver');
+
+module.exports = {
+  label: 'Zip',
+  output(filepath, data) {
+    const archive = archiver('zip');
+
+    return compress(filepath, data, archive);
+  }
+};

lib/methods/archive.js

@@ -0,0 +1,98 @@
+const fs = require('node:fs/promises');
+const path = require('path');
+const util = require('util');
+
+module.exports = self => {
+  return {
+    async createArchive(req, reporting, docs, attachments, options) {
+      const {
+        extension,
+        format,
+        expiration
+      } = options;
+
+      const specificExtension = self.exportFormats[extension].extension;
+
+      const filename = `${self.apos.util.generateId()}-export.${specificExtension || extension}`;
+      const filepath = path.join(self.apos.attachment.uploadfs.getTempPath(), filename);
+
+      const docsData = JSON.stringify(docs, undefined, 2);
+      const attachmentsData = JSON.stringify(attachments, undefined, 2);
+
+      const data = {
+        'aposDocs.json': docsData,
+        'aposAttachments.json': attachmentsData
+        // attachments: 'attachments/' // TODO: add attachment into an "/attachments" folder
+      };
+
+      await format.output(filepath, data);
+
+      // Must copy it to uploadfs, the server that created it
+      // and the server that delivers it might be different
+      const downloadPath = path.join('/exports', filename);
+
+      const copyIn = util.promisify(self.apos.attachment.uploadfs.copyIn);
+
+      try {
+        await copyIn(filepath, downloadPath);
+      } catch (error) {
+        await self.removeArchive(filepath);
+        throw error;
+      }
+
+      const downloadUrl = `${self.apos.attachment.uploadfs.getUrl()}${downloadPath}`;
+
+      if (reporting) {
+        reporting.setResults({
+          url: downloadUrl
+        });
+      } else {
+        // No reporting means no batch operation:
+        // We need to handle the notification manually
+        // when exporting a single document:
+        await self.apos.notification.trigger(req, 'aposImportExport:exported', {
+          interpolate: {
+            count: req.body._ids.length,
+            type: req.body.type
+          },
+          dismiss: true,
+          icon: 'database-export-icon',
+          type: 'success',
+          event: {
+            name: 'export-download',
+            data: {
+              url: downloadUrl
+            }
+          }
+        });
+      }
+
+      await self.removeArchive(filepath);
+      self.removeArchiveFromUploadFs(downloadPath, expiration);
+
+      return {
+        url: downloadUrl
+      };
+    },
+
+    async removeArchive(filepath) {
+      try {
+        await fs.unlink(filepath);
+      } catch (error) {
+        self.apos.util.error(error);
+      }
+    },
+
+    // Report is available for one hour by default
+    removeArchiveFromUploadFs(downloadPath, expiration) {
+      // TODO: use cron instead?
+      setTimeout(() => {
+        self.apos.attachment.uploadfs.remove(downloadPath, error => {
+          if (error) {
+            self.apos.util.error(error);
+          }
+        });
+      }, expiration || 1000 * 60 * 60);
+    }
+  };
+};