From 552a832c445a4bd6a669af336638db3d70a5dcae Mon Sep 17 00:00:00 2001 From: ETLaurent Date: Thu, 24 Aug 2023 11:40:51 +0200 Subject: [PATCH] export docs and related docs (#7) --- i18n/en.json | 3 +- index.js | 65 ++----- lib/apiRoutes.js | 52 ++++++ lib/formats/archiver.js | 36 ++++ lib/formats/gzip.js | 12 ++ lib/formats/zip.js | 11 ++ lib/methods/archive.js | 98 ++++++++++ lib/methods/export.js | 170 ++++++++++++++++++ lib/methods/index.js | 21 +++ .../import-export-doc-type/index.js | 5 +- .../import-export-page/index.js | 17 ++ .../import-export-piece-type/index.js | 40 ++++- package.json | 3 + ui/apos/apps/index.js | 12 ++ ui/apos/components/AposExportModal.vue | 63 +++++-- 15 files changed, 545 insertions(+), 63 deletions(-) create mode 100644 lib/apiRoutes.js create mode 100644 lib/formats/archiver.js create mode 100644 lib/formats/gzip.js create mode 100644 lib/formats/zip.js create mode 100644 lib/methods/archive.js create mode 100644 lib/methods/export.js create mode 100644 lib/methods/index.js create mode 100644 ui/apos/apps/index.js diff --git a/i18n/en.json b/i18n/en.json index 4caee922..cc5e14b9 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -1,6 +1,7 @@ { "export": "Export", - "exporting": "Exporting", + "exported": "Exported {{ count }} {{ type }}", + "exporting": "Exporting {{ type }}...", "exportModalDescription": "You've selected {{ count }} {{ type }} for export", "exportModalRelatedDocumentDescription": "Include the following document types where applicable", "exportModalSettingsLabel": "Export Settings", diff --git a/index.js b/index.js index d1eb559c..1c180040 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,9 @@ const fs = require('fs'); const path = require('path'); +const methods = require('./lib/methods'); +const apiRoutes = require('./lib/apiRoutes'); +const zip = require('./lib/formats/zip'); +const gzip = require('./lib/formats/gzip'); module.exports = { bundle: { @@ -16,56 +20,25 @@ module.exports = { }, init(self) { - self.apos.asset.iconMap['apos-import-export-download-icon'] = 'Download'; - self.apos.asset.iconMap['apos-import-export-upload-icon'] = 'Upload'; - }, - - apiRoutes(self) { - return { - get: { - async related(req) { - if (!req.user) { - throw self.apos.error('forbidden'); - } - - const type = self.apos.launder.string(req.query.type); - if (!type) { - throw self.apos.error('invalid'); - } - - const { schema = [] } = self.apos.modules[type]; - - // Limit recursions in order to avoid the "Maximum call stack size exceeded" error - // if widgets or pieces are related to themselves. - const maxRecursions = 10; - let recursions = 0; + if (self.options.export !== false) { + self.apos.asset.iconMap['apos-import-export-download-icon'] = 'Download'; + } + if (self.options.import !== false) { + self.apos.asset.iconMap['apos-import-export-upload-icon'] = 'Upload'; + } - const relatedTypes = schema - .flatMap(searchRelationships) - .filter(Boolean); + self.exportFormats = { + zip, + gzip, + ...(self.options.exportFormats || {}) + }; - return [ ...new Set(relatedTypes) ]; + self.enableBrowserData(); + }, - function searchRelationships(obj) { - const shouldRecurse = recursions <= maxRecursions; + methods, - if (obj.type === 'relationship') { - return obj.withType; - } else if (obj.type === 'array' || obj.type === 'object') { - recursions++; - return shouldRecurse && obj.schema.flatMap(searchRelationships); - } else if (obj.type === 'area') { - recursions++; - return Object.keys(obj.options.widgets).flatMap(widget => { - const { schema = [] } = self.apos.modules[`${widget}-widget`]; - return shouldRecurse && schema.flatMap(searchRelationships); - }); - } - } - } - } - }; - } + apiRoutes }; function getBundleModuleNames() { diff --git a/lib/apiRoutes.js b/lib/apiRoutes.js new file mode 100644 index 00000000..ad87a24f --- /dev/null +++ b/lib/apiRoutes.js @@ -0,0 +1,52 @@ +module.exports = self => { + if (self.options.export === false) { + return {}; + } + + return { + get: { + async related(req) { + if (!req.user) { + throw self.apos.error('forbidden'); + } + + const type = self.apos.launder.string(req.query.type); + if (!type) { + throw self.apos.error('invalid'); + } + + const { schema = [] } = self.apos.modules[type]; + + // Limit recursions in order to avoid the "Maximum call stack size exceeded" error + // if widgets or pieces are related to themselves. + const maxRecursions = 10; + let recursions = 0; + + const relatedTypes = schema + .flatMap(searchRelationships) + .filter(Boolean); + + return [ ...new Set(relatedTypes) ]; + + // TODO: do not include types that have the `relatedDocument = false` + // option in their module configuration + function searchRelationships(obj) { + const shouldRecurse = recursions <= maxRecursions; + + if (obj.type === 'relationship') { + return obj.withType; + } else if (obj.type === 'array' || obj.type === 'object') { + recursions++; + return shouldRecurse && obj.schema.flatMap(searchRelationships); + } else if (obj.type === 'area') { + recursions++; + return Object.keys(obj.options.widgets).flatMap(widget => { + const { schema = [] } = self.apos.modules[`${widget}-widget`]; + return shouldRecurse && schema.flatMap(searchRelationships); + }); + } + } + } + } + }; +}; diff --git a/lib/formats/archiver.js b/lib/formats/archiver.js new file mode 100644 index 00000000..d9b4b830 --- /dev/null +++ b/lib/formats/archiver.js @@ -0,0 +1,36 @@ +const { createWriteStream } = require('node:fs'); + +module.exports = { + compress +}; + +function compress(filepath, data, archive) { + return new Promise((resolve, reject) => { + const output = createWriteStream(filepath); + + archive.on('warning', function(err) { + if (err.code === 'ENOENT') { + console.warn(err); + } else { + reject(err); + } + }); + archive.on('error', reject); + archive.on('finish', resolve); + + archive.pipe(output); + + for (const filename in data) { + const content = data[filename]; + + if (content.endsWith('/')) { + archive.directory(content, filename); + continue; + } + + archive.append(content, { name: filename }); + } + + archive.finalize(); + }); +} diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js new file mode 100644 index 00000000..962bbf5a --- /dev/null +++ b/lib/formats/gzip.js @@ -0,0 +1,12 @@ +const archiver = require('archiver'); +const { compress } = require('./archiver'); + +module.exports = { + label: 'gzip', + extension: 'tar.gz', + output(filepath, data) { + const archive = archiver('tar', { gzip: true }); + + return compress(filepath, data, archive); + } +}; diff --git a/lib/formats/zip.js b/lib/formats/zip.js new file mode 100644 index 00000000..0d56da2a --- /dev/null +++ b/lib/formats/zip.js @@ -0,0 +1,11 @@ +const archiver = require('archiver'); +const { compress } = require('./archiver'); + +module.exports = { + label: 'Zip', + output(filepath, data) { + const archive = archiver('zip'); + + return compress(filepath, data, archive); + } +}; diff --git a/lib/methods/archive.js b/lib/methods/archive.js new file mode 100644 index 00000000..62cbb1a8 --- /dev/null +++ b/lib/methods/archive.js @@ -0,0 +1,98 @@ +const fs = require('node:fs/promises'); +const path = require('path'); +const util = require('util'); + +module.exports = self => { + return { + async createArchive(req, reporting, docs, attachments, options) { + const { + extension, + format, + expiration + } = options; + + const specificExtension = self.exportFormats[extension].extension; + + const filename = `${self.apos.util.generateId()}-export.${specificExtension || extension}`; + const filepath = path.join(self.apos.attachment.uploadfs.getTempPath(), filename); + + const docsData = JSON.stringify(docs, undefined, 2); + const attachmentsData = JSON.stringify(attachments, undefined, 2); + + const data = { + 'aposDocs.json': docsData, + 'aposAttachments.json': attachmentsData + // attachments: 'attachments/' // TODO: add attachment into an "/attachments" folder + }; + + await format.output(filepath, data); + + // Must copy it to uploadfs, the server that created it + // and the server that delivers it might be different + const downloadPath = path.join('/exports', filename); + + const copyIn = util.promisify(self.apos.attachment.uploadfs.copyIn); + + try { + await copyIn(filepath, downloadPath); + } catch (error) { + await self.removeArchive(filepath); + throw error; + } + + const downloadUrl = `${self.apos.attachment.uploadfs.getUrl()}${downloadPath}`; + + if (reporting) { + reporting.setResults({ + url: downloadUrl + }); + } else { + // No reporting means no batch operation: + // We need to handle the notification manually + // when exporting a single document: + await self.apos.notification.trigger(req, 'aposImportExport:exported', { + interpolate: { + count: req.body._ids.length, + type: req.body.type + }, + dismiss: true, + icon: 'database-export-icon', + type: 'success', + event: { + name: 'export-download', + data: { + url: downloadUrl + } + } + }); + } + + await self.removeArchive(filepath); + self.removeArchiveFromUploadFs(downloadPath, expiration); + + return { + url: downloadUrl + }; + }, + + async removeArchive(filepath) { + try { + await fs.unlink(filepath); + } catch (error) { + self.apos.util.error(error); + } + }, + + // Report is available for one hour by default + removeArchiveFromUploadFs(downloadPath, expiration) { + // TODO: use cron instead? + setTimeout(() => { + self.apos.attachment.uploadfs.remove(downloadPath, error => { + if (error) { + self.apos.util.error(error); + } + }); + }, expiration || 1000 * 60 * 60); + } + }; +}; diff --git a/lib/methods/export.js b/lib/methods/export.js new file mode 100644 index 00000000..e50c4c14 --- /dev/null +++ b/lib/methods/export.js @@ -0,0 +1,170 @@ +// TODO: remove: +const attachmentsMock = [ { foo: 'bar' } ]; + +module.exports = self => { + return { + async export(req, manager, reporting = null) { + if (!req.user) { + throw self.apos.error('forbidden'); + } + + if (reporting) { + reporting.setTotal(req.body._ids.length); + } + + // TODO: add batchSize? + const ids = self.apos.launder.ids(req.body._ids); + const relatedTypes = self.apos.launder.strings(req.body.relatedTypes); + const extension = self.apos.launder.string(req.body.extension, 'zip'); + const expiration = typeof self.options.export === 'object' && + self.apos.launder.integer(self.options.export.expiration); + + const format = self.exportFormats[extension]; + + if (!format) { + throw self.apos.error('invalid'); + } + + const allIds = self.getAllModesIds(ids); + + if (!relatedTypes.length) { + const docs = await self.fetchActualDocs(req, allIds, reporting); + + return self.createArchive(req, reporting, docs, attachmentsMock, { + extension, + expiration, + format + }); + } + + const draftDocs = await self.getPopulatedDocs(req, manager, allIds, 'draft'); + const publishedDocs = await self.getPopulatedDocs(req, manager, allIds, 'published'); + + // Get related docs id from BOTH the draft and published docs, + // since they might have different related documents. + const relatedIds = draftDocs + .concat(publishedDocs) + .flatMap(doc => self.getRelatedDocsIds(manager, doc, relatedTypes)); + + const allRelatedIds = self.getAllModesIds(relatedIds); + + const docs = await self.fetchActualDocs(req, [ ...allIds, ...allRelatedIds ], reporting); + + return self.createArchive(req, reporting, docs, attachmentsMock, { + extension, + expiration, + format + }); + }, + + // Add the published version ID next to each draft ID, + // so we always get both the draft and the published ID. + // If somehow published IDs are sent from the frontend, + // that will not be an issue thanks to this method. + getAllModesIds(ids) { + return ids.flatMap(id => [ + id.replace(':published', ':draft'), + id.replace(':draft', ':published') + ]); + }, + + // Fetch the documents exactly as they are in the database, + // without altering the fields or populating them, as the managers would. + // It is ok if docs corresponding to published IDs do not exist in the database, + // as they simply will not be fetched. + async fetchActualDocs(req, docsIds, reporting) { + const docsIdsUniq = [ ...new Set(docsIds) ]; + + const docs = await self.apos.doc.db + .find({ + _id: { + $in: docsIdsUniq + } + }) + .toArray(); + + // FIXME: seems like reporting is not working very well, + // when calling failure on purpose, we have the progress bar + // at 200%... 🤷‍♂️ + if (reporting) { + const { _ids } = req.body; + const docsId = docs.map(doc => doc._id); + + // Verify that each id sent in the body has its corresponding doc fetched + _ids.forEach(id => { + const fn = docsId.includes(id) + ? 'success' + : 'failure'; + + reporting[fn](); + }); + } + + return docs; + }, + + // Get docs via their manager in order to populate them + // so that we can retrieve their relationships IDs later. + getPopulatedDocs(req, manager, docsIds, mode) { + return manager + .find(req.clone({ mode }), { + _id: { + $in: docsIds + } + }) + .toArray(); + }, + + getRelatedDocsIds(manager, doc, relatedTypes) { + // Use `doc.type` for pages to get the actual schema of the corresponding page type. + const schema = manager.schema || self.apos.modules[doc.type].schema; + + return self + .getRelatedBySchema(doc, schema) + .filter(relatedDoc => relatedTypes.includes(relatedDoc.type)) + .map(relatedDoc => relatedDoc._id); + }, + + // TODO: factorize with the one from AposI18nLocalize.vue + // TODO: limit recursion to 10 as we do when retrieving related types? + getRelatedBySchema(object, schema) { + let related = []; + for (const field of schema) { + if (field.type === 'array') { + for (const value of (object[field.name] || [])) { + related = [ + ...related, + ...self.getRelatedBySchema(value, field.schema) + ]; + } + } else if (field.type === 'object') { + if (object[field.name]) { + related = [ + ...related, + ...self.getRelatedBySchema(object[field.name], field.schema) + ]; + } + } else if (field.type === 'area') { + for (const widget of (object[field.name]?.items || [])) { + related = [ + ...related, + ...self.getRelatedBySchema(widget, self.apos.modules[`${widget?.type}-widget`]?.schema || []) + ]; + } + } else if (field.type === 'relationship') { + related = [ + ...related, + ...(object[field.name] || []) + ]; + // Stop here, don't recurse through relationships or we're soon + // related to the entire site + } + } + // Filter out doc types that opt out completely (pages should + // never be considered "related" to other pages simply because + // of navigation links, the feature is meant for pieces that feel more like + // part of the document being localized) + return related.filter(doc => self.apos.modules[doc.type].relatedDocument !== false); + } + }; +}; diff --git a/lib/methods/index.js b/lib/methods/index.js new file mode 100644 index 00000000..748cb964 --- /dev/null +++ b/lib/methods/index.js @@ -0,0 +1,21 @@ +const exportMethods = require('./export'); +const archiveMethods = require('./archive'); + +module.exports = self => { + return { + // No need to override, the parent method returns `{}`. + getBrowserData() { + return { + extensions: Object + .entries(self.exportFormats) + .map(([ key, value ]) => ({ + label: value.label, + value: key + })) + }; + }, + + ...exportMethods(self), + ...archiveMethods(self) + }; +}; diff --git a/modules/@apostrophecms/import-export-doc-type/index.js b/modules/@apostrophecms/import-export-doc-type/index.js index bd465a1f..b3b90acd 100644 --- a/modules/@apostrophecms/import-export-doc-type/index.js +++ b/modules/@apostrophecms/import-export-doc-type/index.js @@ -8,7 +8,10 @@ module.exports = { action: 'export', context: 'update', label: 'aposImportExport:export', - modal: 'AposExportModal' + modal: 'AposExportModal', + props: { + action: 'export-one' + } }; if (self.options.export === false) { diff --git a/modules/@apostrophecms/import-export-page/index.js b/modules/@apostrophecms/import-export-page/index.js index bd24c21e..eb871ac5 100644 --- a/modules/@apostrophecms/import-export-page/index.js +++ b/modules/@apostrophecms/import-export-page/index.js @@ -22,5 +22,22 @@ module.exports = { } } }; + }, + + apiRoutes(self) { + if (self.options.export === false) { + return {}; + } + + return { + post: { + exportOne(req) { + // Add the page label to req.body for notifications. + req.body.type = req.t('apostrophe:page'); + + return self.apos.modules['@apostrophecms/import-export'].export(req, self); + } + } + }; } }; diff --git a/modules/@apostrophecms/import-export-piece-type/index.js b/modules/@apostrophecms/import-export-piece-type/index.js index 9bac2c81..9faf483e 100644 --- a/modules/@apostrophecms/import-export-piece-type/index.js +++ b/modules/@apostrophecms/import-export-piece-type/index.js @@ -36,7 +36,10 @@ module.exports = { export: { label: 'aposImportExport:export', messages: { - progress: 'aposImportExport:exporting' + progress: 'aposImportExport:exporting', + completed: 'aposImportExport:exported', + icon: 'database-export-icon', + resultsEventName: 'export-download' }, modal: 'AposExportModal' } @@ -48,5 +51,40 @@ module.exports = { } } }; + }, + + apiRoutes(self) { + if (self.options.export === false) { + return {}; + } + + return { + post: { + // NOTE: this route is used in batch operations, and its method should be POST + // in order to make the job work with the progress notification. + // The other `exportOne` routes that are used by context operations on each doc + // are also POST for consistency. + export(req) { + // Add the piece type label to req.body for notifications. + // Should be done before calling the job's `run` method. + req.body.type = req.body._ids.length === 1 + ? req.t(self.options.label) + : req.t(self.options.pluralLabel); + + // FIXME: the progress notification is not always dismissed. + // Probably a fix that needs to be done in job core module. + return self.apos.modules['@apostrophecms/job'].run( + req, + (req, reporting) => self.apos.modules['@apostrophecms/import-export'].export(req, self, reporting) + ); + }, + exportOne(req) { + // Add the piece type label to req.body for notifications. + req.body.type = req.t(self.options.label); + + return self.apos.modules['@apostrophecms/import-export'].export(req, self); + } + } + }; } }; diff --git a/package.json b/package.json index b5d16fd2..3e50adc5 100644 --- a/package.json +++ b/package.json @@ -29,5 +29,8 @@ "stylelint": "^15.9.0", "stylelint-config-apostrophe": "^3.0.0", "vue-eslint-parser": "^9.3.1" + }, + "dependencies": { + "archiver": "^6.0.0" } } diff --git a/ui/apos/apps/index.js b/ui/apos/apps/index.js new file mode 100644 index 00000000..6160f374 --- /dev/null +++ b/ui/apos/apps/index.js @@ -0,0 +1,12 @@ +export default () => { + window.apos.util.onReady(openExportUrl); + + function openExportUrl() { + window.apos.bus && window.apos.bus.$on('export-download', event => { + if (!event.url) { + return; + } + window.open(event.url, '_blank'); + }); + } +}; diff --git a/ui/apos/components/AposExportModal.vue b/ui/apos/components/AposExportModal.vue index 257d38ad..232844b5 100644 --- a/ui/apos/components/AposExportModal.vue +++ b/ui/apos/components/AposExportModal.vue @@ -28,13 +28,10 @@
{{ $t('aposImportExport:exportModalDocumentFormat') }}
-
@@ -125,9 +122,17 @@ export default { type: String, default: '' }, - count: { - type: Number, - default: 1 + checked: { + type: Array, + default: () => [] + }, + action: { + type: String, + required: true + }, + messages: { + type: Object, + default: () => ({}) } }, @@ -145,14 +150,15 @@ export default { relatedChildrenDisabled: true, relatedTypes: null, checkedRelatedTypes: [], - type: this.moduleName + type: this.moduleName, + extension: 'zip' }; }, computed: { moduleLabel() { const moduleOptions = window.apos.modules[this.moduleName]; - const label = this.count > 1 ? moduleOptions.pluralLabel : moduleOptions.label; + const label = this.checked.length > 1 ? moduleOptions.pluralLabel : moduleOptions.label; return this.$t(label).toLowerCase(); }, @@ -163,6 +169,14 @@ export default { set(val) { this.$emit('change', val); } + }, + + count() { + return this.checked.length || 1; + }, + + extensions() { + return window.apos.modules['@apostrophecms/import-export'].extensions; } }, @@ -178,9 +192,27 @@ export default { ready() { this.$refs.exportDocs.$el.querySelector('button').focus(); }, - exportDocs() { + async exportDocs() { + const docsId = this.checked.length + ? this.checked + : [ this.$attrs.doc?._id ]; + + const relatedTypes = this.relatedDocumentsDisabled + ? [] + : this.checkedRelatedTypes; + + const { action } = window.apos.modules[this.moduleName]; + const result = await window.apos.http.post(`${action}/${this.action}`, { + busy: true, + body: { + _ids: docsId, + relatedTypes, + messages: this.messages, + extension: this.extension + } + }); + this.modal.showModal = false; - const result = true; this.$emit('modal-result', result); }, async cancel() { @@ -212,6 +244,9 @@ export default { getRelatedTypeLabel(moduleName) { const moduleOptions = window.apos.modules[moduleName]; return this.$t(moduleOptions.label); + }, + onExtensionChange(value) { + this.extension = this.extensions.find(extension => extension.value === value).value; } } };