Pro 6416 missing fields

lib/apiRoutes.js

@@ -11,40 +11,12 @@ module.exports = self => {
         }
 
         const type = self.apos.launder.string(req.query.type);
-        if (!type) {
+        const manager = self.apos.doc.getManager(type);
+        if (!manager) {
           throw self.apos.error('invalid');
         }
 
-        const { schema = [] } = self.apos.modules[type];
-
-        // Limit recursions in order to avoid the "Maximum call stack size exceeded" error
-        // if widgets or pieces are related to themselves.
-        const maxRecursions = 10;
-        let recursions = 0;
-
-        const relatedTypes = schema
-          .flatMap(searchRelationships)
-          .filter(Boolean);
-
-        return [ ...new Set(relatedTypes) ];
-
-        function searchRelationships(obj) {
-          const shouldRecurse = recursions <= maxRecursions;
-
-          if (obj.type === 'relationship') {
-            return self.canExport(req, obj.withType) && obj.withType;
-          } else if (obj.type === 'array' || obj.type === 'object') {
-            recursions++;
-            return shouldRecurse && obj.schema.flatMap(searchRelationships);
-          } else if (obj.type === 'area') {
-            recursions++;
-            const widgets = self.apos.area.getWidgets(obj.options);
-            return Object.keys(widgets).flatMap(widget => {
-              const { schema = [] } = self.apos.modules[`${widget}-widget`];
-              return shouldRecurse && schema.flatMap(searchRelationships);
-            });
-          }
-        }
+        return self.getRelatedTypes(req, manager.schema);
       }
     },
     post: {

lib/methods/export.js

@@ -32,47 +32,49 @@ module.exports = self => {
       }
 
       const hasRelatedTypes = !!relatedTypes.length;
-      const docs = await self.getDocs(req, ids, hasRelatedTypes, manager, reporting);
-      const cleanDocs = self.clean(docs);
+
+      const docs = (await self.getDocs(req, ids, hasRelatedTypes, manager, reporting))
+        .map((doc) => self.apos.util.clonePermanent(doc));
 
       if (!hasRelatedTypes) {
         return self.exportFile(
           req,
           reporting,
           format,
-          { docs: cleanDocs },
+          { docs },
           expiration
         );
       }
 
-      const relatedDocs = docs.flatMap(doc =>
-        self.getRelatedDocsFromSchema(req, {
+      const allDocs = [ ...docs ];
+      for (const doc of docs) {
+        await self.getRelatedDocsFromSchema(req, {
           doc,
           schema: self.apos.modules[doc.type].schema,
-          relatedTypes
-        })
-      );
-      const allCleanDocs = [ ...self.clean(relatedDocs), ...cleanDocs ];
+          relatedTypes,
+          storedData: allDocs
+        });
+      }
 
       if (!format.includeAttachments) {
         return self.exportFile(
           req,
           reporting,
           format,
-          { docs: allCleanDocs },
+          { docs: allDocs },
           expiration
         );
       }
 
-      const attachmentsIds = uniqBy([ ...docs, ...relatedDocs ], doc => doc._id)
-        .flatMap(doc =>
-          self.getRelatedDocsFromSchema(req, {
-            doc,
-            schema: self.apos.modules[doc.type].schema,
-            type: 'attachment'
-          })
-        )
-        .map(attachment => attachment._id);
+      const attachmentsIds = [];
+      for (const doc of allDocs) {
+        await self.getRelatedDocsFromSchema(req, {
+          doc,
+          schema: self.apos.modules[doc.type].schema,
+          type: 'attachment',
+          storedData: attachmentsIds
+        });
+      }
 
       const attachments = await self.getAttachments(attachmentsIds);
       const cleanAttachments = self.clean(attachments);
@@ -92,7 +94,7 @@ module.exports = self => {
         reporting,
         format,
         {
-          docs: allCleanDocs,
+          docs: allDocs,
           attachments: cleanAttachments,
           attachmentUrls
         },
@@ -198,66 +200,147 @@ module.exports = self => {
       return self.canImportOrExport(req, docType, 'export');
     },
 
-    getRelatedDocsFromSchema(
-      req,
-      {
-        doc,
-        schema,
-        type = 'relationship',
-        relatedTypes,
-        recursion = 0
-      }
-    ) {
-      return schema.flatMap(field => {
+    async getRelatedDocsFromSchema(req, {
+      doc,
+      schema,
+      relatedTypes,
+      storedData,
+      type = 'relationship',
+      recursion = 0,
+      mode = doc.aposMode || req.mode
+    }) {
+      recursion++;
+      for (const field of schema) {
         const fieldValue = doc[field.name];
         const shouldRecurse = recursion <= MAX_RECURSION;
 
-        if (!fieldValue) {
-          return [];
+        if (!field.withType && !fieldValue) {
+          continue;
         }
         if (field.withType && relatedTypes && !relatedTypes.includes(field.withType)) {
-          return [];
+          continue;
         }
         if (field.withType && !self.canExport(req, field.withType)) {
-          return [];
+          continue;
         }
 
         if (shouldRecurse && field.type === 'array') {
-          return fieldValue.flatMap((subField) => self.getRelatedDocsFromSchema(req, {
-            doc: subField,
-            schema: field.schema,
-            type,
-            relatedTypes,
-            recursion: recursion + 1
-          }));
+          for (const subField of fieldValue) {
+            await self.getRelatedDocsFromSchema(req, {
+              doc: subField,
+              schema: field.schema,
+              type,
+              relatedTypes,
+              recursion,
+              storedData,
+              mode
+            });
+          }
+          continue;
         }
 
         if (shouldRecurse && field.type === 'object') {
-          return self.getRelatedDocsFromSchema(req, {
+          await self.getRelatedDocsFromSchema(req, {
             doc: fieldValue,
             schema: field.schema,
             type,
             relatedTypes,
-            recursion: recursion + 1
+            recursion,
+            storedData,
+            mode
           });
+          continue;
         }
 
         if (shouldRecurse && field.type === 'area') {
-          return (fieldValue.items || []).flatMap((widget) => self.getRelatedDocsFromSchema(req, {
-            doc: widget,
-            schema: self.apos.modules[`${widget?.type}-widget`]?.schema || [],
-            type,
-            relatedTypes,
-            recursion: recursion + 1
-          }));
+          for (const widget of (fieldValue.items || [])) {
+            const schema = self.apos.modules[`${widget?.type}-widget`]?.schema || [];
+            await self.getRelatedDocsFromSchema(req, {
+              doc: widget,
+              schema,
+              type,
+              relatedTypes,
+              recursion,
+              storedData,
+              mode
+            });
+          }
+          continue;
         }
 
         if (field.type === type) {
-          return Array.isArray(fieldValue) ? fieldValue : [ fieldValue ];
+          await self.handleRelatedField(req, {
+            doc,
+            field,
+            fieldValue,
+            relatedTypes,
+            type,
+            storedData,
+            shouldRecurse,
+            recursion,
+            mode
+          });
+        }
+      }
+    },
+
+    async handleRelatedField(req, {
+      doc,
+      field,
+      fieldValue,
+      relatedTypes,
+      type,
+      storedData,
+      shouldRecurse,
+      recursion,
+      mode
+    }) {
+      if (type === 'attachment') {
+        if (fieldValue && !storedData.includes(fieldValue._id)) {
+          storedData.push(fieldValue._id);
         }
+        return;
+      }
 
-        return [];
-      });
+      const manager = self.apos.doc.getManager(field.withType);
+      const relatedIds = doc[field.idsStorage];
+      if (!relatedIds?.length) {
+        return;
+      }
+      const criteria = {
+        aposDocId: { $in: relatedIds },
+        aposLocale: doc.aposLocale || `${req.locale}:${mode}`
+      };
+
+      const clonedReq = mode ? req.clone({ mode }) : req;
+      const foundRelated = await manager
+        .findForEditing(clonedReq, criteria)
+        .permission('view')
+        .relationships(false)
+        .areas(false)
+        .toArray();
+
+      for (const related of foundRelated) {
+        const alreadyAdded = storedData.some(({ _id }) => _id === related._id);
+        if (alreadyAdded) {
+          continue;
+        }
+
+        storedData.push(self.apos.util.clonePermanent(related));
+        if (!shouldRecurse) {
+          continue;
+        }
+
+        const relatedManager = self.apos.doc.getManager(related.type);
+        await self.getRelatedDocsFromSchema(req, {
+          doc: related,
+          schema: relatedManager.schema,
+          type,
+          relatedTypes,
+          recursion,
+          storedData
+        });
+      }
     },
 
     async exportFile(req, reporting, format, data, expiration) {
@@ -379,6 +462,36 @@ module.exports = self => {
           }
         });
       }, ms);
+    },
+
+    getRelatedTypes(req, schema = []) {
+      return findSchemaRelatedTypes(schema);
+
+      function findSchemaRelatedTypes(schema, related = [], recursions = 0) {
+        recursions++;
+        if (recursions >= MAX_RECURSION) {
+          return related;
+        }
+        for (const field of schema) {
+          if (field.type === 'relationship') {
+            if (self.canExport(req, field.withType) && !related.includes(field.withType)) {
+              related.push(field.withType);
+              const relatedManager = self.apos.doc.getManager(field.withType);
+              findSchemaRelatedTypes(relatedManager.schema, related, recursions);
+            }
+          } else if ([ 'array', 'object' ].includes(field.type)) {
+            findSchemaRelatedTypes(field.schema, related, recursions);
+          } else if (field.type === 'area') {
+            const widgets = self.apos.area.getWidgets(field.options);
+            for (const widget of Object.keys(widgets)) {
+              const { schema = [] } = self.apos.modules[`${widget}-widget`];
+              findSchemaRelatedTypes(schema, related, recursions);
+            }
+          }
+        }
+
+        return related;
+      }
     }
   };
 };

lib/methods/import.js

@@ -321,7 +321,7 @@ module.exports = self => {
             continue;
           }
         }
-        if (aposDocId) {
+        if (aposDocId && !docIds.includes(aposDocId)) {
           docIds.push(aposDocId);
         }
       }
@@ -742,23 +742,25 @@ module.exports = self => {
       for (const doc of filterDocs) {
         try {
           if (attachmentsInfo.length) {
-            const attachmentsToOverride = self.getRelatedDocsFromSchema(req, {
+            const attachmentsIds = [];
+            await self.getRelatedDocsFromSchema(req, {
               doc,
               schema: self.apos.modules[doc.type].schema,
-              type: 'attachment'
+              type: 'attachment',
+              storedData: attachmentsIds
             });
 
-            for (const { _id } of attachmentsToOverride) {
-              if (importedAttachments.includes(_id)) {
+            for (const id of attachmentsIds) {
+              if (importedAttachments.includes(id)) {
                 continue;
               }
               const attachmentInfo = attachmentsInfo
-                .find(({ attachment }) => attachment._id === _id);
+                .find(({ attachment }) => attachment._id === id);
 
               try {
                 await self.insertOrUpdateAttachment(req, { attachmentInfo });
                 jobManager.success(job);
-                importedAttachments.push(_id);
+                importedAttachments.push(id);
               } catch (err) {
                 jobManager.failure(job);
               }