MDS-3074: Accept shape files for NoW file uploads (#1537)

Main
Users can now upload spatial files to a NoW
Other
Fixes an issue where TUSD wouldn't return proper "file type" metadata for obscure file types, such as spatial files
Tidies some code
Screenshots
image

Notes
It's not possible to nicely display all accepted file extensions for files that share the same MIME type (in our case, "application/octet-stream"--many of these files have no MIME type). pqina/filepond-plugin-file-validate-type#13

services/core-api/app/api/services/document_manager_service.py

@@ -90,13 +90,13 @@ def _parse_upload_folders(cls, mine, document_category):
 
     @classmethod
     def _parse_request_metadata(cls, request):
-        request_metadata = request.headers.get("Upload-Metadata")
+        request_metadata = request.headers.get('Upload-Metadata')
         metadata = {}
         if not request_metadata:
             return metadata
 
-        for key_value in request_metadata.split(","):
-            (key, value) = key_value.split(" ")
-            metadata[key] = base64.b64decode(value).decode("utf-8")
+        for key_value in request_metadata.split(','):
+            (key, value) = key_value.split(' ')
+            metadata[key] = base64.b64decode(value).decode('utf-8')
 
         return metadata

services/core-web/package.json

@@ -44,7 +44,7 @@
     "react": "^16.13.1",
     "react-async-script-loader": "^0.3.0",
     "react-dom": "^16.13.1",
-    "react-filepond": "^7.0.1",
+    "react-filepond": "^7.1.0",
     "react-highlighter": "^0.4.3",
     "react-hot-loader": "^4.12.21",
     "react-iframe-resizer-super": "^0.2.2",

services/core-web/src/components/Forms/noticeOfWork/EditNoticeOfWorkDocumentForm.js

@@ -10,6 +10,7 @@ import { required, maxLength, validateSelectOptions } from "@common/utils/Valida
 import { resetForm } from "@common/utils/helpers";
 import { getDropdownNoticeOfWorkApplicationDocumentTypeOptions } from "@common/selectors/staticContentSelectors";
 import { NOTICE_OF_WORK_DOCUMENT } from "@common/constants/API";
+import { DOCUMENT, EXCEL, SPATIAL } from "@/constants/fileTypes";
 import { renderConfig } from "@/components/common/config";
 import * as FORM from "@/constants/forms";
 import CustomPropTypes from "@/customPropTypes";
@@ -122,6 +123,7 @@ export class EditNoticeOfWorkDocumentForm extends Component {
                   component={FileUpload}
                   addFileStart={() => this.toggleDisabled(true)}
                   uploadUrl={NOTICE_OF_WORK_DOCUMENT(this.props.now_application_guid)}
+                  acceptedFileTypesMap={{ ...DOCUMENT, ...EXCEL, ...SPATIAL }}
                   allowMultiple
                   allowRevert
                 />

services/core-web/src/components/common/FileUpload.js

@@ -4,12 +4,14 @@ import PropTypes from "prop-types";
 import "filepond-polyfill";
 import { FilePond, registerPlugin } from "react-filepond";
 import { Switch } from "antd";
+import { invert, uniq } from "lodash";
 import { FunnelPlotOutlined } from "@ant-design/icons";
 import "filepond/dist/filepond.min.css";
 import FilePondPluginFileValidateSize from "filepond-plugin-file-validate-size";
 import FilePondPluginFileValidateType from "filepond-plugin-file-validate-type";
 import tus from "tus-js-client";
 import { ENVIRONMENT } from "@common/constants/environment";
+import { APPLICATION_OCTET_STREAM } from "@/constants/fileTypes";
 import { createRequestHeader } from "@common/utils/RequestHeaders";
 import { FLUSH_SOUND, WATER_SOUND } from "@/constants/assets";
 
@@ -21,10 +23,10 @@ const propTypes = {
   acceptedFileTypesMap: PropTypes.objectOf(PropTypes.string),
   onFileLoad: PropTypes.func,
   onRemoveFile: PropTypes.func,
+  addFileStart: PropTypes.func,
   chunkSize: PropTypes.number,
   allowRevert: PropTypes.bool,
   allowMultiple: PropTypes.bool,
-  addFileStart: PropTypes.func,
 };
 
 const defaultProps = {
@@ -53,7 +55,7 @@ class FileUpload extends React.Component {
           chunkSize: this.props.chunkSize,
           metadata: {
             filename: file.name,
-            filetype: file.type,
+            filetype: file.type || APPLICATION_OCTET_STREAM,
           },
           headers: createRequestHeader().headers,
           onError: (err) => {
@@ -89,7 +91,8 @@ class FileUpload extends React.Component {
   }
 
   render() {
-    const acceptedFileTypes = Object.values(this.props.acceptedFileTypesMap);
+    const fileValidateTypeLabelExpectedTypesMap = invert(this.props.acceptedFileTypesMap);
+    const acceptedFileTypes = uniq(Object.values(this.props.acceptedFileTypesMap));
 
     return (
       <div
@@ -125,7 +128,22 @@ class FileUpload extends React.Component {
           maxFileSize={this.props.maxFileSize}
           allowFileTypeValidation={acceptedFileTypes.length > 0}
           acceptedFileTypes={acceptedFileTypes}
-          fileValidateTypeLabelExpectedTypesMap={this.props.acceptedFileTypesMap}
+          fileValidateTypeLabelExpectedTypesMap={fileValidateTypeLabelExpectedTypesMap}
+          fileValidateTypeDetectType={(source, type) =>
+            new Promise((resolve, reject) => {
+              // If the browser can't automatically detect the file's MIME type, use the one stored in the "accepted file types" map.
+              if (!type) {
+                const exts = source.name.split(".");
+                const ext = exts && exts.length > 0 && `.${exts.pop()}`;
+                if (ext && ext in this.props.acceptedFileTypesMap) {
+                  type = this.props.acceptedFileTypesMap[ext];
+                } else {
+                  reject(type);
+                }
+              }
+              resolve(type);
+            })
+          }
         />
       </div>
     );

services/core-web/src/components/noticeOfWork/applications/NOWDocuments.js

@@ -75,7 +75,7 @@ export const NOWDocuments = (props) => {
       props: {
         onSubmit: handleAddDocument,
         now_application_guid: props.noticeOfWork.now_application_guid,
-        title: `Add Notice of Work document`,
+        title: "Add Notice of Work document",
         categoriesToShow: props.categoriesToShow,
       },
       content: modalConfig.EDIT_NOTICE_OF_WORK_DOCUMENT,

services/core-web/src/constants/fileTypes.js

@@ -25,22 +25,24 @@ export const DOCUMENT = { ...PDF, ...DOC, ...OPEN_DOC };
 
 export const IMAGE = {
   ".jpeg": "image/jpeg",
-  ".PNG": "image/png",
+  ".png": "image/png",
 };
 
 export const UNIQUELY_SPATIAL = {
   ".dbf": "application/dbf",
   ".geoJSon": "application/vnd.geo+json",
   ".gml": "application/gml+xml",
-  ".kml": "application/vnd.google-earth.kml+xml ",
+  ".kml": "application/vnd.google-earth.kml+xml",
   ".kmz": "application/vnd.google-earth.kmz",
-  ".prj": "application/octet-stream",
-  ".sbn": "application/octet-stream",
-  ".sbx": "application/octet-stream",
-  ".shp": "application/octet-stream",
-  ".shpz": "application/octet-stream",
-  ".shx": "application/octet-stream",
-  ".wkt": "application/octet-stream",
+  ".prj": APPLICATION_OCTET_STREAM,
+  ".sbn": APPLICATION_OCTET_STREAM,
+  ".sbx": APPLICATION_OCTET_STREAM,
+  ".shp": APPLICATION_OCTET_STREAM,
+  ".shpz": APPLICATION_OCTET_STREAM,
+  ".shx": APPLICATION_OCTET_STREAM,
+  ".wkt": APPLICATION_OCTET_STREAM,
 };
 
 export const SPATIAL = { ...UNIQUELY_SPATIAL, ".csv": "text/csv", ".xml": "application/xml" };
+
+export const APPLICATION_OCTET_STREAM = "application/octet-stream";

services/minespace-web/src/components/Forms/reports/ReportSubmissions.js

@@ -9,8 +9,6 @@ import LinkButton from "@/components/common/LinkButton";
 import { MINE_REPORT_DOCUMENT } from "@/constants/API";
 import { DOCUMENT, EXCEL } from "@/constants/fileTypes";
 
-import { ReportsUploadedFilesList } from "@/components/Forms/reports/ReportsUploadedFilesList";
-
 const propTypes = {
   mineGuid: PropTypes.string.isRequired,
   updateMineReportSubmissions: PropTypes.func.isRequired,