From 92b14da14cfb07dfada349e1c89df64151d7b703 Mon Sep 17 00:00:00 2001
From: steps39 <srgibbon@googlemail.com>
Date: Wed, 31 Jul 2024 17:16:27 +0100
Subject: [PATCH] Started to get MAP to play with CEFAS dredge data dump

---
 SedimentDataExplorer.html |  13 +-
 SedimentDataExplorer.js   |  22 +++-
 sdeCharts.js              |   4 +-
 sdeDredgeData.js          | 244 ++++++++++++++++++--------------------
 sdeMaps.js                |  14 ++-
 5 files changed, 159 insertions(+), 138 deletions(-)

diff --git a/SedimentDataExplorer.html b/SedimentDataExplorer.html
index 739bbcc..aed3d1b 100644
--- a/SedimentDataExplorer.html
+++ b/SedimentDataExplorer.html
@@ -25,13 +25,16 @@ <h1>Sediment Template Data Explorer - v0.20240725</h1>
         <button onclick="exportChart()">Export Chart</button>
         <button onclick="clearData()">Clear Data</button>
         <br>
-        CEFAS Sediment Data -
+        CEFAS Sediment Data - 
         <button onclick="openCEFASSelection()">Select CEFAS Data Sets</button>
-        <div id="sedDataModal" style="display: none;">
+        <div id="sedDredgeData" style="display: none;">
+        </div>
+            <div id="sedDataModal" style="display: none;">
             <h2>Select Data Sets</h2>
-            <input type="file" id="fileInputDD" multiple onchange="importDredgeData()"> <!-- Allow multiple file selection -->
-            <input type="text" id="urlInputDD" placeholder="URLs comma-separated" onchange="importDredgeData()">
-            <!-- Accept comma-separated URLs -->
+            <label for="fileInputDD">CEFAS dredge data file</label>
+            <input type="file" id="fileInputDD" onchange="importDredgeData()"> <!-- Could allow multiple file selection -->
+            <input type="text" id="urlInputDD" placeholder="URL" onchange="importDredgeData()">
+            <!-- Would accept comma-separated URLs -->
              <br>
             <input type="text" id="mlApplications" placeholder="MLA ids comma-separated" onchange="closeCEFASSelection()">
             <button onclick="closeCEFASSelection()">Finish CEFAS Selection</button>
diff --git a/SedimentDataExplorer.js b/SedimentDataExplorer.js
index 73fa20c..8aa82b0 100644
--- a/SedimentDataExplorer.js
+++ b/SedimentDataExplorer.js
@@ -35,6 +35,7 @@
         instanceSheet[i] = null;
     }
     dataSheetNames = ['Physical Data','Trace metal data','PAH data','PCB data','BDE data','Organotins data','Organochlorine data'];
+    dataSheetAbr = {'Physical Data': 'Phys','Trace metal data': 'TM','PAH data': 'PAH','PCB data': 'PCB','BDE data': 'BDE','Organotins data': 'OT','Organochlorine data': 'OC'};
     dataSheetNamesCheckboxes = [];
     for (let i = 0; i < dataSheetNames.length; i++) {
         dataSheetNamesCheckboxes[i] = dataSheetNames[i].replace(/\s/g, '').toLowerCase();
@@ -1396,7 +1397,11 @@ function filenameDisplay() {
     fileDisplayDiv.innerHTML = "";
 
     iconNo = 0;
-    for (dateSampled in selectedSampleInfo) {
+    const datesSampled = Object.keys(selectedSampleInfo);
+    datesSampled.sort();
+    datesSampled.forEach(dateSampled => {
+
+//    for (dateSampled in selectedSampleInfo) {
         currentIcon = markerPath + markerPngs[iconNo];
         iconNo = (iconNo + 1) % 9;
 
@@ -1413,16 +1418,27 @@ function filenameDisplay() {
         linkElement.href = fileURL;
         // console.log(fileURL);
         // console.log(dateSampled);
-        linkElement.textContent = `File for ${dateSampled}: ${fileURL}`;
+        positions = Object.keys(selectedSampleInfo[dateSampled].position);
+        infos = '';
+        for (dataType in selectedSampleMeasurements[dateSampled]) {
+            infos += dataSheetAbr[dataType] + ' ';
+        }
+//        infos = 'PAH BDE OC OT PCB Phys TM';
+        textElement1 = document.createTextNode(`${dateSampled} has ${positions.length} samples with ${infos}:`);
+        textElement2 = document.createTextNode(`File `);
+        linkElement.textContent = `${fileURL}`;
         linkElement.target = "_blank"; // Open link in a new tab/window
 
         // Append the icon before the link
         fileDisplayDiv.appendChild(iconElement);
+        fileDisplayDiv.appendChild(textElement1);
+        fileDisplayDiv.appendChild(document.createElement("br"));
+        fileDisplayDiv.appendChild(textElement2);
         fileDisplayDiv.appendChild(linkElement);
 
         // Add a line break for better readability
         fileDisplayDiv.appendChild(document.createElement("br"));
-    };
+    });
     fileDisplayDiv.style.display = 'none';
 }
 
diff --git a/sdeCharts.js b/sdeCharts.js
index a91ce3b..3d634a7 100644
--- a/sdeCharts.js
+++ b/sdeCharts.js
@@ -1218,7 +1218,9 @@ function displayAnyChart(meas, all, datasets, instanceNo, title, yTitle, showLeg
             for (let i = 0; i < chart.scales.x.ticks.length; i++) {
                 if (x >= left + (right * i) && x <= left + (right * (i + 1))) {
                     console.log('x label', i);
-                    const regexPattern = /^(\S+): (.+)$/;
+//                    const regexPattern = /^(\S+): (.+)$/;
+                    const regexPattern = /^(.+): (.+)$/;
+console.log(all[i]);
                     const matchResult = all[i].match(regexPattern);
                     if (matchResult) {
                         // Extracted parts
diff --git a/sdeDredgeData.js b/sdeDredgeData.js
index 051ae93..27f69c5 100644
--- a/sdeDredgeData.js
+++ b/sdeDredgeData.js
@@ -1,5 +1,6 @@
 CEFASdata = {};
 CEFASfile = {};
+CEFASUniqueRows = {};
 ddLookup = {};
 ddLookup.chemical = {};
 ddLookup.sheet = {};
@@ -161,102 +162,104 @@ ddLookup.sheet['pb']="Trace metal data";
 ddLookup.sheet['zn']="Trace metal data";
 
 
+CEFASconcentration='concentration';
+CEFASlatitude = 'lat';
+CEFASlongitude = 'lon';
+CEFASmla = 'ml_application';
+CEFASchemical = 'parameter_measured';
+CEFASsampledate = 'sample_date_collection';
+CEFASdepth = 'sample_depth';
+CEFASsamplename = 'sample_reference';
+
 everything = {};
 
-    function importDredgeData() {
-        urls = {};
-        const fileInputDD = document.getElementById('fileInputDD');
-        const urlInputDD = document.getElementById('urlInputDD');
-        files = fileInputDD.files; // Files is now a FileList object containing multiple files
-//console.log(files);
-        urls = urlInputDD.value.trim().split(',').map(url => url.trim()); // Split comma-separated URLs
-        if (files.length === 0 && urls.length === 0) {
-            alert('Please select files or enter URLs.');
-            return;
+function importDredgeData() {
+    urls = {};
+    const fileInputDD = document.getElementById('fileInputDD');
+    const urlInputDD = document.getElementById('urlInputDD');
+    files = fileInputDD.files; // Files is now a FileList object containing multiple files
+    //console.log(files);
+    urls = urlInputDD.value.trim().split(',').map(url => url.trim()); // Split comma-separated URLs
+    if (files.length === 0 && urls.length === 0) {
+        alert('Please select files or enter URLs.');
+        return;
+    }
+    // Process files
+    if (files.length > 0) {
+        let filesProcessed = 0; // counter to track the number of files processed
+        let fL = files.length;
+        console.log(files.length);
+        for (let i = 0; i < files.length; i++) {
+            CEFASfilename = files[i].name;
+            const reader = new FileReader();
+            reader.onload = function (e) {
+                data = new Uint8Array(e.target.result);
+                filesProcessed++; // Increment the counter after processing each file
+                ret = loadDredgeData(data);
+                CEFASdata = ret['df'];
+                CEFASUniqueRows = ret['uniqueRows'];
+            };
+            reader.readAsArrayBuffer(files[i]);
         }
-        // Process files
-        if (files.length > 0) {
-            let filesProcessed = 0; // counter to track the number of files processed
-            let fL = files.length;
-console.log(files.length);
-            for (let i = 0; i < files.length; i++) {
-                CEFASfilename = files[i].name;
-//console.log(filename);
-                const reader = new FileReader();
+    }
+    if (urls.length > 0) {
+        // Array to store all fetch promises
+        const fetchPromises = [];
 
-                reader.onload = function (e) {
-                    CEFASdata = new Uint8Array(e.target.result);
-//                    var workbook = XLSX.read(data, {type:"array"});
-//console.log(workbook);
-//everything = workbook;
-//everything 
-//                    processDDExcelData(data,filename);
-                    filesProcessed++; // Increment the counter after processing each file
-                    // Check if all files have been processed
-//console.log(files.length,fL, filesProcessed);
-/*                    if (filesProcessed === fL) {
-//console.log('calling updateChart');
-                        updateChart(); // Call updateChart once all files have been processed
-                    }*/
-                };
-                reader.readAsArrayBuffer(files[i]);
+        urls.forEach(url => {
+            // Check if the URL is a valid URL before fetching
+            if (!/^https?:\/\//i.test(url)) {
+                console.error('Invalid URL:', url);
+                return;
             }
-        }
-        if (urls.length > 0) {
-            // Array to store all fetch promises
-            const fetchPromises = [];
-        
-            urls.forEach(url => {
-                // Check if the URL is a valid URL before fetching
-                if (!/^https?:\/\//i.test(url)) {
-                    console.error('Invalid URL:', url);
-                    return;
-                }
-        
-                // Push each fetch promise into the array
-                fetchPromises.push(
-                    fetch(url)
-                        .then(response => response.arrayBuffer())
-                        .then(data => {
-                            processDDExcelData(new Uint8Array(data), url);
-console.log('processexcelDDdata again');
-                        })
-                        .catch(error => {
-                            console.error('Error fetching the DD file:', error);
-                        })
-                );
-            });
-/*        
-            // Wait for all fetch promises to resolve
-            Promise.all(fetchPromises)
-                .then(() => {
-                    updateChart();
-//console.log('there again');
-                });
-                */
-        }
 
-//console.log('Import Data out of fetch');
-//        updateChart();
-// Clear the input field after reading data
-//CEFASdata = data;
-//CEFASfile = filename;
-        fileInputDD.value = '';
-        urlInputDD.value = '';
+            // Push each fetch promise into the array
+            fetchPromises.push(
+                fetch(url)
+                    .then(response => response.arrayBuffer())
+                    .then(data => {
+                        var data = new Uint8Array(e.target.result);
+                        ret = loadDredgeData(data);
+                        CEFASdata = ret['df'];
+                        CEFASUniqueRows = ret['uniqueRows'];
+                    })
+                    .catch(error => {
+                        console.error('Error fetching the DD file:', error);
+                    })
+            );
+        });
     }
+    fileInputDD.value = '';
+    urlInputDD.value = '';
+}
 
-function processDDExcelData(data, url) {
-    console.log('processexceldata', url);
+function loadDredgeData(data) {
     const workbook = XLSX.read(data, { type: 'array' });
     //everything = workbook;        
     sheetData = workbook.Sheets['Dredge Contaminant Seabed Data '];
-//    mlApplication = 'MLA/2016/00341,MLA/2017/00002';
+    let uniqueRows = [];
+    let uniqueMLAs = [];
+    const df = XLSX.utils.sheet_to_json(sheetData, { cellText: true });
+    console.log(df);
+    df.forEach(row => {
+        let mlaName = row[CEFASmla];
+        if (mlaName && !uniqueMLAs[mlaName]) {
+            console.log('found a unique one ', mlaName);
+            uniqueMLAs[mlaName] = true;
+            uniqueRows.push(row);
+        }
+    });
+    sedDredgeDataDisplay(uniqueRows.length);
+    return {df,uniqueRows};
+}
+
+        function processDDExcelData(data, url) {
+    console.log('processexceldata', url);
+sheetData = data;
     const mlaInput = document.getElementById('mlApplications');
-//console.log(mlaInput);
     mlas = mlaInput.value.trim().split(',').map(mla => mla.trim()); // Split comma-separated URLs
 if (mlas.length > 0) {
         mlas.forEach(mlApplication => {
-//console.log(mlApplication);
             extractDataFromSheet(sheetData, mlApplication);
         });
     }
@@ -266,36 +269,32 @@ if (mlas.length > 0) {
     updateChart();
 
 }
+
     
 function extractDataFromSheet(sheetData, mlApplication) {
-    const df = XLSX.utils.sheet_to_json(sheetData, { header: 1, cellText: true });
-    //                const df = XLSX.utils.sheet_to_json(sheetData, { header: 1, cellText: true });
-    let startRow = -1;
-    let startCol = -1;
-    let measurementUnit = 'Not set';
-    let totalSum = 0;
-    testDD = df.filter(row => row[4] && row[4].includes(mlApplication));
-    let sampleColumnIndex = 3;
+df = sheetData;
+    testDD = df.filter(row => row[CEFASmla] && row[CEFASmla].includes(mlApplication));
     let uniqueSamples = {};
     let uniqueRows = [];
 
     testDD.forEach(row => {
-        let sampleName = row[sampleColumnIndex];
+        let sampleName = row[CEFASsamplename];
         if (sampleName && !uniqueSamples[sampleName]) {
             uniqueSamples[sampleName] = true;
             uniqueRows.push(row);
         }
     });
     //console.log(uniqueRows);
-    sampleDate = parseDates(uniqueRows[0][5]);
-    dateSampled = sampleDate + ' ' + uniqueRows[0][4];
-console.log(dateSampled);
+    sampleDate = parseDates(uniqueRows[0][CEFASsampledate]);
+    dateSampled = sampleDate + ' ' + uniqueRows[0][CEFASmla];
+//console.log(dateSampled);
     sampleInfo[dateSampled] = {};
     sampleInfo[dateSampled]['Date Sampled'] = sampleDate;
     sampleInfo[dateSampled]['fileURL'] = CEFASfilename;
     sampleInfo[dateSampled].position = {};
     uniqueRows.forEach(row => {
-        sampleInfo[dateSampled].position[row[3]] = { 'Position latitude': row[7], 'Position longitude': row[6], 'Sampling depth (m)': row[12] };
+        sampleInfo[dateSampled].position[row[CEFASsamplename]] = { 'Position latitude': row[CEFASlatitude], 'Position longitude': row[CEFASlongitude],
+             'Sampling depth (m)': row[CEFASdepth] };
     });
     if (!(dateSampled in sampleMeasurements)) {
         meas = {};
@@ -305,12 +304,12 @@ console.log(dateSampled);
     everything = testDD;
     //meas = {};
     testDD.forEach(row => {
-        chemicalAbr = row[8].trim();
+        chemicalAbr = row[CEFASchemical].trim();
         if (chemicalAbr in ddLookup.sheet) {
             sheetName = ddLookup.sheet[chemicalAbr];
             chemicalName = ddLookup.chemical[chemicalAbr];
-            sample = row[3];
-            concentration = parseFloat(row[9]);
+            sample = row[CEFASsamplename];
+            concentration = parseFloat(row[CEFASconcentration]);
             if (!(sheetName in meas)) {
                 meas[sheetName] = {};
                 meas[sheetName].chemicals = {};
@@ -326,8 +325,6 @@ console.log(dateSampled);
                     meas[sheetName].chemicals[chemicalName] = {};
                     meas[sheetName].chemicals[chemicalName].samples = {};
                 }
-//                    console.log(sheetName, chemicalName);
-                //console.log(ddLookup.chemical[chemicalAbr]);
                 if (!(sample in meas[sheetName].total)) {
                     meas[sheetName].total[sample] = 0;
                 }
@@ -370,53 +367,46 @@ function closeCEFASSearch() {
     const radius = parseFloat(document.getElementById('radius').value);
     startDate = new Date(document.getElementById('startDate').value);
     finishDate = new Date(document.getElementById('finishDate').value);
-console.log(startDate,finishDate);
-    //    console.log('processexceldata', url);
-    const workbook = XLSX.read(CEFASdata, { type: 'array' });
-    //everything = workbook;        
-    sheetData = workbook.Sheets['Dredge Contaminant Seabed Data '];
-    let uniqueRows = [];
-    let uniqueMLAs = [];
-    mlaColumnIndex = 4;
-    const df = XLSX.utils.sheet_to_json(sheetData, { header: 1, cellText: true });
-    df.forEach(row => {
-        let mlaName = row[mlaColumnIndex];
-        if (mlaName && !uniqueMLAs[mlaName]) {
-            uniqueMLAs[mlaName] = true;
-            uniqueRows.push(row);
-        }
-    });
     mlas = [];
+    uniqueRows = CEFASUniqueRows;
+    console.log(uniqueRows);
     uniqueRows.forEach(row => {
-        samplingDate = new Date(parseDates(row[5])[0]);
-        sampleLat = row[7];
-        sampleLon = row[6];
+        samplingDate = new Date(parseDates(row[CEFASsampledate])[0]);
+        sampleLat = row[CEFASlatitude];
+        sampleLon = row[CEFASlongitude];
         distance = 1000 * haversineDistance(sampleLat, sampleLon, centreLat, centreLon);
         if (distance <= radius) {
             console.log(startDate);
             if(startDate.toString() === 'Invalid Date' || finishDate.toString() === 'Invalid Date'){
-//            if(startDate.toString() === 'Invalid Date'){
                     console.log('Pushing Invalid Date');
-console.log(samplingDate);
-                mlas.push(row[4]);
+                mlas.push(row[CEFASmla]);
             } else {
-                console.log(isBetweenDates(startDate,finishDate,samplingDate));
                 if (isBetweenDates(startDate,finishDate,samplingDate)){
-console.log('Pushing');
-console.log(samplingDate);
-                    mlas.push(row[4]);
+                    mlas.push(row[CEFASmla]);
                 }
             }
         }
     });
-//console.log(mlas);
+console.log(mlas);
     if (mlas.length > 0) {
         mlas.forEach(mlApplication => {
             //console.log(mlApplication);
-            extractDataFromSheet(sheetData, mlApplication);
+            extractDataFromSheet(CEFASdata, mlApplication);
         });
         selectedSampleInfo = sampleInfo;
         selectedSampleMeasurements = sampleMeasurements;
         updateChart();
     }
 }
+
+function sedDredgeDataDisplay(noMLAs) {
+    const sedDDDisplayDiv = document.getElementById("sedDredgeData");
+    // blank it each time
+    sedDDDisplayDiv.innerHTML = "";
+    var countNode = document.createTextNode(`${noMLAs} marine applications available`);
+    sedDDDisplayDiv.appendChild(countNode);
+        // Add a line break for better readability
+        sedDDDisplayDiv.appendChild(document.createElement("br"));
+    sedDDDisplayDiv.style.display = 'block';
+}
+
diff --git a/sdeMaps.js b/sdeMaps.js
index 5411b70..4f8823d 100644
--- a/sdeMaps.js
+++ b/sdeMaps.js
@@ -117,8 +117,14 @@ function sampleMap(meas) {
                                 //console.log('distance1',distance);
                                 if (distance <= 10) {
                                     hoveredSample = ds + ': ' + s;
+console.log(meas);
+console.log(ds);
+console.log(hoveredSample);
                                     createHighlights(meas, ds, hoveredSample);
-                                    popupInstance[hoveredSample].update();
+console.log(popupInstance);
+                                    if(hoveredSample in popupInstance) {
+                                        popupInstance[hoveredSample].update();
+                                    }
                                     let popup = marker.getPopup();
                                     let chart_div = document.getElementById("c_radar_" + hoveredSample);
                                     chart_div.style.height = '300px';
@@ -156,8 +162,12 @@ console.log(popup);
                                 //console.log('distance2',distance);
                                 if (distance <= 10) {
                                     hoveredSample = ds + ': ' + s;
+console.log(meas,ds,hoveredSample);
                                     createHighlights(meas, ds, hoveredSample);
-                                    popupInstance[hoveredSample].update();
+console.log(popupInstance);
+                                    if(hoveredSample in popupInstance) {
+                                        popupInstance[hoveredSample].update();
+                                    }
                                     popup = marker.getPopup();
                                     chart_div = document.getElementById("c_radar_" + hoveredSample);
 /*                                    chart_div.style.height = '200px';