Merge pull request #125 from codytodonnell/feature/data-defaults

Update explore-data default data

strudel-taskflows/src/pages/contribute-data/_context/utils.ts

@@ -71,12 +71,8 @@ export const filterByDataFilters = (allData: any[], filters: DataFilter[]) => {
                 && f.value[0]
                 && f.value[1]
               ) {
-                console.log(d[f.field]);
-                console.log(f.value);
-                console.log(dayjs(d[f.field]));
                 const dateValue = dayjs(d[f.field]);
                 if (dateValue.isAfter(f.value[0]) && dateValue.isBefore(f.value[1])) {
-                  console.log('match');
                   match = true;
                 }
               } else {

strudel-taskflows/src/pages/explore-data/_components/DataView.tsx

@@ -4,7 +4,7 @@ import { keepPreviousData, useQuery } from '@tanstack/react-query';
 import React, { useState } from 'react';
 import { SciDataGrid } from '../../../components/SciDataGrid';
 import { filterData } from '../../../utils/filters.utils';
-import { createFilterParams } from '../../../utils/queryParams.utils';
+import { createFilterParams, fetchData } from '../../../utils/queryParams.utils';
 import { taskflow } from '../_config/taskflow.config';
 import { useFilters } from '../../../components/FilterContext';
 
@@ -48,8 +48,12 @@ export const DataView: React.FC<DataViewProps> = ({ searchTerm, setPreviewItem }
     queryKey,
     queryFn: async (): Promise<any> => {
       const queryString = queryParams.toString()
-      const response = await fetch(`${dataSource}?${queryString}`);
-      return await response.json();
+      let fullDataSourcePath = dataSource;
+      if (queryString && queryString.length > 0) {
+        fullDataSourcePath = `${dataSource}?${queryString}`;
+      }
+      const results = await fetchData(fullDataSourcePath);
+      return results;
     },
     placeholderData: keepPreviousData,
   });

strudel-taskflows/src/pages/explore-data/_config/taskflow.config.ts

@@ -9,18 +9,15 @@ export const taskflow: ExploreDataConfig = {
       /**
        * URL or path to the data source
        */
-      source: "https://exoplanetarchive.ipac.caltech.edu/cgi-bin/nstedAPI/nph-nstedAPI",
+      source: "data/default/explore-data/exoplanets.csv",
       /**
        * Key-value object of params that should always be included in the query URL
        */
-      staticParams: {
-        table: 'cumulative',
-        format: 'json'
-      },
+      staticParams: null,
       /**
        * Name of the field in the data that represents a unique identifier for each record.
        */
-      idField: "kepoi_name",
+      idField: "Id",
       /**
        * Method by which data should be filtered, either client or server.
        */
@@ -30,12 +27,9 @@ export const taskflow: ExploreDataConfig = {
      * Data definition for the item detail page
      */
     detail: {
-      source: "https://exoplanetarchive.ipac.caltech.edu/cgi-bin/nstedAPI/nph-nstedAPI",
-      staticParams: {
-        table: 'cumulative',
-        format: 'json'
-      },
-      idField: "kepoi_name",
+      source: "data/default/explore-data/exoplanets.csv",
+      staticParams: null,
+      idField: "Id",
       queryMode: "client",
     }
   },
@@ -54,63 +48,92 @@ export const taskflow: ExploreDataConfig = {
        */
       tableColumns: [
         {
-          field: "kepid",
-          headerName: "Kepler ID",
+          field: "Planet Name",
+          headerName: "Planet Name",
           width: 200
         },
         {
-          field: "kepoi_name",
-          headerName: "Kepler OI Name",
-          width: 150
+          field: "Planet Host",
+          headerName: "Planet Host",
+          width: 200
         },
         {
-          field: "kepler_name",
-          headerName: "Kepler Name",
-          width: 150
+          field: "Discovery Method",
+          headerName: "Discovery Method",
+          width: 200
+        },
+        {
+          field: "Orbital Period Days",
+          headerName: "Orbital Period",
+          units: 'days',
+          type: 'number',
+          width: 200
         },
         {
-          field: "koi_disposition",
-          headerName: "Disposition",
-          width: 150
+          field: "Mass",
+          headerName: "Mass",
+          units: 'Earth Mass',
+          type: 'number',
+          width: 200
         },
         {
-          field: "koi_period",
-          headerName: "Period",
-          width: 150,
-          type: 'number'
-        }
+          field: "Eccentricity",
+          headerName: "Eccentricity",
+          type: 'number',
+          width: 200
+        },
       ],
       /**
        * List of filters to display on the main page and use to filter the main table data. 
        * Each filter has a definition object to determine how it renders and functions.
        */
       tableFilters: [
         {
-          field: "koi_disposition",
-          label: "Disposition",
+          field: "Discovery Method",
+          label: "Discovery Method",
           operator: "contains-one-of",
           filterComponent: "CheckboxList",
           filterProps: {
             options: [
               {
-                label: "CONFIRMED",
-                value: "CONFIRMED"
+                label: "Astrometry",
+                value: "Astrometry"
+              },
+              {
+                label: "Disk Kinematics",
+                value: "Disk Kinematics"
               },
               {
-                label: "FALSE POSITIVE",
-                value: "FALSE POSITIVE"
-              }
+                label: "Eclipse Timing Variations",
+                value: "Eclipse Timing Variations"
+              },
+              {
+                label: "Imaging",
+                value: "Imaging"
+              },
+              {
+                label: "Microlensing",
+                value: "Microlensing"
+              },
+              {
+                label: "Radial Velocity",
+                value: "Radial Velocity"
+              },
+              {
+                label: "Transit",
+                value: "Transit"
+              },
             ]
           }
         },
         {
-          field: "koi_period",
-          label: "Period",
+          field: "Mass",
+          label: "Mass",
           operator: "between-inclusive",
           filterComponent: "RangeSlider",
           filterProps: {
             min: 0,
-            max: 100
+            max: 10000
           }
         },
       ]

strudel-taskflows/src/pages/explore-data/_tests/explore-data.cy.ts

@@ -7,15 +7,15 @@ describe('The Explore Data Task Flow', () => {
     cy.contains('Explore Data')
   });
 
-  // it('filters by disposition', () => {
-  //   cy.get('span').contains('CONFIRMED').click();
-  //   cy.wait(5000);
-  //   cy.contains('1–25 of 2743');
-  // });
+  it('filters by checkbox', () => {
+    cy.get('span').contains('Astrometry').click();
+    cy.wait(3000);
+    cy.contains('of 1');
+  });
 
-  // it('opens the preview panel', () => {
-  //   cy.wait(5000);
-  //   cy.get('div[role="gridcell"]').first().click();
-  //   cy.contains('Property Group 1');
-  // });
+  it('opens the preview panel', () => {
+    cy.wait(3000);
+    cy.get('div[role="gridcell"]').first().click();
+    cy.contains('Property Group 1');
+  });
 });

strudel-taskflows/src/pages/search-data-repositories/_context/ContextProvider.tsx

@@ -81,7 +81,6 @@ function searchDataRepositoriesReducer(state: SearchDataRepositoriesState, actio
       }
     }
     case SearchDataRepositoriesActionType.SET_FILTER: {
-      console.log(action);
       const filter = action.payload;
       const existingIndex = state.activeFilters.findIndex((f) => f.field === filter.field);
       const activeFilters = [...state.activeFilters];
@@ -116,7 +115,6 @@ export const SearchDataRepositoriesProvider: React.FC<SearchDataRepositoriesProv
   const value = { state, dispatch };
 
   useEffect(() => {
-    console.log(props.data);
     dispatch(setData(props.data));
   }, [props.data]);
 

strudel-taskflows/src/utils/filters.utils.ts

@@ -106,7 +106,6 @@ export const filterByDataFilters = (allData: any[], filters: DataFilter[], filte
 };
 
 export const filterData = (allData: any[], filters: DataFilter[], filterConfigs: FilterConfig[], searchText?: string) => {
-  console.log(searchText);
   const filteredByText = filterBySearchText(allData, searchText);
   const filteredByTextAndDataFilters = filterByDataFilters(filteredByText, filters, filterConfigs);
   return filteredByTextAndDataFilters;

strudel-taskflows/src/utils/queryParams.utils.ts

@@ -1,4 +1,5 @@
 import { DataFilter, FilterConfig } from "../types/filters.types";
+import * as d3 from 'd3-fetch';
 
 /**
  * Convert an array of values to a URL param by
@@ -52,7 +53,6 @@ export const buildParamsString = (filters: DataFilter[], filterConfigs: FilterCo
   let paramsString = '';
   filters.forEach((filter, i) => {
     const filterConfig = filterConfigs.find((c) => c.field === filter.field); 
-    console.log(filterConfig);
     switch(filterConfig?.paramType) {
       case 'array-string':
         paramsString = paramsString.concat(toParamArrayString(filter.field, filter.value, filterConfig.paramTypeOptions.separator));
@@ -105,4 +105,33 @@ export const createFilterParams = (filters: DataFilter[], filterConfigs: FilterC
     }
   });
   return params;
+}
+
+export const cleanUrl = (url: string) => {
+  return url.replace(/([^:]\/)\/+/g, "$1");
+}
+
+/**
+ * Fetch data from a local CSV, TSV, or JSON, or an external API
+ * that returns JSON.
+ */
+export const fetchData = async (dataSource: string) => {
+  // Get the base portion of the URL. Will be blank when running locally.
+  const base = document.querySelector('base')?.getAttribute('href') ?? '';
+  // Use the VITE_BASE_URL env variable to specify a path prefix that 
+  // should be added to routes and local requests
+  const basename = base + import.meta.env.VITE_BASE_URL;
+  const fileExtension = dataSource.split('.').pop();
+  const isExternal = dataSource.startsWith('http');
+  const dataSourcePath = isExternal ? cleanUrl(dataSource) : cleanUrl(`${basename}/${dataSource}`);
+  let data: any = [];
+  if (fileExtension === 'csv') {
+    data = await d3.csv(dataSourcePath);
+  } else if (fileExtension === 'tsv') {
+    data = await d3.tsv(dataSourcePath);
+  } else if (fileExtension === 'json' || isExternal) {
+    const response = await fetch(dataSourcePath);
+    data = await response.json();
+  }
+  return data;
 }

strudel-taskflows/src/utils/useDataFromSource.tsx

@@ -22,7 +22,7 @@ export const useDataFromSource = (dataSource: string): any => {
     const fetchData = async () => {
       const fileExtension = dataSource.split('.').pop();
       const isExternal = dataSource.startsWith('http');
-      const dataSourcePath = isExternal ? dataSource : `${basename}/data/${dataSource}`;
+      const dataSourcePath = isExternal ? dataSource : `${basename}/${dataSource}`;
       let data: any = [];
       if (fileExtension === 'csv') {
         data = await d3.csv(dataSourcePath);