From bba9077490d046b1cd9698376c7ae16e6b3908d2 Mon Sep 17 00:00:00 2001 From: Zakki Date: Tue, 1 Aug 2023 13:15:55 +0700 Subject: [PATCH 1/4] Add Server Side processing in View Listing --- django_project/dashboard/api_views/views.py | 218 ++++++-- django_project/dashboard/src/app/store.ts | 4 +- .../dashboard/src/reducers/viewTable.ts | 52 ++ .../dashboard/src/views/View/Views.tsx | 493 ++++++++++++++---- .../dashboard/src/views/View/ViewsFilter.ts | 21 + .../dashboard/tests/test_api_views.py | 25 - .../dashboard/tests/test_view_filter_value.py | 81 +++ .../dashboard/tests/test_view_list.py | 144 +++++ django_project/dashboard/urls.py | 6 +- .../georepo/api_views/dataset_view.py | 2 +- django_project/georepo/models/dataset_view.py | 4 + django_project/georepo/utils/permission.py | 35 ++ 12 files changed, 916 insertions(+), 169 deletions(-) create mode 100644 django_project/dashboard/src/reducers/viewTable.ts create mode 100644 django_project/dashboard/src/views/View/ViewsFilter.ts create mode 100644 django_project/dashboard/tests/test_view_filter_value.py create mode 100644 django_project/dashboard/tests/test_view_list.py diff --git a/django_project/dashboard/api_views/views.py b/django_project/dashboard/api_views/views.py index b32ffa0b..36b6d67c 100644 --- a/django_project/dashboard/api_views/views.py +++ b/django_project/dashboard/api_views/views.py @@ -1,17 +1,18 @@ import re import uuid import os.path -from django.db.models.expressions import RawSQL +import math +from django.db.models.expressions import RawSQL, Q from django.conf import settings from django.contrib.auth.mixins import UserPassesTestMixin from django.db import connection from django.http import Http404, HttpResponseForbidden, HttpResponse +from django.core.paginator import Paginator from django.shortcuts import get_object_or_404 from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView from rest_framework.exceptions import ValidationError -from guardian.shortcuts import get_objects_for_user from azure_auth.backends import AzureAuthRequiredMixin from celery.result import AsyncResult from core.celery import app @@ -41,10 +42,8 @@ from georepo.tasks.simplify_geometry import simplify_geometry_in_view from georepo.utils.permission import ( check_user_has_view_permission, - get_dataset_for_user, - get_dataset_views_for_user, - get_view_permission_privacy_level, - EXTERNAL_READ_VIEW_PERMISSION_LIST + get_views_for_user, + get_view_permission_privacy_level ) from georepo.utils.exporter_base import APIDownloaderBase @@ -201,47 +200,174 @@ class ViewList(AzureAuthRequiredMixin, APIView): """ permission_classes = [IsAuthenticated] - def get(self, *args, **kwargs): - datasets = Dataset.objects.all() - datasets = get_dataset_for_user(self.request.user, datasets) - views_querysets = DatasetView.objects.none() - user_privacy_levels = {} - for dataset in datasets: - views = DatasetView.objects.filter( - dataset=dataset - ) - views, _ = get_dataset_views_for_user( - self.request.user, - dataset, - views - ) - views_querysets = views_querysets.union(views) - privacy_level = get_view_permission_privacy_level( - self.request.user, - dataset - ) - user_privacy_levels[dataset.id] = privacy_level - # include external user - external_views = DatasetView.objects.all() - external_views = get_objects_for_user( - self.request.user, - EXTERNAL_READ_VIEW_PERMISSION_LIST, - klass=external_views, - use_groups=True, - any_perm=True, - accept_global_perms=False + def _filter_tags(self, request): + tags = dict(request.data).get('tags', []) + if not tags: + return {} + return {'tags__name__in': tags} + + def _filter_mode(self, request): + mode = dict(request.data).get('mode', []) + if not mode or sorted(mode) == ['Dynamic', 'Static']: + return {} + + return {'is_static': True if mode[0] == 'Static' else False} + + def _filter_dataset(self, request): + dataset = dict(request.data).get('dataset', []) + if not dataset: + return {} + + return {'dataset__label__in': dataset} + + def _filter_is_default(self, request): + is_default = dict(request.data).get('is_default', []) + if not is_default or sorted(is_default) == ['No', 'Yes']: + return {} + + return {'default_type': True if is_default[0] == 'Yes' else False} + + def _filter_min_privacy(self, request): + min_privacy = dict(request.data).get('min_privacy', []) + if not min_privacy: + return {} + + return {'min_privacy_level__in': min_privacy} + + def _filter_max_privacy(self, request): + max_privacy = dict(request.data).get('max_privacy', []) + if not max_privacy: + return {} + + return {'max_privacy_level__in': max_privacy} + + def _filter_queryset(self, queryset, request): + filter_kwargs = {} + filter_kwargs.update(self._filter_tags(request)) + filter_kwargs.update(self._filter_mode(request)) + filter_kwargs.update(self._filter_dataset(request)) + filter_kwargs.update(self._filter_min_privacy(request)) + filter_kwargs.update(self._filter_max_privacy(request)) + return queryset.filter(**filter_kwargs) + + def _search_queryset(self, queryset, request): + search_text = request.data.get('search_text', '') + if not search_text: + return queryset + char_fields = [ + field.name for field in DatasetView.get_fields() if + field.get_internal_type() in ['UUIDField', 'CharField', 'TextField'] + ] + q_args = [ + Q(**{f"{field}__icontains": search_text}) for field in char_fields + ] + args = Q() + for arg in q_args: + args |= arg + queryset = queryset.filter(*(args,)) + return queryset + + def _sort_queryset(self, queryset, request): + sort_by = request.query_params.get('sort_by', 'name') + sort_direction = request.query_params.get('sort_direction', 'asc') + if not sort_by: + sort_by = 'name' + if not sort_direction: + sort_direction = 'asc' + ordering = sort_by if sort_direction == 'asc' else f"-{sort_by}" + queryset = queryset.order_by(ordering) + return queryset + + def post(self, *args, **kwargs): + user_privacy_levels, views_querysets = get_views_for_user(self.request.user) + # It seems we cannot use values_list on views_queryset + views_querysets = DatasetView.objects.\ + filter(id__in=[v.id for v in views_querysets]) + views_querysets = self._search_queryset(views_querysets, self.request) + views_querysets = self._filter_queryset(views_querysets, self.request) + page = int(self.request.GET.get('page', '1')) + page_size = int(self.request.query_params.get('page_size', '10')) + views_querysets = self._sort_queryset(views_querysets, self.request) + paginator = Paginator(views_querysets, page_size) + total_page = math.ceil(paginator.count / page_size) + if page > total_page: + output = [] + else: + paginated_entities = paginator.get_page(page) + output = DatasetViewSerializer( + paginated_entities, + many=True, + context={ + 'user': self.request.user, + 'user_privacy_levels': user_privacy_levels + } + ).data + + return Response({ + 'count': paginator.count, + 'page': page, + 'total_page': total_page, + 'page_size': page_size, + 'results': output, + }) + + +class ViewFilterValue( + AzureAuthRequiredMixin, + APIView +): + """ + Get filter value for given View and criteria + """ + permission_classes = [IsAuthenticated] + views_querysets = DatasetView.objects.none() + + def get_user_views(self): + _, views_querysets = get_views_for_user(self.request.user) + views_querysets = DatasetView.objects.filter( + id__in=[v.id for v in views_querysets] ) - views_querysets = views_querysets.union(external_views) - views_querysets = views_querysets.order_by('created_at') - views_serializer = DatasetViewSerializer( - views_querysets, - many=True, - context={ - 'user': self.request.user, - 'user_privacy_levels': user_privacy_levels - } - ).data - return Response(views_serializer) + return views_querysets + + def fetch_tags(self): + tags = self.views_querysets.order_by().\ + values_list('tags__name', flat=True).distinct() + return [tag for tag in tags if tag] + + def fetch_mode(self): + return ( + 'Static', + 'Dynamic' + ) + + def fetch_dataset(self): + return self.views_querysets.exclude( + dataset__label__isnull=True + ).exclude( + dataset__label__exact='' + ).order_by().values_list('dataset__label', flat=True).distinct() + + def fetch_is_default(self): + return ( + 'No', + 'Yes' + ) + + def fetch_min_privacy(self): + return self.views_querysets.order_by().\ + values_list('min_privacy_level', flat=True).distinct() + + def fetch_max_privacy(self): + return self.views_querysets.order_by().\ + values_list('max_privacy_level', flat=True).distinct() + + def get(self, request, criteria, *args, **kwargs): + self.views_querysets = self.get_user_views() + try: + data = eval(f"self.fetch_{criteria}()") + except AttributeError: + data = [] + return Response(status=200, data=data) class SQLColumnsTablesList(APIView): diff --git a/django_project/dashboard/src/app/store.ts b/django_project/dashboard/src/app/store.ts index 8813c0b1..794e762e 100644 --- a/django_project/dashboard/src/app/store.ts +++ b/django_project/dashboard/src/app/store.ts @@ -5,6 +5,7 @@ import moduleReducer from "../reducers/module" import pollIntervalReducer from "../reducers/notificationPoll" import maintenanceReducer from "../reducers/maintenanceItem" import reviewActionReducer from "../reducers/reviewAction" +import viewTableReducer from "../reducers/viewTable" export const store = configureStore({ reducer: { @@ -13,7 +14,8 @@ export const store = configureStore({ module: moduleReducer, pollInterval: pollIntervalReducer, maintenanceItem: maintenanceReducer, - reviewAction: reviewActionReducer + reviewAction: reviewActionReducer, + viewTable: viewTableReducer }, }); diff --git a/django_project/dashboard/src/reducers/viewTable.ts b/django_project/dashboard/src/reducers/viewTable.ts new file mode 100644 index 00000000..5645f6dc --- /dev/null +++ b/django_project/dashboard/src/reducers/viewTable.ts @@ -0,0 +1,52 @@ +import {createSlice, PayloadAction} from "@reduxjs/toolkit"; +import {getDefaultFilter, ViewsFilterInterface} from "../views/View/ViewsFilter" +import {RootState} from "../app/store"; + + +export interface TableState { + currentColumns: string[]; + currentFilters: ViewsFilterInterface; + availableFilters: ViewsFilterInterface; +} + +const initialState: TableState = { + currentColumns: [ + 'name', + 'description', + 'tags', + 'dataset', + 'min_privacy', + 'max_privacy', + 'status' + ], + currentFilters: getDefaultFilter(), + availableFilters: getDefaultFilter() +}; + +export const viewTableSlice = createSlice({ + name: 'viewTable', + initialState, + reducers: { + setCurrentColumns: (state, action: PayloadAction) => { + state.currentColumns = JSON.parse(action.payload) + }, + setCurrentFilters: (state, action: PayloadAction) => { + state.currentFilters = JSON.parse(action.payload) + }, + setAvailableFilters: (state, action: PayloadAction) => { + state.availableFilters = JSON.parse(action.payload) + } + } +}) + +export const { + setCurrentColumns, + setCurrentFilters, + setAvailableFilters +} = viewTableSlice.actions + +export default viewTableSlice.reducer; + +export const currentColumns = (state: RootState) => state.viewTable.currentColumns +export const currentFilters = (state: RootState) => state.viewTable.currentFilters +export const availableFilters = (state: RootState) => state.viewTable.availableFilters \ No newline at end of file diff --git a/django_project/dashboard/src/views/View/Views.tsx b/django_project/dashboard/src/views/View/Views.tsx index ef7bda89..e084bcc0 100644 --- a/django_project/dashboard/src/views/View/Views.tsx +++ b/django_project/dashboard/src/views/View/Views.tsx @@ -1,22 +1,69 @@ -import React, {useEffect, useState} from 'react'; -import {useNavigate, useSearchParams} from "react-router-dom"; -import View from "../../models/view"; -import List, {ActionDataInterface} from "../../components/List"; +import React, {Fragment, useCallback, useEffect, useRef, useState} from 'react'; +import {useNavigate} from "react-router-dom"; +import {TABLE_OFFSET_HEIGHT} from "../../components/List"; import Loading from "../../components/Loading"; -import {fetchData, postData} from "../../utils/Requests"; +import {postData} from "../../utils/Requests"; import {ViewEditRoute} from "../routes"; import AlertDialog from '../../components/AlertDialog' import {Button, Chip} from '@mui/material'; import Grid from '@mui/material/Grid'; -import EditIcon from "@mui/icons-material/Edit"; import DeleteIcon from "@mui/icons-material/Delete"; import MoreVertIcon from '@mui/icons-material/MoreVert'; import Popover from '@mui/material/Popover'; import Typography from '@mui/material/Typography'; +import PaginationInterface, {getDefaultPagination, rowsPerPageOptions} from "../../models/pagination"; +import axios from "axios"; +import IconButton from "@mui/material/IconButton"; +import {getDefaultFilter, ViewsFilterInterface} from "./ViewsFilter" +import ResizeTableEvent from "../../components/ResizeTableEvent"; +import MUIDataTable, {debounceSearchRender, MUISortOptions} from "mui-datatables"; +import FilterAlt from "@mui/icons-material/FilterAlt"; + +import { useAppSelector, useAppDispatch } from '../../app/hooks'; +import { + setAvailableFilters, + setCurrentColumns as setInitialColumns, + setCurrentFilters as setInitialFilters +} from "../../reducers/viewTable"; +import {RootState} from "../../app/store"; const VIEW_LIST_URL = '/api/view-list/' +const FILTER_VALUES_API_URL = '/api/view-filter/values/' const DELETE_VIEW_URL = '/api/delete-view' +const USER_COLUMNS = [ + 'id', + 'name', + 'description', + 'tags', + 'mode', + 'dataset', + 'is_default', + 'min_privacy', + 'max_privacy', + 'layer_tiles', + 'status', + 'uuid', + 'permissions' +] + +interface ViewTableRowInterface { + id: number, + name: string, + description: string, + tags: string[], + mode: string, + dataset: string, + is_default: string, + min_privacy: number, + max_privacy: number, + layer_tiles: string, + status: string, + uuid: string, + permissions: string[] +} + + const copyToClipboard = (value: string) => { navigator.clipboard.writeText(value) alert('Link copied') @@ -27,21 +74,21 @@ function ViewPopover(props: any) { return null } return ( - + - Mode: {props.view.mode} + Mode: {props.view.mode} - Is Default: {props.view.is_default} + Is Default: {props.view.is_default} - UUID: + UUID: - {props.view.uuid} + {props.view.uuid} - Layer Tiles: + Layer Tiles: @@ -51,85 +98,299 @@ function ViewPopover(props: any) { } export default function Views() { - const [searchParams] = useSearchParams() - const [views, setViews] = useState([]) - const [selectedView, setSelectedView] = useState(null) - const [loading, setLoading] = useState(false) + const dispatch = useAppDispatch() + const initialColumns = useAppSelector((state: RootState) => state.viewTable.currentColumns) + const initialFilters = useAppSelector((state: RootState) => state.viewTable.currentFilters) + const availableFilters = useAppSelector((state: RootState) => state.viewTable.availableFilters) + const [selectedView, setSelectedView] = useState>(null) const [confirmationOpen, setConfirmationOpen] = useState(false) const [confirmationText, setConfirmationText] = useState('') const [deleteButtonDisabled, setDeleteButtonDisabled] = useState(false) const navigate = useNavigate() const [anchorEl, setAnchorEl] = React.useState(null); - const handleCloseMoreInfo = () => { - setAnchorEl(null); - setSelectedView(null) - }; - const open = Boolean(anchorEl); - const id = open ? 'view-popover' : undefined; + const [loading, setLoading] = useState(true) + const [columns, setColumns] = useState([]) + const [data, setData] = useState([]) + const [userPermissions, setUserPermissions] = useState([]) + const [totalCount, setTotalCount] = useState(0) + const [pagination, setPagination] = useState(getDefaultPagination()) + const [filterValues, setFilterValues] = useState(availableFilters) + const [currentFilters, setCurrentFilters] = useState(initialFilters) + const axiosSource = useRef(null) + const newCancelToken = useCallback(() => { + axiosSource.current = axios.CancelToken.source(); + return axiosSource.current.token; + }, []) + const ref = useRef(null) + const [tableHeight, setTableHeight] = useState(0) + + const FilterIcon: any = FilterAlt + + const fetchFilterValues = async () => { + let filters = [] + filters.push(axios.get(`${FILTER_VALUES_API_URL}tags/`)) + filters.push(axios.get(`${FILTER_VALUES_API_URL}mode/`)) + filters.push(axios.get(`${FILTER_VALUES_API_URL}dataset/`)) + filters.push(axios.get(`${FILTER_VALUES_API_URL}is_default/`)) + filters.push(axios.get(`${FILTER_VALUES_API_URL}min_privacy/`)) + filters.push(axios.get(`${FILTER_VALUES_API_URL}max_privacy/`)) + let resultData = await Promise.all(filters) + let filterVals = { + 'tags': resultData[0].data, + 'mode': resultData[1].data, + 'dataset': resultData[2].data, + 'is_default': resultData[3].data, + 'min_privacy': resultData[4].data, + 'max_privacy': resultData[5].data, + 'search_text': '' + } + setFilterValues(filterVals) + dispatch(setAvailableFilters(JSON.stringify(filterVals))) + return filterVals + } - const fetchViews = () => { + const fetchViewList = () => { + if (axiosSource.current) axiosSource.current.cancel() + let cancelFetchToken = newCancelToken() setLoading(true) - fetchData(VIEW_LIST_URL).then( + let _additional_filters = '' + let sortBy = pagination.sortOrder.name ? pagination.sortOrder.name : '' + let sortDirection = pagination.sortOrder.direction ? pagination.sortOrder.direction : '' + + axios.post(`${VIEW_LIST_URL}?` + `page=${pagination.page + 1}&page_size=${pagination.rowsPerPage}` + + `&sort_by=${sortBy}&sort_direction=${sortDirection}` + + `${_additional_filters}`, + currentFilters, + { + cancelToken: cancelFetchToken + }).then( response => { - setViews(response.data) setLoading(false) + setData(response.data.results as ViewTableRowInterface[]) + setTotalCount(response.data.count) + setUserPermissions(response.data.permissions) } - ).catch(e => setLoading(false)) + ).catch(error => { + if (!axios.isCancel(error)) { + console.log(error) + setLoading(false) + if (error.response) { + if (error.response.status == 403) { + // TODO: use better way to handle 403 + navigate('/invalid_permission') + } + } + } + }) + } + + const getExistingFilterValue = (colName: string): string[] => { + let values: string[] = [] + switch (colName) { + case 'tags': + values = currentFilters.tags + break; + case 'mode': + values = currentFilters.mode + break; + case 'dataset': + values = currentFilters.dataset + break; + case 'is_default': + values = currentFilters.is_default + break; + case 'min_privacy': + values = currentFilters.min_privacy + break; + case 'max_privacy': + values = currentFilters.max_privacy + break; + default: + break; + } + return values } useEffect(() => { - fetchViews() - }, [searchParams]) + const fetchFilterValuesData = async () => { - const handleClose = () => { - setConfirmationOpen(false) - } + let filterVals: any = {} + if (filterValues.mode.length > 0 ) { + filterVals = filterValues + } else { + filterVals = await fetchFilterValues() + } + let _init_columns = USER_COLUMNS + let _columns = _init_columns.map((columnName) => { + let _options: any = { + name: columnName, + label: columnName.charAt(0).toUpperCase() + columnName.slice(1).replaceAll('_', ' '), + options: { + display: initialColumns.includes(columnName), + sort: !['tags', 'permissions', 'status', 'mode'].includes(columnName) + } + } + if (columnName == 'tags') { + _options['options']['filterType'] = 'multiselect' + _options['options']['filter'] = true + _options['options']['customBodyRender'] = (value: any, tableMeta: any) => { + return
+ {value.map((tag: any, index: number) => )} +
+ } + } + if (['tags', 'mode', 'dataset', 'is_default', 'min_privacy', 'max_privacy'].includes(columnName)) { + // set filter values in dropdown + _options.options.filterOptions = { + names: filterVals[columnName] + } + _options.options.filter = true + } else { + _options.options.filter = false + } + if (['tags', 'mode', 'dataset', 'is_default', 'min_privacy', 'max_privacy'].includes(columnName)) { + // set existing filter values + _options.options.filterList = getExistingFilterValue(columnName) + } + return _options + }) + _columns.push({ + name: '', + options: { + customBodyRender: (value: any, tableMeta: any, updateValue: any) => { + let rowData = tableMeta.rowData + return ( +
+ ) => { + let obj: any = {} + USER_COLUMNS.forEach((element, index) => { + obj[element] = rowData[index]; + }); + setSelectedView(obj) + setAnchorEl(event.currentTarget); + }} + className='' + > + + - const actionDeleteButton: ActionDataInterface = { - field: '', - name: 'Delete', - getName: (data: any) => { - if (!data.permissions.includes('Own')) { - return 'You are not owner of this view' - } else if (data.is_default === 'Yes') { - return 'Cannot remove default view' + { + setSelectedView(rowData) + setConfirmationText( + `Are you sure you want to delete ${rowData[1]}?`) + setConfirmationOpen(true) + }} + className='' + > + + +
+ ) + }, + filter: false } - return 'Delete' - }, - color: 'error', - icon: , - isDisabled: (data: any) => { - return !data.permissions.includes('Own') || data.is_default === 'Yes' - }, - onClick: (data: View) => { - setSelectedView(data) - setConfirmationText( - `Are you sure you want to delete ${data.name}?`) - setConfirmationOpen(true) + }) + setColumns(_columns) + dispatch(setInitialColumns(JSON.stringify(_columns.map))) + } + fetchFilterValuesData() + }, [pagination, currentFilters]) + + useEffect(() => { + fetchViewList() + }, [pagination, filterValues, currentFilters]) + + const onTableChangeState = (action: string, tableState: any) => { + switch (action) { + case 'changePage': + setPagination({ + ...pagination, + page: tableState.page + }) + break; + case 'sort': + setPagination({ + ...pagination, + page: 0, + sortOrder: tableState.sortOrder + }) + break; + case 'changeRowsPerPage': + setPagination({ + ...pagination, + page: 0, + rowsPerPage: tableState.rowsPerPage + }) + break; + default: } } - const actionMoreInfoButton: ActionDataInterface = { - field: '', - name: 'More Info', - color: 'primary', - icon: , - onClick: (data: View, event?: React.MouseEvent) => { - setSelectedView(data) - setAnchorEl(event.currentTarget); + const handleFilterSubmit = (applyFilters: any) => { + let filterList = applyFilters() + let filter = getDefaultFilter() + type Column = { + name: string, + label: string, + options: any + } + for (let idx in filterList) { + let col: Column = columns[idx] + if (!col.options.filter) + continue + if (filterList[idx] && filterList[idx].length) { + const key = col.name as string + filter[key as keyof ViewsFilterInterface] = filterList[idx] + } } + setCurrentFilters({...filter, 'search_text': currentFilters['search_text']}) + dispatch(setInitialFilters(JSON.stringify({...filter, 'search_text': currentFilters['search_text']}))) + } + + const handleSearchOnChange = (search_text: string) => { + setPagination({ + ...pagination, + page: 0, + sortOrder: {} + }) + setCurrentFilters({...currentFilters, 'search_text': search_text}) + dispatch(setInitialFilters(JSON.stringify({...currentFilters, 'search_text': search_text}))) + } + + const handleCloseMoreInfo = () => { + setAnchorEl(null); + setSelectedView(null) + }; + + const open = Boolean(anchorEl); + const id = open ? 'view-popover' : undefined; + + const handleClose = () => { + setConfirmationOpen(false) } const handleDeleteClick = () => { setDeleteButtonDisabled(true) postData( - `${DELETE_VIEW_URL}/${selectedView.id}`, {} + `${DELETE_VIEW_URL}/${selectedView['0']}`, {} ).then( response => { setDeleteButtonDisabled(false) - fetchViews() + fetchViewList() setConfirmationOpen(false) } ).catch(error => { @@ -138,43 +399,85 @@ export default function Views() { }) } - const handleRowClick = (rowData: string[], rowMeta: { dataIndex: number, rowIndex: number }) => { - navigate(ViewEditRoute.path + `?id=${rowData[0]}`) - } - return ( -
+
{ loading ? : - { - return
- {value.map((tag: any, index:number) => )} -
- }} - }} - excludedColumns={['permissions', 'uuid', 'layer_tiles', 'mode', 'is_default']} - /> + +
+ setTableHeight(0)} + onResize={(clientHeight: number) => setTableHeight(clientHeight - TABLE_OFFSET_HEIGHT)}/> +
+ { + navigate(ViewEditRoute.path + `?id=${rowData[0]}`) + }, + onTableChange: (action: string, tableState: any) => onTableChangeState(action, tableState), + customSearchRender: debounceSearchRender(500), + selectableRows: 'none', + onRowSelectionChange: (currentRowsSelected: Array, allRowsSelected: Array, rowsSelected: Array) => { + console.log(currentRowsSelected) + console.log(allRowsSelected) + console.log(rowsSelected) + }, + tableBodyHeight: `${tableHeight}px`, + tableBodyMaxHeight: `${tableHeight}px`, + textLabels: { + body: { + noMatch: loading ? + : + 'Sorry, there is no matching data to display', + }, + }, + onSearchChange: (searchText: string) => { + handleSearchOnChange(searchText) + }, + customFilterDialogFooter: (currentFilterList, applyNewFilters) => { + return ( +
+ +
+ ); + }, + onFilterChange: (column, filterList, type) => { + var newFilters = () => (filterList) + handleFilterSubmit(newFilters) + }, + searchText: currentFilters.search_text, + searchOpen: (currentFilters.search_text != null && currentFilters.search_text.length > 0), + filter: true, + filterType: 'multiselect', + confirmFilters: true + }} + components={{ + icons: { + FilterIcon + } + }} + /> +
+
+ } - +
) -} +} \ No newline at end of file diff --git a/django_project/dashboard/src/views/View/ViewsFilter.ts b/django_project/dashboard/src/views/View/ViewsFilter.ts new file mode 100644 index 00000000..f1b2fc40 --- /dev/null +++ b/django_project/dashboard/src/views/View/ViewsFilter.ts @@ -0,0 +1,21 @@ +export interface ViewsFilterInterface { + tags: string[], + mode: string[], + dataset: string[], + is_default: string[], + min_privacy: string[], + max_privacy: string[], + search_text: string, +} + +export function getDefaultFilter():ViewsFilterInterface { + return { + tags: [], + mode: [], + dataset: [], + is_default: [], + min_privacy: [], + max_privacy: [], + search_text: '', + } +} diff --git a/django_project/dashboard/tests/test_api_views.py b/django_project/dashboard/tests/test_api_views.py index 422b311f..a46b5f36 100644 --- a/django_project/dashboard/tests/test_api_views.py +++ b/django_project/dashboard/tests/test_api_views.py @@ -1365,31 +1365,6 @@ def test_create_dataset(self): ) self.assertEqual(response.status_code, 400) - def test_list_views(self): - creator = UserF.create() - dataset_1 = DatasetViewF.create( - created_by=creator - ) - grant_dataset_manager(dataset_1.dataset, creator) - request = self.factory.get( - reverse('view-list') - ) - request.user = self.superuser - list_view = ViewList.as_view() - response = list_view(request) - self.assertEqual(response.status_code, 200) - self.assertEqual(response.data[0].get('id'), dataset_1.id) - - request = self.factory.get( - reverse('view-list') - ) - - request.user = creator - list_view = ViewList.as_view() - response = list_view(request) - self.assertEqual(response.status_code, 200) - self.assertEqual(response.data[0].get('id'), dataset_1.id) - def test_delete_view(self): # Test no permission user = UserF.create() diff --git a/django_project/dashboard/tests/test_view_filter_value.py b/django_project/dashboard/tests/test_view_filter_value.py new file mode 100644 index 00000000..1ae255c2 --- /dev/null +++ b/django_project/dashboard/tests/test_view_filter_value.py @@ -0,0 +1,81 @@ +__author__ = 'zakki@kartoza.com' +__date__ = '31/07/23' +__copyright__ = ('Copyright 2023, Unicef') + +from django.test import TestCase +from django.urls import reverse +from rest_framework.test import APIRequestFactory + +from dashboard.api_views.views import ( + ViewFilterValue +) +from georepo.tests.model_factories import ( + UserF, DatasetF, DatasetViewF, ModuleF +) +from georepo.utils.permission import ( + grant_dataset_manager +) + + +class TestViewFilterValue(TestCase): + + def setUp(self) -> None: + self.factory = APIRequestFactory() + self.module = ModuleF.create( + name='Admin Boundaries' + ) + self.dataset = DatasetF.create( + module=self.module, + generate_adm0_default_views=True + ) + self.superuser = UserF.create(is_superuser=True) + self.creator = UserF.create() + self.dataset_view_1 = DatasetViewF.create( + created_by=self.creator + ) + grant_dataset_manager(self.dataset_view_1.dataset, self.creator) + + def test_list_dataset(self): + request = self.factory.get( + reverse('view-filter-value', kwargs={'criteria': 'dataset'}) + ) + request.user = self.superuser + list_view = ViewFilterValue.as_view() + response = list_view(request, 'dataset') + self.assertTrue(response.data, [self.dataset_view_1.dataset]) + + def test_list_mode(self): + request = self.factory.get( + reverse('view-filter-value', kwargs={'criteria': 'mode'}) + ) + request.user = self.superuser + list_view = ViewFilterValue.as_view() + response = list_view(request, 'mode') + self.assertTrue(response.data, ['Static', 'Dynamic']) + + def test_list_is_default(self): + request = self.factory.get( + reverse('view-filter-value', kwargs={'criteria': 'is_default'}) + ) + request.user = self.superuser + list_view = ViewFilterValue.as_view() + response = list_view(request, 'is_default') + self.assertTrue(response.data, ['Yes', 'No']) + + def test_list_max_privacy(self): + request = self.factory.get( + reverse('view-filter-value', kwargs={'criteria': 'max_privacy'}) + ) + request.user = self.superuser + list_view = ViewFilterValue.as_view() + response = list_view(request, 'max_privacy') + self.assertTrue(response.data, [self.dataset_view_1.max_privacy_level]) + + def test_list_min_privacy(self): + request = self.factory.get( + reverse('view-filter-value', kwargs={'criteria': 'min_privacy'}) + ) + request.user = self.superuser + list_view = ViewFilterValue.as_view() + response = list_view(request, 'min_privacy') + self.assertTrue(response.data, [self.dataset_view_1.max_privacy_level]) diff --git a/django_project/dashboard/tests/test_view_list.py b/django_project/dashboard/tests/test_view_list.py new file mode 100644 index 00000000..556e6434 --- /dev/null +++ b/django_project/dashboard/tests/test_view_list.py @@ -0,0 +1,144 @@ +__author__ = 'zakki@kartoza.com' +__date__ = '31/07/23' +__copyright__ = ('Copyright 2023, Unicef') + +import urllib.parse + +from django.test import TestCase +from django.urls import reverse +from rest_framework.test import APIRequestFactory + +from dashboard.api_views.views import ( + ViewList +) +from georepo.tests.model_factories import ( + UserF, DatasetF, DatasetViewF, ModuleF +) +from georepo.utils.permission import ( + grant_dataset_manager +) + + +class TestViewList(TestCase): + + def setUp(self) -> None: + self.factory = APIRequestFactory() + self.module = ModuleF.create( + name='Admin Boundaries' + ) + self.dataset = DatasetF.create( + module=self.module, + generate_adm0_default_views=True + ) + self.superuser = UserF.create(is_superuser=True) + self.creator = UserF.create() + self.dataset_view_1 = DatasetViewF.create( + created_by=self.creator + ) + grant_dataset_manager(self.dataset_view_1.dataset, self.creator) + + def test_list_views(self): + request = self.factory.post( + reverse('view-list') + ) + request.user = self.superuser + list_view = ViewList.as_view() + response = list_view(request) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['count'], 1) + self.assertEqual(response.data['page'], 1) + self.assertEqual(response.data['total_page'], 1) + self.assertEqual(response.data['results'][0].get('id'), self.dataset_view_1.id) + + request = self.factory.post( + reverse('view-list') + ) + + request.user = self.creator + list_view = ViewList.as_view() + response = list_view(request) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['count'], 1) + self.assertEqual(response.data['page'], 1) + self.assertEqual(response.data['total_page'], 1) + self.assertEqual(response.data['results'][0].get('id'), self.dataset_view_1.id) + + def test_sort(self): + dataset_view_2 = DatasetViewF.create( + created_by=self.creator + ) + grant_dataset_manager(dataset_view_2.dataset, self.creator) + query_params = { + 'sort_by': 'id', + 'sort_direction': 'desc' + } + request = self.factory.post( + f"{reverse('view-list')}?{urllib.parse.urlencode(query_params)}" + ) + request.user = self.superuser + list_view = ViewList.as_view() + response = list_view(request) + self.assertEqual(response.data['count'], 2) + self.assertEqual(response.data['page'], 1) + self.assertEqual(response.data['total_page'], 1) + self.assertEqual(response.data['results'][0].get('id'), dataset_view_2.id) + self.assertEqual(response.data['results'][1].get('id'), self.dataset_view_1.id) + + def test_pagination(self): + dataset_view_2 = DatasetViewF.create( + created_by=self.creator + ) + grant_dataset_manager(dataset_view_2.dataset, self.creator) + query_params = { + 'page': 2, + 'page_size': 1 + } + request = self.factory.post( + f"{reverse('view-list')}?{urllib.parse.urlencode(query_params)}" + ) + request.user = self.superuser + list_view = ViewList.as_view() + response = list_view(request) + self.assertEqual(response.data['page'], 2) + self.assertEqual(response.data['total_page'], 2) + self.assertEqual(len(response.data['results']), 1) + self.assertEqual(response.data['results'][0].get('id'), dataset_view_2.id) + + def test_search(self): + dataset_view_2 = DatasetViewF.create( + created_by=self.creator + ) + grant_dataset_manager(dataset_view_2.dataset, self.creator) + request = self.factory.post( + reverse('view-list'), + { + 'search_text': dataset_view_2.description + } + ) + request.user = self.superuser + list_view = ViewList.as_view() + response = list_view(request) + self.assertEqual(response.data['count'], 1) + self.assertEqual(response.data['page'], 1) + self.assertEqual(response.data['total_page'], 1) + self.assertEqual(response.data['results'][0].get('id'), dataset_view_2.id) + + def test_filter(self): + dataset_view_2 = DatasetViewF.create( + created_by=self.creator + ) + grant_dataset_manager(dataset_view_2.dataset, self.creator) + request = self.factory.post( + reverse('view-list'), + { + 'dataset': [dataset_view_2.dataset.label], + 'min_privacy': [4], + } + ) + request.user = self.superuser + list_view = ViewList.as_view() + response = list_view(request) + self.assertEqual(response.data['count'], 1) + self.assertEqual(response.data['page'], 1) + self.assertEqual(response.data['total_page'], 1) + self.assertEqual(response.data['results'][0].get('id'), dataset_view_2.id) diff --git a/django_project/dashboard/urls.py b/django_project/dashboard/urls.py index 9caee397..8cc69b4a 100644 --- a/django_project/dashboard/urls.py +++ b/django_project/dashboard/urls.py @@ -97,7 +97,7 @@ CreateNewView, ViewList, DeleteView, ViewDetail, UpdateView, QueryViewCheck, SQLColumnsTablesList, QueryViewPreview, GetViewTags, - DownloadView + DownloadView, ViewFilterValue ) from dashboard.api_views.tiling_config import ( FetchDatasetTilingConfig, UpdateDatasetTilingConfig, @@ -490,6 +490,10 @@ re_path(r'api/view-list/?$', ViewList.as_view(), name='view-list'), + re_path(r'^api/view-filter/values/' + r'(?P\w+)/?$', + ViewFilterValue.as_view(), + name='view-filter-value'), re_path(r'api/tag-list/?$', GetViewTags.as_view(), name='get-tag-list'), diff --git a/django_project/georepo/api_views/dataset_view.py b/django_project/georepo/api_views/dataset_view.py index d220648f..7192db73 100644 --- a/django_project/georepo/api_views/dataset_view.py +++ b/django_project/georepo/api_views/dataset_view.py @@ -212,7 +212,7 @@ def get_response_data(self, request, *args, **kwargs): type=openapi.TYPE_INTEGER ), 'results': openapi.Schema( - title='Dtaset view list', + title='Dataset view list', type=openapi.TYPE_ARRAY, items=openapi.Items( type=openapi.TYPE_OBJECT, diff --git a/django_project/georepo/models/dataset_view.py b/django_project/georepo/models/dataset_view.py index 16acadda..78bf2099 100644 --- a/django_project/georepo/models/dataset_view.py +++ b/django_project/georepo/models/dataset_view.py @@ -209,6 +209,10 @@ def save(self, *args, **kwargs): def __str__(self): return self.name + @classmethod + def get_fields(cls): + return cls._meta.fields + class DatasetViewUserObjectPermission(UserObjectPermissionBase): content_object = models.ForeignKey(DatasetView, on_delete=models.CASCADE) diff --git a/django_project/georepo/utils/permission.py b/django_project/georepo/utils/permission.py index 2e313394..3154ee24 100644 --- a/django_project/georepo/utils/permission.py +++ b/django_project/georepo/utils/permission.py @@ -314,6 +314,41 @@ def get_dataset_view_privacy_level_from_perms(permissions, dataset_perms): return 0 +def get_views_for_user(user): + datasets = Dataset.objects.all() + datasets = get_dataset_for_user(user, datasets) + views_querysets = DatasetView.objects.none() + user_privacy_levels = {} + for dataset in datasets: + views = DatasetView.objects.filter( + dataset=dataset + ) + views, _ = get_dataset_views_for_user( + user, + dataset, + views + ) + views_querysets = views_querysets.union(views) + privacy_level = get_view_permission_privacy_level( + user, + dataset + ) + user_privacy_levels[dataset.id] = privacy_level + # include external user + external_views = DatasetView.objects.all() + external_views = get_objects_for_user( + user, + EXTERNAL_READ_VIEW_PERMISSION_LIST, + klass=external_views, + use_groups=True, + any_perm=True, + accept_global_perms=False + ) + views_querysets = views_querysets.union(external_views) + views_querysets = views_querysets.order_by('created_at') + return user_privacy_levels, views_querysets + + def get_dataset_for_user(user, queryset, use_groups=True): """ Return queryset for dataset with filter user can access From 2b46a13256d38a5691983b8329987b4a62e87d9c Mon Sep 17 00:00:00 2001 From: Zakki Date: Tue, 1 Aug 2023 14:31:34 +0700 Subject: [PATCH 2/4] Fix flake8 test --- django_project/dashboard/api_views/dataset.py | 37 +++++++------------ django_project/dashboard/api_views/views.py | 8 +++- .../dashboard/src/views/View/Views.tsx | 1 + .../dashboard/tests/test_api_views.py | 2 +- .../dashboard/tests/test_view_list.py | 35 ++++++++++++++---- 5 files changed, 50 insertions(+), 33 deletions(-) diff --git a/django_project/dashboard/api_views/dataset.py b/django_project/dashboard/api_views/dataset.py index b1abfa40..ff125b1b 100644 --- a/django_project/dashboard/api_views/dataset.py +++ b/django_project/dashboard/api_views/dataset.py @@ -115,29 +115,20 @@ def get_sort_attribute(self, sort_by, sort_direction): return None, None if sort_direction not in ['asc', 'desc']: return None, None - match sort_by: - case 'id': - return 'gg.id', sort_direction - case 'country': - return 'parent_0.label', sort_direction - case 'level': - return 'gg.level', sort_direction - case 'type': - return 'ge.label', sort_direction - case 'name': - return 'gg.label', sort_direction - case 'default_code': - return 'gg.internal_code', sort_direction - case 'code': - return 'gg.unique_code', sort_direction - case 'cucode': - return 'gg.concept_ucode', sort_direction - case 'updated': - return 'gg.start_date', sort_direction - case 'rev': - return 'gg.revision_number', sort_direction - case 'status': - return 'gg.is_approved', sort_direction + field_mapping = { + 'id': 'gg.id', + 'country': 'parent_0.label', + 'level': 'gg.level', + 'type': 'gg.type', + 'name': 'gg.label', + 'default_code': 'gg.internal_code', + 'code': 'gg.unique_code', + 'cucode': 'gg.concept_code', + 'updated': 'gg.start_date', + 'rev': 'gg.revision_number', + 'status': 'gg.is_approved' + } + return field_mapping[sort_by], sort_direction def do_run_query( self, diff --git a/django_project/dashboard/api_views/views.py b/django_project/dashboard/api_views/views.py index 36b6d67c..a8c6ce14 100644 --- a/django_project/dashboard/api_views/views.py +++ b/django_project/dashboard/api_views/views.py @@ -256,7 +256,8 @@ def _search_queryset(self, queryset, request): return queryset char_fields = [ field.name for field in DatasetView.get_fields() if - field.get_internal_type() in ['UUIDField', 'CharField', 'TextField'] + field.get_internal_type() in + ['UUIDField', 'CharField', 'TextField'] ] q_args = [ Q(**{f"{field}__icontains": search_text}) for field in char_fields @@ -279,7 +280,10 @@ def _sort_queryset(self, queryset, request): return queryset def post(self, *args, **kwargs): - user_privacy_levels, views_querysets = get_views_for_user(self.request.user) + ( + user_privacy_levels, + views_querysets + ) = get_views_for_user(self.request.user) # It seems we cannot use values_list on views_queryset views_querysets = DatasetView.objects.\ filter(id__in=[v.id for v in views_querysets]) diff --git a/django_project/dashboard/src/views/View/Views.tsx b/django_project/dashboard/src/views/View/Views.tsx index e084bcc0..18ed96fe 100644 --- a/django_project/dashboard/src/views/View/Views.tsx +++ b/django_project/dashboard/src/views/View/Views.tsx @@ -270,6 +270,7 @@ export default function Views() { disabled={false} color='primary' onClick={(event: React.MouseEvent) => { + event.stopPropagation(); let obj: any = {} USER_COLUMNS.forEach((element, index) => { obj[element] = rowData[index]; diff --git a/django_project/dashboard/tests/test_api_views.py b/django_project/dashboard/tests/test_api_views.py index a46b5f36..3036a095 100644 --- a/django_project/dashboard/tests/test_api_views.py +++ b/django_project/dashboard/tests/test_api_views.py @@ -50,7 +50,7 @@ DatasetAdminLevelName, BoundaryType ) from dashboard.api_views.views import ( - CreateNewView, ViewList, DeleteView, UpdateView, ViewDetail, + CreateNewView, DeleteView, UpdateView, ViewDetail, DownloadView ) from dashboard.models import ( diff --git a/django_project/dashboard/tests/test_view_list.py b/django_project/dashboard/tests/test_view_list.py index 556e6434..b68d9c68 100644 --- a/django_project/dashboard/tests/test_view_list.py +++ b/django_project/dashboard/tests/test_view_list.py @@ -48,7 +48,10 @@ def test_list_views(self): self.assertEqual(response.data['count'], 1) self.assertEqual(response.data['page'], 1) self.assertEqual(response.data['total_page'], 1) - self.assertEqual(response.data['results'][0].get('id'), self.dataset_view_1.id) + self.assertEqual( + response.data['results'][0].get('id'), + self.dataset_view_1.id + ) request = self.factory.post( reverse('view-list') @@ -61,7 +64,10 @@ def test_list_views(self): self.assertEqual(response.data['count'], 1) self.assertEqual(response.data['page'], 1) self.assertEqual(response.data['total_page'], 1) - self.assertEqual(response.data['results'][0].get('id'), self.dataset_view_1.id) + self.assertEqual( + response.data['results'][0].get('id'), + self.dataset_view_1.id + ) def test_sort(self): dataset_view_2 = DatasetViewF.create( @@ -81,8 +87,14 @@ def test_sort(self): self.assertEqual(response.data['count'], 2) self.assertEqual(response.data['page'], 1) self.assertEqual(response.data['total_page'], 1) - self.assertEqual(response.data['results'][0].get('id'), dataset_view_2.id) - self.assertEqual(response.data['results'][1].get('id'), self.dataset_view_1.id) + self.assertEqual( + response.data['results'][0].get('id'), + dataset_view_2.id + ) + self.assertEqual( + response.data['results'][1].get('id'), + self.dataset_view_1.id + ) def test_pagination(self): dataset_view_2 = DatasetViewF.create( @@ -102,7 +114,10 @@ def test_pagination(self): self.assertEqual(response.data['page'], 2) self.assertEqual(response.data['total_page'], 2) self.assertEqual(len(response.data['results']), 1) - self.assertEqual(response.data['results'][0].get('id'), dataset_view_2.id) + self.assertEqual( + response.data['results'][0].get('id'), + dataset_view_2.id + ) def test_search(self): dataset_view_2 = DatasetViewF.create( @@ -121,7 +136,10 @@ def test_search(self): self.assertEqual(response.data['count'], 1) self.assertEqual(response.data['page'], 1) self.assertEqual(response.data['total_page'], 1) - self.assertEqual(response.data['results'][0].get('id'), dataset_view_2.id) + self.assertEqual( + response.data['results'][0].get('id'), + dataset_view_2.id + ) def test_filter(self): dataset_view_2 = DatasetViewF.create( @@ -141,4 +159,7 @@ def test_filter(self): self.assertEqual(response.data['count'], 1) self.assertEqual(response.data['page'], 1) self.assertEqual(response.data['total_page'], 1) - self.assertEqual(response.data['results'][0].get('id'), dataset_view_2.id) + self.assertEqual( + response.data['results'][0].get('id'), + dataset_view_2.id + ) From 8ba680ca0845d672d2b2ca61e3635e7c8d596dfa Mon Sep 17 00:00:00 2001 From: Zakki Date: Wed, 2 Aug 2023 10:45:31 +0700 Subject: [PATCH 3/4] Use version 6.0.0 in flake8 --- deployment/docker/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/docker/requirements-dev.txt b/deployment/docker/requirements-dev.txt index 0dba7b10..0817083f 100644 --- a/deployment/docker/requirements-dev.txt +++ b/deployment/docker/requirements-dev.txt @@ -2,7 +2,7 @@ django-nose coverage pep8 pylint -flake8 +flake8==6.0.0 factory_boy django-devserver From 0b3b8df1934e08059fdd1f23c711974ae9473883 Mon Sep 17 00:00:00 2001 From: Zakki Date: Wed, 2 Aug 2023 22:23:39 +0700 Subject: [PATCH 4/4] Address flake8 feedback --- django_project/dashboard/api_views/views.py | 12 ++++++++---- .../dashboard/tests/test_view_filter_value.py | 10 ++++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/django_project/dashboard/api_views/views.py b/django_project/dashboard/api_views/views.py index 38c6e0e5..7df6c90b 100644 --- a/django_project/dashboard/api_views/views.py +++ b/django_project/dashboard/api_views/views.py @@ -358,12 +358,16 @@ def fetch_is_default(self): ] def fetch_min_privacy(self): - return list(self.views_querysets.order_by().\ - values_list('min_privacy_level', flat=True).distinct()) + return list( + self.views_querysets.order_by(). + values_list('min_privacy_level', flat=True).distinct() + ) def fetch_max_privacy(self): - return list(self.views_querysets.order_by().\ - values_list('max_privacy_level', flat=True).distinct()) + return list( + self.views_querysets.order_by(). + values_list('max_privacy_level', flat=True).distinct() + ) def get(self, request, criteria, *args, **kwargs): self.views_querysets = self.get_user_views() diff --git a/django_project/dashboard/tests/test_view_filter_value.py b/django_project/dashboard/tests/test_view_filter_value.py index be410156..2b7ac3fc 100644 --- a/django_project/dashboard/tests/test_view_filter_value.py +++ b/django_project/dashboard/tests/test_view_filter_value.py @@ -69,7 +69,10 @@ def test_list_max_privacy(self): request.user = self.superuser list_view = ViewFilterValue.as_view() response = list_view(request, 'max_privacy') - self.assertEquals(response.data, [self.dataset_view_1.max_privacy_level]) + self.assertEquals( + response.data, + [self.dataset_view_1.max_privacy_level] + ) def test_list_min_privacy(self): request = self.factory.get( @@ -78,4 +81,7 @@ def test_list_min_privacy(self): request.user = self.superuser list_view = ViewFilterValue.as_view() response = list_view(request, 'min_privacy') - self.assertEquals(response.data, [self.dataset_view_1.max_privacy_level]) + self.assertEquals( + response.data, + [self.dataset_view_1.max_privacy_level] + )