update export file access

frontend/src/components/dialog/sysadmin-dialog/sysadmin-logs-export-excel-dialog.js

@@ -2,6 +2,8 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import { Button, Modal, ModalHeader, ModalBody, ModalFooter, FormGroup, Label, Input, Alert } from 'reactstrap';
 import { gettext, siteRoot } from '../../../utils/constants';
+import { seafileAPI } from '../../../utils/seafile-api';
+import toaster from '../../../components/toast';
 import moment from 'moment';
 
 class LogsExportExcelDialog extends React.Component {
@@ -13,6 +15,7 @@ class LogsExportExcelDialog extends React.Component {
       startDateStr: '',
       endDateStr: '',
       errMsg: '',
+      taskId: '',
     };
   }
 
@@ -28,7 +31,8 @@ class LogsExportExcelDialog extends React.Component {
         url += 'sys/loginadmin/export-excel/';
         break;
       case 'fileAccess':
-        url += 'sys/log/fileaudit/export-excel/';
+        // url += 'sys/log/fileaudit/export-excel/';
+        this.exportFileAccess();
         break;
       case 'fileUpdate':
         url += 'sys/log/fileupdate/export-excel/';
@@ -37,8 +41,43 @@ class LogsExportExcelDialog extends React.Component {
         url += 'sys/log/permaudit/export-excel/';
         break;
     }
-    location.href = url + '?start=' + startDateStr + '&end=' + endDateStr;
-    this.props.toggle();
+    if (this.props.logType != 'fileAccess'){
+      location.href = url + '?start=' + startDateStr + '&end=' + endDateStr;
+      this.props.toggle();
+    }
+  };
+
+  exportFileAccess = () =>{
+    let { startDateStr, endDateStr } = this.state;
+    let task_id = '';
+    seafileAPI.sysAdminExportFileAccessExcel(startDateStr, endDateStr).then(res => {
+      task_id = res.data.task_id;
+      this.setState({
+        taskId: task_id
+      });
+      this.props.toggle();
+      return seafileAPI.queryAsyncOperationExportExcel(task_id);
+    }).then(res => {
+      if (res.data.is_finished === true){
+        location.href = siteRoot + 'sys/log/fileaudit/export-excel/?task_id=' + task_id;
+      } else {
+        this.timer = setInterval(() => {
+          seafileAPI.queryAsyncOperationExportExcel(task_id).then(res => {
+            if (res.data.is_finished === true){
+              this.setState({isFinished: true});
+              clearInterval(this.timer);
+              location.href = siteRoot + 'sys/log/fileaudit/export-excel/?task_id=' + task_id;
+            }
+          }).catch(err => {
+            if (this.state.isFinished === false){
+              clearInterval(this.timer);
+              toaster.danger(gettext('Failed to export. Please check whether the size of table attachments exceeds the limit.'));
+
+            }
+          });
+        }, 1000);
+      }
+    });
   };
 
   isValidDateStr = () => {

seahub/api2/endpoints/admin/file_audit.py

@@ -10,7 +10,7 @@
 from seaserv import seafile_api
 
 from seahub.api2.endpoints.utils import check_time_period_valid, \
-    get_log_events_by_type_and_time
+    get_log_events_by_type_and_time, export_file_audit_to_excel
 
 from seahub.api2.authentication import TokenAuthentication
 from seahub.api2.throttling import UserRateThrottle
@@ -22,6 +22,11 @@
 
 from seahub.utils.timeutils import datetime_to_isoformat_timestr
 
+from django.contrib import messages
+from seahub.settings import SITE_ROOT
+from seahub.utils import is_pro_version, query_fileaudit_export_status
+import json
+
 logger = logging.getLogger(__name__)
 
 
@@ -94,3 +99,54 @@ def get(self, request):
                 })
 
         return Response(result)
+
+
+class FileAuditExport(APIView):
+    authentication_classes = (TokenAuthentication, SessionAuthentication)
+    permission_classes = (IsAdminUser, IsProVersion)
+    throttle_classes = (UserRateThrottle,)
+
+    def get(self, request):
+        next_page = request.headers.get('referer', None)
+        if not next_page:
+            next_page = SITE_ROOT
+
+        if not is_pro_version():
+            messages.error(request, _('Failed to export excel, this feature is only in professional version.'))
+            return HttpResponseRedirect(next_page)
+
+        start = request.GET.get('start', None)
+        end = request.GET.get('end', None)
+        if not check_time_period_valid(start, end):
+            messages.error(request, _('Failed to export excel, invalid start or end date'))
+            return HttpResponseRedirect(next_page)
+        task_id = export_file_audit_to_excel(start, end)
+        return Response({'task_id': task_id}, status=status.HTTP_200_OK)
+
+
+class FileAuditExportStatus(APIView):
+    authentication_classes = (TokenAuthentication, SessionAuthentication)
+    permission_classes = (IsAdminUser, IsProVersion)
+    throttle_classes = (UserRateThrottle,)
+
+    def get(self, request):
+        """
+        Get task status by task id
+        :param request:
+        :return:
+        """
+        task_id = request.GET.get('task_id', '')
+        if not task_id:
+            error_msg = 'task_id invalid.'
+            return api_error(status.HTTP_400_BAD_REQUEST, error_msg)
+
+        resp = query_fileaudit_export_status(task_id)
+        if resp.status_code == 500:
+            logger.error('seafile io query status error: %s, %s' % (task_id, resp.text))
+            return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, 'Internal Server Error')
+
+        if not resp.status_code == 200:
+            return api_error(resp.status_code, resp.content)
+
+        is_finished = json.loads(resp.content)['is_finished']
+        return Response({'is_finished': is_finished})

seahub/api2/endpoints/utils.py

@@ -16,7 +16,8 @@
 
 from seahub.api2.utils import api_error
 from seahub.base.templatetags.seahub_tags import email2nickname, email2contact_email
-from seahub.utils import get_log_events_by_time, is_pro_version, is_org_context
+from seahub.utils import get_log_events_by_time, is_pro_version, is_org_context, \
+                        get_file_audit_task
 
 from seahub.settings import SEADOC_PRIVATE_KEY, FILE_CONVERTER_SERVER_URL
 
@@ -282,3 +283,16 @@ def sdoc_export_to_docx(path, username, doc_uuid, download_token,
     resp = requests.post(url, json=params, headers=headers, timeout=30)
 
     return resp
+
+
+def export_file_audit_to_excel(start, end):
+    start_struct_time = datetime.datetime.strptime(start, "%Y-%m-%d")
+    start_timestamp = time.mktime(start_struct_time.timetuple())
+
+    end_struct_time = datetime.datetime.strptime(end, "%Y-%m-%d")
+    end_timestamp = time.mktime(end_struct_time.timetuple())
+    end_timestamp += 24 * 60 * 60
+
+    task_id = get_file_audit_task(start_timestamp, end_timestamp)
+    task_id = task_id if task_id else None
+    return task_id

seahub/sysadmin_extra/views.py

@@ -6,12 +6,13 @@
 
 from django.utils.translation import gettext as _
 from django.contrib import messages
-from django.http import HttpResponse, HttpResponseRedirect, Http404
+from django.http import HttpResponse, HttpResponseRedirect, Http404, FileResponse
 from django.core.exceptions import ValidationError
-
+from urllib.parse import quote
 from seahub.api2.endpoints.utils import check_time_period_valid, \
     get_log_events_by_type_and_time
 
+from seahub.api2.utils import api_error
 from seahub.base.decorators import sys_staff_required
 from seahub.auth.decorators import login_required
 from seahub.sysadmin_extra.models import UserLoginLog
@@ -77,56 +78,19 @@ def sys_login_admin_export_excel(request):
 def sys_log_file_audit_export_excel(request):
     """ Export file access logs to excel.
     """
-    next_page = request.headers.get('referer', None)
-    if not next_page:
-        next_page = SITE_ROOT
-
-    if not is_pro_version():
-        messages.error(request, _('Failed to export excel, this feature is only in professional version.'))
-        return HttpResponseRedirect(next_page)
-
-    start = request.GET.get('start', None)
-    end = request.GET.get('end', None)
-    if not check_time_period_valid(start, end):
-        messages.error(request, _('Failed to export excel, invalid start or end date'))
-        return HttpResponseRedirect(next_page)
-
-    events = get_log_events_by_type_and_time('file_audit', start, end)
-
-    head = [_("User"), _("Type"), _("IP"), _("Device"), _("Date"),
-            _("Library Name"), _("Library ID"), _("Library Owner"), _("File Path")]
-    data_list = []
-
-    events.sort(key=lambda x: x.timestamp, reverse=True)
-    for ev in events:
-        event_type, ev.show_device = generate_file_audit_event_type(ev)
-
-        repo_id = ev.repo_id
-        repo = seafile_api.get_repo(repo_id)
-        if repo:
-            repo_name = repo.name
-            repo_owner = seafile_api.get_repo_owner(repo_id) or \
-                    seafile_api.get_org_repo_owner(repo_id)
-        else:
-            repo_name = _('Deleted')
-            repo_owner = '--'
-
-        username = ev.user if ev.user else _('Anonymous User')
-        date = utc_to_local(ev.timestamp).strftime('%Y-%m-%d %H:%M:%S') if \
-            ev.timestamp else ''
-
-        row = [username, event_type, ev.ip, ev.show_device,
-               date, repo_name, ev.repo_id, repo_owner, ev.file_path]
-        data_list.append(row)
-
-    wb = write_xls('file-access-logs', head, data_list)
-    if not wb:
-        messages.error(request, _('Failed to export excel'))
-        return HttpResponseRedirect(next_page)
-
-    response = HttpResponse(content_type='application/ms-excel')
-    response['Content-Disposition'] = 'attachment; filename=file-access-logs.xlsx'
-    wb.save(response)
+    task_id = request.GET.get('task_id', '')
+    if not task_id:
+        error_msg = 'task_id invalid.'
+        return api_error(status.HTTP_400_BAD_REQUEST, error_msg)
+    excel_name = 'file-access-logs.xlsx'
+    target_dir = '/tmp/seafile_events/'
+    tmp_excel_path = os.path.join(target_dir, excel_name)
+    if not os.path.isfile(tmp_excel_path):
+        return api_error(status.HTTP_404_NOT_FOUND, excel_name + ' not found.')
+
+    # tmp-excel-file is in container, for download multiple times do not delete it
+    response = FileResponse(open(tmp_excel_path, 'rb'), content_type='application/ms-excel', as_attachment=True)
+    response['Content-Disposition'] = 'attachment;filename*=UTF-8\'\'' + quote(excel_name)
     return response
 
 @login_required

seahub/urls.py

@@ -126,7 +126,7 @@
 from seahub.api2.endpoints.admin.abuse_reports import AdminAbuseReportsView, AdminAbuseReportView
 from seahub.api2.endpoints.admin.revision_tag import AdminTaggedItemsView
 from seahub.api2.endpoints.admin.login_logs import LoginLogs, AdminLoginLogs
-from seahub.api2.endpoints.admin.file_audit import FileAudit
+from seahub.api2.endpoints.admin.file_audit import FileAudit, FileAuditExport, FileAuditExportStatus
 from seahub.api2.endpoints.admin.file_update import FileUpdate
 from seahub.api2.endpoints.admin.perm_audit import PermAudit
 from seahub.api2.endpoints.admin.sysinfo import SysInfo
@@ -886,6 +886,8 @@
         path('sys/loginadmin/export-excel/', sys_login_admin_export_excel, name='sys_login_admin_export_excel'),
 
         re_path(r'^api/v2.1/admin/logs/file-audit/$', FileAudit.as_view(), name='api-v2.1-admin-logs-file-audit'),
+        re_path(r'^api/v2.1/admin/logs/file-audit/export-excel/$', FileAuditExport.as_view(), name='api-v2.1-admin-logs-file-audit-export-excel'),
+        re_path(r'^api/v2.1/query-export-status/$', FileAuditExportStatus.as_view(), name='api-v2.1-query-export-status'),
         path('sys/log/fileaudit/export-excel/', sys_log_file_audit_export_excel, name='sys_log_file_audit_export_excel'),
 
         re_path(r'^api/v2.1/admin/logs/file-update/$', FileUpdate.as_view(), name='api-v2.1-admin-logs-file-update'),

seahub/utils/__init__.py

@@ -658,6 +658,22 @@ def get_log_events_by_time(log_type, tstart, tend):
 
         return events if events else None
 
+    def get_file_audit_task(tstart, tend):
+        """
+        Return task id
+        """
+        with _get_seafevents_session() as session:
+            task_id = seafevents_api.get_file_audit_task(session, tstart, tend)
+
+        return task_id if task_id else None
+
+    def query_fileaudit_export_status(task_id):
+        """
+        Return task status
+        """
+        res = seafevents_api.query_status(task_id)
+        return res
+
     def generate_file_audit_event_type(e):
 
         event_type_dict = {
@@ -1165,7 +1181,6 @@ def prepare_converted_html(raw_path, obj_id, doctype, ret_dict):
         try:
             add_office_convert_task(obj_id, doctype, raw_path)
         except Exception as e:
-            print(e)
             logging.exception('failed to add_office_convert_task: %s' % e)
             return _('Internal Server Error')
         return None