From e1182494e798563d34f25384db29ec904ff74fe0 Mon Sep 17 00:00:00 2001 From: Harsh Gupta <42064744+Harshg999@users.noreply.github.com> Date: Thu, 20 Jun 2024 18:20:55 +0530 Subject: [PATCH] [api][fb] Refactor existing filebrowser APIs to clean public APIs (#3739) - Old Filebrowser APIs were bulky with all the form and args looping logic and rendering server side mako files. - This PR implements few of the original APIs for public V1 with only core logic for the new storage browser sending only the required data for UI to use and getting rid of server side rendering slowly. - APIs ported for V1: - storage/move - storage/copy - storage/set_replication - storage/rmtree - storage/trash/restore - storage/trash/purge --- apps/filebrowser/src/filebrowser/api.py | 94 +++++++++++-- desktop/core/src/desktop/api_public.py | 127 +++++++++++++++--- .../core/src/desktop/api_public_urls_v1.py | 49 ++++--- desktop/core/src/desktop/lib/fs/proxyfs.py | 3 + 4 files changed, 216 insertions(+), 57 deletions(-) diff --git a/apps/filebrowser/src/filebrowser/api.py b/apps/filebrowser/src/filebrowser/api.py index 660c69c1b9..d2380f1080 100644 --- a/apps/filebrowser/src/filebrowser/api.py +++ b/apps/filebrowser/src/filebrowser/api.py @@ -44,6 +44,7 @@ def decorator(*args, **kwargs): response['status'] = -1 response['message'] = smart_unicode(e) return JsonResponse(response) + return decorator @@ -78,10 +79,12 @@ def get_filesystems_with_home_dirs(request): # Using as a public API only for n elif fs == 'ofs': user_home_dir = get_ofs_home_directory() - filesystems.append({ - 'file_system': fs, - 'user_home_directory': user_home_dir, - }) + filesystems.append( + { + 'file_system': fs, + 'user_home_directory': user_home_dir, + } + ) return JsonResponse(filesystems, safe=False) @@ -116,11 +119,11 @@ def rename(request): dest_path = request.POST.get('dest_path') if "#" in dest_path: - raise Exception(_( - "Error renaming %s to %s. Hashes are not allowed in file or directory names." % (os.path.basename(src_path), dest_path) - )) + raise Exception( + _("Error renaming %s to %s. Hashes are not allowed in file or directory names." % (os.path.basename(src_path), dest_path)) + ) - # If dest_path doesn't have a directory specified, use same dir. + # If dest_path doesn't have a directory specified, use same directory. if "/" not in dest_path: src_dir = os.path.dirname(src_path) dest_path = request.fs.join(src_dir, dest_path) @@ -132,6 +135,37 @@ def rename(request): return HttpResponse(status=200) +@error_handler +def move(request): + src_path = request.POST.get('src_path') + dest_path = request.POST.get('dest_path') + + if src_path == dest_path: + raise Exception(_('Source and destination path cannot be same.')) + + request.fs.rename(src_path, dest_path) + return HttpResponse(status=200) + + +@error_handler +def copy(request): + src_path = request.POST.get('src_path') + dest_path = request.POST.get('dest_path') + + if src_path == dest_path: + raise Exception(_('Source and destination path cannot be same.')) + + # Copy method for Ozone FS returns a string of skipped files if their size is greater than configured chunk size. + if src_path.startswith('ofs://'): + ofs_skip_files = request.fs.copy(src_path, dest_path, recursive=True, owner=request.user) + if ofs_skip_files: + return HttpResponse(ofs_skip_files, status=200) + else: + request.fs.copy(src_path, dest_path, recursive=True, owner=request.user) + + return HttpResponse(status=200) + + @error_handler def content_summary(request, path): path = _normalize_path(path) @@ -144,11 +178,49 @@ def content_summary(request, path): return JsonResponse(response, status=404) try: - stats = request.fs.get_content_summary(path) + content_summary = request.fs.get_content_summary(path) replication_factor = request.fs.stats(path)['replication'] - stats.summary.update({'replication': replication_factor}) - response['summary'] = stats.summary + + content_summary.summary.update({'replication': replication_factor}) + response['summary'] = content_summary.summary except Exception as e: raise Exception(_('Failed to fetch content summary for "%s". ') % path) return JsonResponse(response) + + +@error_handler +def set_replication(request): + src_path = request.POST.get('src_path') + replication_factor = request.POST.get('replication_factor') + + result = request.fs.set_replication(src_path, replication_factor) + if not result: + raise Exception(_("Failed to set the replication factor.")) + + return HttpResponse(status=200) + + +@error_handler +def rmtree(request): + path = request.POST.get('path') + skip_trash = request.POST.get('skip_trash', False) + + request.fs.rmtree(path, skip_trash) + + return HttpResponse(status=200) + + +@error_handler +def trash_restore(request): + path = request.POST.get('path') + request.fs.restore(path) + + return HttpResponse(status=200) + + +@error_handler +def trash_purge(request): + request.fs.purge_trash() + + return HttpResponse(status=200) diff --git a/desktop/core/src/desktop/api_public.py b/desktop/core/src/desktop/api_public.py index bd0f9327b8..d579e3c299 100644 --- a/desktop/core/src/desktop/api_public.py +++ b/desktop/core/src/desktop/api_public.py @@ -15,66 +15,70 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import json +import logging -from django.http import QueryDict, HttpResponse -from rest_framework.permissions import AllowAny +from django.http import HttpResponse, QueryDict from rest_framework.decorators import api_view, authentication_classes, permission_classes +from rest_framework.permissions import AllowAny -from filebrowser import views as filebrowser_views, api as filebrowser_api -from indexer import api3 as indexer_api3 -from metadata import optimizer_api -from notebook import api as notebook_api -from notebook.conf import get_ordered_interpreters - +from beeswax import api as beeswax_api from desktop import api2 as desktop_api from desktop.auth.backend import rewrite_user from desktop.lib import fsmanager from desktop.lib.connectors import api as connector_api - -from useradmin import views as useradmin_views, api as useradmin_api - -from beeswax import api as beeswax_api - +from filebrowser import api as filebrowser_api, views as filebrowser_views +from indexer import api3 as indexer_api3 +from metadata import optimizer_api +from notebook import api as notebook_api +from notebook.conf import get_ordered_interpreters +from useradmin import api as useradmin_api, views as useradmin_views LOG = logging.getLogger() # Core + @api_view(["POST"]) def get_config(request): django_request = get_django_request(request) return desktop_api.get_config(django_request) + @api_view(["GET"]) def get_context_namespaces(request, interface): django_request = get_django_request(request) return desktop_api.get_context_namespaces(django_request, interface) + @api_view(["GET"]) @permission_classes([AllowAny]) def get_banners(request): return desktop_api.get_banners(request) + # Editor + @api_view(["POST"]) def create_notebook(request): django_request = get_django_request(request) return notebook_api.create_notebook(django_request) + @api_view(["POST"]) def create_session(request): django_request = get_django_request(request) return notebook_api.create_session(django_request) + @api_view(["POST"]) def close_session(request): django_request = get_django_request(request) return notebook_api.close_session(django_request) + @api_view(["POST"]) def execute(request, dialect=None): django_request = get_django_request(request) @@ -88,14 +92,13 @@ def execute(request, dialect=None): 'interpreter': '%(type)s' % interpreter, # If connectors off, we expect a string 'interpreter_id': ('%(type)s' if interpreter['type'].isdigit() else '"%(type)s"') % interpreter, - 'dialect': '%(dialect)s' % interpreter + 'dialect': '%(dialect)s' % interpreter, } data = { 'notebook': '{"type":"query-%(interpreter)s","snippets":[{"id":%(interpreter_id)s,"statement_raw":"",' - '"type":"%(interpreter)s","status":"","variables":[],"properties":{}}],' - '"name":"","isSaved":false,"sessions":[]}' % params, - 'snippet': '{"id":%(interpreter_id)s,"type":"%(interpreter)s","result":{},"statement":"%(statement)s","properties":{}}' % params + '"type":"%(interpreter)s","status":"","variables":[],"properties":{}}],"name":"","isSaved":false,"sessions":[]}' % params, + 'snippet': '{"id":%(interpreter_id)s,"type":"%(interpreter)s","result":{},"statement":"%(statement)s","properties":{}}' % params, } # Optional database param for specific query statements like "show tables;" @@ -106,12 +109,12 @@ def execute(request, dialect=None): data['snippet'] = json.dumps(snippet) - django_request.POST = QueryDict(mutable=True) django_request.POST.update(data) return notebook_api.execute(django_request, dialect) + @api_view(["POST"]) def check_status(request): django_request = get_django_request(request) @@ -120,6 +123,7 @@ def check_status(request): return notebook_api.check_status(django_request) + @api_view(["POST"]) def fetch_result_data(request): django_request = get_django_request(request) @@ -128,26 +132,31 @@ def fetch_result_data(request): return notebook_api.fetch_result_data(django_request) + @api_view(["POST"]) def fetch_result_metadata(request): django_request = get_django_request(request) return notebook_api.fetch_result_metadata(django_request) + @api_view(["POST"]) def fetch_result_size(request): django_request = get_django_request(request) return notebook_api.fetch_result_size(django_request) + @api_view(["POST"]) def cancel_statement(request): django_request = get_django_request(request) return notebook_api.cancel_statement(django_request) + @api_view(["POST"]) def close_statement(request): django_request = get_django_request(request) return notebook_api.close_statement(django_request) + @api_view(["POST"]) def get_logs(request): django_request = get_django_request(request) @@ -156,6 +165,7 @@ def get_logs(request): return notebook_api.get_logs(django_request) + @api_view(["POST"]) def get_sample_data(request, server=None, database=None, table=None, column=None): django_request = get_django_request(request) @@ -164,6 +174,7 @@ def get_sample_data(request, server=None, database=None, table=None, column=None return notebook_api.get_sample_data(django_request, server, database, table, column) + @api_view(["POST"]) def autocomplete(request, server=None, database=None, table=None, column=None, nested=None): django_request = get_django_request(request) @@ -172,16 +183,19 @@ def autocomplete(request, server=None, database=None, table=None, column=None, n return notebook_api.autocomplete(django_request, server, database, table, column, nested) + @api_view(["POST"]) def describe(request, database, table=None, column=None): django_request = get_django_request(request) return notebook_api.describe(django_request, database, table, column) + @api_view(["GET"]) def get_history(request): django_request = get_django_request(request) return notebook_api.get_history(django_request) + @api_view(["POST"]) def analyze_table(request, dialect, database, table, columns=None): django_request = get_django_request(request) @@ -194,58 +208,106 @@ def analyze_table(request, dialect, database, table, columns=None): # Storage + @api_view(["GET"]) def storage_get_filesystems(request): django_request = get_django_request(request) return filebrowser_api.get_filesystems_with_home_dirs(django_request) + @api_view(["GET"]) def storage_view(request, path): django_request = get_django_request(request) return filebrowser_views.view(django_request, path) + @api_view(["GET"]) def storage_download(request, path): django_request = get_django_request(request) return filebrowser_views.download(django_request, path) + @api_view(["POST"]) def storage_upload_file(request): django_request = get_django_request(request) return filebrowser_views.upload_file(django_request) + @api_view(["POST"]) def storage_mkdir(request): django_request = get_django_request(request) return filebrowser_api.mkdir(django_request) + @api_view(["POST"]) def storage_touch(request): django_request = get_django_request(request) return filebrowser_api.touch(django_request) + @api_view(["POST"]) def storage_rename(request): django_request = get_django_request(request) return filebrowser_api.rename(django_request) + @api_view(["GET"]) def storage_content_summary(request, path): django_request = get_django_request(request) return filebrowser_api.content_summary(django_request, path) + +@api_view(["POST"]) +def storage_move(request): + django_request = get_django_request(request) + return filebrowser_api.move(django_request) + + +@api_view(["POST"]) +def storage_copy(request): + django_request = get_django_request(request) + return filebrowser_api.copy(django_request) + + +@api_view(["POST"]) +def storage_set_replication(request): + django_request = get_django_request(request) + return filebrowser_api.set_replication(django_request) + + +@api_view(["POST"]) +def storage_rmtree(request): + django_request = get_django_request(request) + return filebrowser_api.rmtree(django_request) + + +@api_view(["POST"]) +def storage_trash_restore(request): + django_request = get_django_request(request) + return filebrowser_api.trash_restore(django_request) + + +@api_view(["POST"]) +def storage_trash_purge(request): + django_request = get_django_request(request) + return filebrowser_api.trash_purge(django_request) + + # Importer + @api_view(["POST"]) def guess_format(request): django_request = get_django_request(request) return indexer_api3.guess_format(django_request) + @api_view(["POST"]) def guess_field_types(request): django_request = get_django_request(request) return indexer_api3.guess_field_types(django_request) + @api_view(["POST"]) def importer_submit(request): django_request = get_django_request(request) @@ -254,41 +316,49 @@ def importer_submit(request): # Connector + @api_view(["GET"]) def get_connector_types(request): django_request = get_django_request(request) return connector_api.get_connector_types(django_request) + @api_view(["GET"]) def get_connectors_instances(request): django_request = get_django_request(request) return connector_api.get_connectors_instances(django_request) + @api_view(["POST"]) def new_connector(request, dialect, interface): django_request = get_django_request(request) return connector_api.new_connector(django_request, dialect, interface) + @api_view(["GET"]) def get_connector(request, id): django_request = get_django_request(request) return connector_api.get_connector(django_request, id) + @api_view(["POST"]) def update_connector(request): django_request = get_django_request(request) return connector_api.update_connector(django_request) + @api_view(["POST"]) def delete_connector(request): django_request = get_django_request(request) return connector_api.delete_connector(django_request) + @api_view(["POST"]) def test_connector(request): django_request = get_django_request(request) return connector_api.test_connector(django_request) + @api_view(["POST"]) def install_connector_examples(request): django_request = get_django_request(request) @@ -297,56 +367,67 @@ def install_connector_examples(request): # Metadata + @api_view(["POST"]) def predict(request): django_request = get_django_request(request) return optimizer_api.predict(django_request) + @api_view(["POST"]) def query_risk(request): django_request = get_django_request(request) return optimizer_api.query_risk(django_request) + @api_view(["POST"]) def query_compatibility(request): django_request = get_django_request(request) return optimizer_api.query_compatibility(django_request) + @api_view(["POST"]) def similar_queries(request): django_request = get_django_request(request) return optimizer_api.similar_queries(django_request) + @api_view(["POST"]) def top_databases(request): django_request = get_django_request(request) return optimizer_api.top_databases(django_request) + @api_view(["POST"]) def top_tables(request): django_request = get_django_request(request) return optimizer_api.top_tables(django_request) + @api_view(["POST"]) def top_columns(request): django_request = get_django_request(request) return optimizer_api.top_columns(django_request) + @api_view(["POST"]) def top_joins(request): django_request = get_django_request(request) return optimizer_api.top_joins(django_request) + @api_view(["POST"]) def top_filters(request): django_request = get_django_request(request) return optimizer_api.top_filters(django_request) + @api_view(["POST"]) def top_aggs(request): django_request = get_django_request(request) return optimizer_api.top_aggs(django_request) + @api_view(["POST"]) def search_entities_interactive(request): django_request = get_django_request(request) @@ -355,16 +436,19 @@ def search_entities_interactive(request): # IAM + @api_view(["GET"]) def list_for_autocomplete(request): django_request = get_django_request(request) return useradmin_views.list_for_autocomplete(django_request) + @api_view(["GET"]) def get_users_by_id(request): django_request = get_django_request(request) return useradmin_views.get_users_by_id(django_request) + @api_view(["GET"]) def get_users(request): django_request = get_django_request(request) @@ -373,13 +457,14 @@ def get_users(request): # Utils + def _get_interpreter_from_dialect(dialect, user): if not dialect: interpreter = get_ordered_interpreters(user=user)[0] elif '-' in dialect: interpreter = { 'dialect': dialect.split('-')[0], - 'type': dialect.split('-')[1] # Id + 'type': dialect.split('-')[1], # Id } else: interpreter = [i for i in get_ordered_interpreters(user=user) if i['dialect'] == dialect][0] @@ -393,7 +478,7 @@ def _patch_operation_id_request(django_request): if not django_request.POST.get('snippet'): data['snippet'] = '{"type":"1","result":{}}' - django_request.POST = django_request.POST.copy() # Makes it mutable along with copying the object + django_request.POST = django_request.POST.copy() # Makes it mutable along with copying the object django_request.POST.update(data) diff --git a/desktop/core/src/desktop/api_public_urls_v1.py b/desktop/core/src/desktop/api_public_urls_v1.py index c2aa2333b4..d480e4c498 100644 --- a/desktop/core/src/desktop/api_public_urls_v1.py +++ b/desktop/core/src/desktop/api_public_urls_v1.py @@ -39,7 +39,9 @@ re_path(r'^banners/?$', api_public.get_banners, name='core_banners'), re_path(r'^get_config/?$', api_public.get_config), re_path(r'^get_namespaces/(?P[\w\-]+)/?$', api_public.get_context_namespaces), # To remove +] +urlpatterns += [ re_path(r'^editor/create_notebook/?$', api_public.create_notebook, name='editor_create_notebook'), re_path(r'^editor/create_session/?$', api_public.create_session, name='editor_create_session'), re_path(r'^editor/close_session/?$', api_public.close_session, name='editor_close_session'), @@ -52,42 +54,37 @@ re_path(r'^editor/close_statement/?$', api_public.close_statement, name='editor_close_statement'), re_path(r'^editor/get_logs/?$', api_public.get_logs, name='editor_get_logs'), re_path(r'^editor/get_history/?', api_public.get_history, name='editor_get_history'), - re_path(r'^editor/describe/(?P[^/]*)/?$', api_public.describe, name='editor_describe_database'), re_path(r'^editor/describe/(?P[^/]*)/(?P[\w_\-]+)/?$', api_public.describe, name='editor_describe_table'), re_path( - r'^editor/describe/(?P[^/]*)/(?P
\w+)/stats(?:/(?P\w+))?/?$', - api_public.describe, - name='editor_describe_column' + r'^editor/describe/(?P[^/]*)/(?P
\w+)/stats(?:/(?P\w+))?/?$', api_public.describe, name='editor_describe_column' ), - re_path(r'^editor/autocomplete/?$', api_public.autocomplete, name='editor_autocomplete_databases'), re_path( - r"^editor/autocomplete/(?P[^/?]*)/?$", - api_public.autocomplete, - name="editor_autocomplete_tables", + r"^editor/autocomplete/(?P[^/?]*)/?$", + api_public.autocomplete, + name="editor_autocomplete_tables", ), re_path( - r"^editor/autocomplete/(?P[^/?]*)/(?P
[\w_\-]+)/?$", - api_public.autocomplete, - name="editor_autocomplete_columns", + r"^editor/autocomplete/(?P[^/?]*)/(?P
[\w_\-]+)/?$", + api_public.autocomplete, + name="editor_autocomplete_columns", ), re_path( - r"^editor/autocomplete/(?P[^/?]*)/(?P
[\w_\-]+)/(?P\w+)/?$", - api_public.autocomplete, - name="editor_autocomplete_column", + r"^editor/autocomplete/(?P[^/?]*)/(?P
[\w_\-]+)/(?P\w+)/?$", + api_public.autocomplete, + name="editor_autocomplete_column", ), re_path( - r"^editor/autocomplete/(?P[^/?]*)/(?P
[\w_\-]+)/(?P\w+)/(?P.+)/?$", - api_public.autocomplete, - name="editor_autocomplete_nested", + r"^editor/autocomplete/(?P[^/?]*)/(?P
[\w_\-]+)/(?P\w+)/(?P.+)/?$", + api_public.autocomplete, + name="editor_autocomplete_nested", ), - re_path(r'^editor/sample/(?P[^/?]*)/(?P
[\w_\-]+)/?$', api_public.get_sample_data, name='editor_sample_data'), re_path( r'^editor/sample/(?P[^/?]*)/(?P
[\w_\-]+)/(?P\w+)/?$', api_public.get_sample_data, - name='editor_sample_data_column' + name='editor_sample_data_column', ), ] @@ -100,13 +97,19 @@ re_path(r'^storage/touch$', api_public.storage_touch, name='storage_touch'), re_path(r'^storage/rename$', api_public.storage_rename, name='storage_rename'), re_path(r'^storage/content_summary=(?P.*)$', api_public.storage_content_summary, name='storage_content_summary'), + re_path(r'^storage/move$', api_public.storage_move, name='storage_move'), + re_path(r'^storage/copy$', api_public.storage_copy, name='storage_copy'), + re_path(r'^storage/set_replication$', api_public.storage_set_replication, name='storage_set_replication'), + re_path(r'^storage/rmtree$', api_public.storage_rmtree, name='storage_rmtree'), + re_path(r'^storage/trash/restore$', api_public.storage_trash_restore, name='storage_trash_restore'), + re_path(r'^storage/trash/purge$', api_public.storage_trash_purge, name='storage_trash_purge'), ] urlpatterns += [ re_path( r'^(?P.+)/analyze/(?P\w+)/(?P
\w+)(?:/(?P\w+))?/?$', api_public.analyze_table, - name='dialect_analyze_table' + name='dialect_analyze_table', ), ] @@ -124,13 +127,11 @@ urlpatterns += [ re_path(r'^connector/types/?$', api_public.get_connector_types, name='connector_get_types'), re_path(r'^connector/instances/?$', api_public.get_connectors_instances, name='connector_get_instances'), - re_path(r'^connector/instance/new/(?P[\w\-]+)/(?P[\w\-]+)$', api_public.new_connector, name='connector_new'), re_path(r'^connector/instance/get/(?P\d+)$', api_public.get_connector, name='connector_get'), re_path(r'^connector/instance/delete/?$', api_public.delete_connector, name='connector_delete'), re_path(r'^connector/instance/update/?$', api_public.update_connector, name='connector_update'), re_path(r'^connector/instance/test/?$', api_public.test_connector, name='connector_test'), - re_path(r'^connector/examples/install/?$', api_public.install_connector_examples, name='connector_install_examples'), ] @@ -141,7 +142,6 @@ re_path(r'^optimizer/top_joins/?$', api_public.top_joins, name='optimizer_top_joins'), re_path(r'^optimizer/top_filters/?$', api_public.top_filters, name='optimizer_top_filters'), re_path(r'^optimizer/top_aggs/?$', api_public.top_aggs, name='optimizer_top_aggs'), - re_path(r'^optimizer/query_risk/?$', api_public.query_risk, name='optimizer_query_risk'), re_path(r'^optimizer/predict/?$', api_public.predict, name='optimizer_predict'), re_path(r'^optimizer/query_compatibility/?$', api_public.query_compatibility, name='optimizer_query_compatibility'), @@ -155,6 +155,5 @@ urlpatterns += [ re_path(r'^iam/users/autocomplete', api_public.list_for_autocomplete, name='iam_users_list_for_autocomplete'), re_path(r'^iam/users/?$', api_public.get_users_by_id, name='iam_get_users_by_id'), - re_path(r'^iam/get_users/?', api_public.get_users, name='iam_get_users'), -] \ No newline at end of file +] diff --git a/desktop/core/src/desktop/lib/fs/proxyfs.py b/desktop/core/src/desktop/lib/fs/proxyfs.py index 3137c2d603..ad9c17309a 100644 --- a/desktop/core/src/desktop/lib/fs/proxyfs.py +++ b/desktop/core/src/desktop/lib/fs/proxyfs.py @@ -191,6 +191,9 @@ def remove(self, path, skip_trash=False): def restore(self, path): self._get_fs(path).restore(path) + def set_replication(self, src_path, replication_factor): + return self._get_fs(src_path).set_replication(src_path, replication_factor) + def create(self, path, *args, **kwargs): self._get_fs(path).create(path, *args, **kwargs)