diff --git a/apps/filebrowser/src/filebrowser/api.py b/apps/filebrowser/src/filebrowser/api.py index 660c69c1b9..d2380f1080 100644 --- a/apps/filebrowser/src/filebrowser/api.py +++ b/apps/filebrowser/src/filebrowser/api.py @@ -44,6 +44,7 @@ def decorator(*args, **kwargs): response['status'] = -1 response['message'] = smart_unicode(e) return JsonResponse(response) + return decorator @@ -78,10 +79,12 @@ def get_filesystems_with_home_dirs(request): # Using as a public API only for n elif fs == 'ofs': user_home_dir = get_ofs_home_directory() - filesystems.append({ - 'file_system': fs, - 'user_home_directory': user_home_dir, - }) + filesystems.append( + { + 'file_system': fs, + 'user_home_directory': user_home_dir, + } + ) return JsonResponse(filesystems, safe=False) @@ -116,11 +119,11 @@ def rename(request): dest_path = request.POST.get('dest_path') if "#" in dest_path: - raise Exception(_( - "Error renaming %s to %s. Hashes are not allowed in file or directory names." % (os.path.basename(src_path), dest_path) - )) + raise Exception( + _("Error renaming %s to %s. Hashes are not allowed in file or directory names." % (os.path.basename(src_path), dest_path)) + ) - # If dest_path doesn't have a directory specified, use same dir. + # If dest_path doesn't have a directory specified, use same directory. if "/" not in dest_path: src_dir = os.path.dirname(src_path) dest_path = request.fs.join(src_dir, dest_path) @@ -132,6 +135,37 @@ def rename(request): return HttpResponse(status=200) +@error_handler +def move(request): + src_path = request.POST.get('src_path') + dest_path = request.POST.get('dest_path') + + if src_path == dest_path: + raise Exception(_('Source and destination path cannot be same.')) + + request.fs.rename(src_path, dest_path) + return HttpResponse(status=200) + + +@error_handler +def copy(request): + src_path = request.POST.get('src_path') + dest_path = request.POST.get('dest_path') + + if src_path == dest_path: + raise Exception(_('Source and destination path cannot be same.')) + + # Copy method for Ozone FS returns a string of skipped files if their size is greater than configured chunk size. + if src_path.startswith('ofs://'): + ofs_skip_files = request.fs.copy(src_path, dest_path, recursive=True, owner=request.user) + if ofs_skip_files: + return HttpResponse(ofs_skip_files, status=200) + else: + request.fs.copy(src_path, dest_path, recursive=True, owner=request.user) + + return HttpResponse(status=200) + + @error_handler def content_summary(request, path): path = _normalize_path(path) @@ -144,11 +178,49 @@ def content_summary(request, path): return JsonResponse(response, status=404) try: - stats = request.fs.get_content_summary(path) + content_summary = request.fs.get_content_summary(path) replication_factor = request.fs.stats(path)['replication'] - stats.summary.update({'replication': replication_factor}) - response['summary'] = stats.summary + + content_summary.summary.update({'replication': replication_factor}) + response['summary'] = content_summary.summary except Exception as e: raise Exception(_('Failed to fetch content summary for "%s". ') % path) return JsonResponse(response) + + +@error_handler +def set_replication(request): + src_path = request.POST.get('src_path') + replication_factor = request.POST.get('replication_factor') + + result = request.fs.set_replication(src_path, replication_factor) + if not result: + raise Exception(_("Failed to set the replication factor.")) + + return HttpResponse(status=200) + + +@error_handler +def rmtree(request): + path = request.POST.get('path') + skip_trash = request.POST.get('skip_trash', False) + + request.fs.rmtree(path, skip_trash) + + return HttpResponse(status=200) + + +@error_handler +def trash_restore(request): + path = request.POST.get('path') + request.fs.restore(path) + + return HttpResponse(status=200) + + +@error_handler +def trash_purge(request): + request.fs.purge_trash() + + return HttpResponse(status=200) diff --git a/desktop/core/src/desktop/api_public.py b/desktop/core/src/desktop/api_public.py index bd0f9327b8..d579e3c299 100644 --- a/desktop/core/src/desktop/api_public.py +++ b/desktop/core/src/desktop/api_public.py @@ -15,66 +15,70 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import json +import logging -from django.http import QueryDict, HttpResponse -from rest_framework.permissions import AllowAny +from django.http import HttpResponse, QueryDict from rest_framework.decorators import api_view, authentication_classes, permission_classes +from rest_framework.permissions import AllowAny -from filebrowser import views as filebrowser_views, api as filebrowser_api -from indexer import api3 as indexer_api3 -from metadata import optimizer_api -from notebook import api as notebook_api -from notebook.conf import get_ordered_interpreters - +from beeswax import api as beeswax_api from desktop import api2 as desktop_api from desktop.auth.backend import rewrite_user from desktop.lib import fsmanager from desktop.lib.connectors import api as connector_api - -from useradmin import views as useradmin_views, api as useradmin_api - -from beeswax import api as beeswax_api - +from filebrowser import api as filebrowser_api, views as filebrowser_views +from indexer import api3 as indexer_api3 +from metadata import optimizer_api +from notebook import api as notebook_api +from notebook.conf import get_ordered_interpreters +from useradmin import api as useradmin_api, views as useradmin_views LOG = logging.getLogger() # Core + @api_view(["POST"]) def get_config(request): django_request = get_django_request(request) return desktop_api.get_config(django_request) + @api_view(["GET"]) def get_context_namespaces(request, interface): django_request = get_django_request(request) return desktop_api.get_context_namespaces(django_request, interface) + @api_view(["GET"]) @permission_classes([AllowAny]) def get_banners(request): return desktop_api.get_banners(request) + # Editor + @api_view(["POST"]) def create_notebook(request): django_request = get_django_request(request) return notebook_api.create_notebook(django_request) + @api_view(["POST"]) def create_session(request): django_request = get_django_request(request) return notebook_api.create_session(django_request) + @api_view(["POST"]) def close_session(request): django_request = get_django_request(request) return notebook_api.close_session(django_request) + @api_view(["POST"]) def execute(request, dialect=None): django_request = get_django_request(request) @@ -88,14 +92,13 @@ def execute(request, dialect=None): 'interpreter': '%(type)s' % interpreter, # If connectors off, we expect a string 'interpreter_id': ('%(type)s' if interpreter['type'].isdigit() else '"%(type)s"') % interpreter, - 'dialect': '%(dialect)s' % interpreter + 'dialect': '%(dialect)s' % interpreter, } data = { 'notebook': '{"type":"query-%(interpreter)s","snippets":[{"id":%(interpreter_id)s,"statement_raw":"",' - '"type":"%(interpreter)s","status":"","variables":[],"properties":{}}],' - '"name":"","isSaved":false,"sessions":[]}' % params, - 'snippet': '{"id":%(interpreter_id)s,"type":"%(interpreter)s","result":{},"statement":"%(statement)s","properties":{}}' % params + '"type":"%(interpreter)s","status":"","variables":[],"properties":{}}],"name":"","isSaved":false,"sessions":[]}' % params, + 'snippet': '{"id":%(interpreter_id)s,"type":"%(interpreter)s","result":{},"statement":"%(statement)s","properties":{}}' % params, } # Optional database param for specific query statements like "show tables;" @@ -106,12 +109,12 @@ def execute(request, dialect=None): data['snippet'] = json.dumps(snippet) - django_request.POST = QueryDict(mutable=True) django_request.POST.update(data) return notebook_api.execute(django_request, dialect) + @api_view(["POST"]) def check_status(request): django_request = get_django_request(request) @@ -120,6 +123,7 @@ def check_status(request): return notebook_api.check_status(django_request) + @api_view(["POST"]) def fetch_result_data(request): django_request = get_django_request(request) @@ -128,26 +132,31 @@ def fetch_result_data(request): return notebook_api.fetch_result_data(django_request) + @api_view(["POST"]) def fetch_result_metadata(request): django_request = get_django_request(request) return notebook_api.fetch_result_metadata(django_request) + @api_view(["POST"]) def fetch_result_size(request): django_request = get_django_request(request) return notebook_api.fetch_result_size(django_request) + @api_view(["POST"]) def cancel_statement(request): django_request = get_django_request(request) return notebook_api.cancel_statement(django_request) + @api_view(["POST"]) def close_statement(request): django_request = get_django_request(request) return notebook_api.close_statement(django_request) + @api_view(["POST"]) def get_logs(request): django_request = get_django_request(request) @@ -156,6 +165,7 @@ def get_logs(request): return notebook_api.get_logs(django_request) + @api_view(["POST"]) def get_sample_data(request, server=None, database=None, table=None, column=None): django_request = get_django_request(request) @@ -164,6 +174,7 @@ def get_sample_data(request, server=None, database=None, table=None, column=None return notebook_api.get_sample_data(django_request, server, database, table, column) + @api_view(["POST"]) def autocomplete(request, server=None, database=None, table=None, column=None, nested=None): django_request = get_django_request(request) @@ -172,16 +183,19 @@ def autocomplete(request, server=None, database=None, table=None, column=None, n return notebook_api.autocomplete(django_request, server, database, table, column, nested) + @api_view(["POST"]) def describe(request, database, table=None, column=None): django_request = get_django_request(request) return notebook_api.describe(django_request, database, table, column) + @api_view(["GET"]) def get_history(request): django_request = get_django_request(request) return notebook_api.get_history(django_request) + @api_view(["POST"]) def analyze_table(request, dialect, database, table, columns=None): django_request = get_django_request(request) @@ -194,58 +208,106 @@ def analyze_table(request, dialect, database, table, columns=None): # Storage + @api_view(["GET"]) def storage_get_filesystems(request): django_request = get_django_request(request) return filebrowser_api.get_filesystems_with_home_dirs(django_request) + @api_view(["GET"]) def storage_view(request, path): django_request = get_django_request(request) return filebrowser_views.view(django_request, path) + @api_view(["GET"]) def storage_download(request, path): django_request = get_django_request(request) return filebrowser_views.download(django_request, path) + @api_view(["POST"]) def storage_upload_file(request): django_request = get_django_request(request) return filebrowser_views.upload_file(django_request) + @api_view(["POST"]) def storage_mkdir(request): django_request = get_django_request(request) return filebrowser_api.mkdir(django_request) + @api_view(["POST"]) def storage_touch(request): django_request = get_django_request(request) return filebrowser_api.touch(django_request) + @api_view(["POST"]) def storage_rename(request): django_request = get_django_request(request) return filebrowser_api.rename(django_request) + @api_view(["GET"]) def storage_content_summary(request, path): django_request = get_django_request(request) return filebrowser_api.content_summary(django_request, path) + +@api_view(["POST"]) +def storage_move(request): + django_request = get_django_request(request) + return filebrowser_api.move(django_request) + + +@api_view(["POST"]) +def storage_copy(request): + django_request = get_django_request(request) + return filebrowser_api.copy(django_request) + + +@api_view(["POST"]) +def storage_set_replication(request): + django_request = get_django_request(request) + return filebrowser_api.set_replication(django_request) + + +@api_view(["POST"]) +def storage_rmtree(request): + django_request = get_django_request(request) + return filebrowser_api.rmtree(django_request) + + +@api_view(["POST"]) +def storage_trash_restore(request): + django_request = get_django_request(request) + return filebrowser_api.trash_restore(django_request) + + +@api_view(["POST"]) +def storage_trash_purge(request): + django_request = get_django_request(request) + return filebrowser_api.trash_purge(django_request) + + # Importer + @api_view(["POST"]) def guess_format(request): django_request = get_django_request(request) return indexer_api3.guess_format(django_request) + @api_view(["POST"]) def guess_field_types(request): django_request = get_django_request(request) return indexer_api3.guess_field_types(django_request) + @api_view(["POST"]) def importer_submit(request): django_request = get_django_request(request) @@ -254,41 +316,49 @@ def importer_submit(request): # Connector + @api_view(["GET"]) def get_connector_types(request): django_request = get_django_request(request) return connector_api.get_connector_types(django_request) + @api_view(["GET"]) def get_connectors_instances(request): django_request = get_django_request(request) return connector_api.get_connectors_instances(django_request) + @api_view(["POST"]) def new_connector(request, dialect, interface): django_request = get_django_request(request) return connector_api.new_connector(django_request, dialect, interface) + @api_view(["GET"]) def get_connector(request, id): django_request = get_django_request(request) return connector_api.get_connector(django_request, id) + @api_view(["POST"]) def update_connector(request): django_request = get_django_request(request) return connector_api.update_connector(django_request) + @api_view(["POST"]) def delete_connector(request): django_request = get_django_request(request) return connector_api.delete_connector(django_request) + @api_view(["POST"]) def test_connector(request): django_request = get_django_request(request) return connector_api.test_connector(django_request) + @api_view(["POST"]) def install_connector_examples(request): django_request = get_django_request(request) @@ -297,56 +367,67 @@ def install_connector_examples(request): # Metadata + @api_view(["POST"]) def predict(request): django_request = get_django_request(request) return optimizer_api.predict(django_request) + @api_view(["POST"]) def query_risk(request): django_request = get_django_request(request) return optimizer_api.query_risk(django_request) + @api_view(["POST"]) def query_compatibility(request): django_request = get_django_request(request) return optimizer_api.query_compatibility(django_request) + @api_view(["POST"]) def similar_queries(request): django_request = get_django_request(request) return optimizer_api.similar_queries(django_request) + @api_view(["POST"]) def top_databases(request): django_request = get_django_request(request) return optimizer_api.top_databases(django_request) + @api_view(["POST"]) def top_tables(request): django_request = get_django_request(request) return optimizer_api.top_tables(django_request) + @api_view(["POST"]) def top_columns(request): django_request = get_django_request(request) return optimizer_api.top_columns(django_request) + @api_view(["POST"]) def top_joins(request): django_request = get_django_request(request) return optimizer_api.top_joins(django_request) + @api_view(["POST"]) def top_filters(request): django_request = get_django_request(request) return optimizer_api.top_filters(django_request) + @api_view(["POST"]) def top_aggs(request): django_request = get_django_request(request) return optimizer_api.top_aggs(django_request) + @api_view(["POST"]) def search_entities_interactive(request): django_request = get_django_request(request) @@ -355,16 +436,19 @@ def search_entities_interactive(request): # IAM + @api_view(["GET"]) def list_for_autocomplete(request): django_request = get_django_request(request) return useradmin_views.list_for_autocomplete(django_request) + @api_view(["GET"]) def get_users_by_id(request): django_request = get_django_request(request) return useradmin_views.get_users_by_id(django_request) + @api_view(["GET"]) def get_users(request): django_request = get_django_request(request) @@ -373,13 +457,14 @@ def get_users(request): # Utils + def _get_interpreter_from_dialect(dialect, user): if not dialect: interpreter = get_ordered_interpreters(user=user)[0] elif '-' in dialect: interpreter = { 'dialect': dialect.split('-')[0], - 'type': dialect.split('-')[1] # Id + 'type': dialect.split('-')[1], # Id } else: interpreter = [i for i in get_ordered_interpreters(user=user) if i['dialect'] == dialect][0] @@ -393,7 +478,7 @@ def _patch_operation_id_request(django_request): if not django_request.POST.get('snippet'): data['snippet'] = '{"type":"1","result":{}}' - django_request.POST = django_request.POST.copy() # Makes it mutable along with copying the object + django_request.POST = django_request.POST.copy() # Makes it mutable along with copying the object django_request.POST.update(data) diff --git a/desktop/core/src/desktop/api_public_urls_v1.py b/desktop/core/src/desktop/api_public_urls_v1.py index c2aa2333b4..d480e4c498 100644 --- a/desktop/core/src/desktop/api_public_urls_v1.py +++ b/desktop/core/src/desktop/api_public_urls_v1.py @@ -39,7 +39,9 @@ re_path(r'^banners/?$', api_public.get_banners, name='core_banners'), re_path(r'^get_config/?$', api_public.get_config), re_path(r'^get_namespaces/(?P[\w\-]+)/?$', api_public.get_context_namespaces), # To remove +] +urlpatterns += [ re_path(r'^editor/create_notebook/?$', api_public.create_notebook, name='editor_create_notebook'), re_path(r'^editor/create_session/?$', api_public.create_session, name='editor_create_session'), re_path(r'^editor/close_session/?$', api_public.close_session, name='editor_close_session'), @@ -52,42 +54,37 @@ re_path(r'^editor/close_statement/?$', api_public.close_statement, name='editor_close_statement'), re_path(r'^editor/get_logs/?$', api_public.get_logs, name='editor_get_logs'), re_path(r'^editor/get_history/?', api_public.get_history, name='editor_get_history'), - re_path(r'^editor/describe/(?P[^/]*)/?$', api_public.describe, name='editor_describe_database'), re_path(r'^editor/describe/(?P[^/]*)/(?P[\w_\-]+)/?$', api_public.describe, name='editor_describe_table'), re_path( - r'^editor/describe/(?P[^/]*)/(?P
\w+)/stats(?:/(?P\w+))?/?$', - api_public.describe, - name='editor_describe_column' + r'^editor/describe/(?P[^/]*)/(?P
\w+)/stats(?:/(?P\w+))?/?$', api_public.describe, name='editor_describe_column' ), - re_path(r'^editor/autocomplete/?$', api_public.autocomplete, name='editor_autocomplete_databases'), re_path( - r"^editor/autocomplete/(?P[^/?]*)/?$", - api_public.autocomplete, - name="editor_autocomplete_tables", + r"^editor/autocomplete/(?P[^/?]*)/?$", + api_public.autocomplete, + name="editor_autocomplete_tables", ), re_path( - r"^editor/autocomplete/(?P[^/?]*)/(?P
[\w_\-]+)/?$", - api_public.autocomplete, - name="editor_autocomplete_columns", + r"^editor/autocomplete/(?P[^/?]*)/(?P
[\w_\-]+)/?$", + api_public.autocomplete, + name="editor_autocomplete_columns", ), re_path( - r"^editor/autocomplete/(?P[^/?]*)/(?P
[\w_\-]+)/(?P\w+)/?$", - api_public.autocomplete, - name="editor_autocomplete_column", + r"^editor/autocomplete/(?P[^/?]*)/(?P
[\w_\-]+)/(?P\w+)/?$", + api_public.autocomplete, + name="editor_autocomplete_column", ), re_path( - r"^editor/autocomplete/(?P[^/?]*)/(?P
[\w_\-]+)/(?P\w+)/(?P.+)/?$", - api_public.autocomplete, - name="editor_autocomplete_nested", + r"^editor/autocomplete/(?P[^/?]*)/(?P
[\w_\-]+)/(?P\w+)/(?P.+)/?$", + api_public.autocomplete, + name="editor_autocomplete_nested", ), - re_path(r'^editor/sample/(?P[^/?]*)/(?P
[\w_\-]+)/?$', api_public.get_sample_data, name='editor_sample_data'), re_path( r'^editor/sample/(?P[^/?]*)/(?P
[\w_\-]+)/(?P\w+)/?$', api_public.get_sample_data, - name='editor_sample_data_column' + name='editor_sample_data_column', ), ] @@ -100,13 +97,19 @@ re_path(r'^storage/touch$', api_public.storage_touch, name='storage_touch'), re_path(r'^storage/rename$', api_public.storage_rename, name='storage_rename'), re_path(r'^storage/content_summary=(?P.*)$', api_public.storage_content_summary, name='storage_content_summary'), + re_path(r'^storage/move$', api_public.storage_move, name='storage_move'), + re_path(r'^storage/copy$', api_public.storage_copy, name='storage_copy'), + re_path(r'^storage/set_replication$', api_public.storage_set_replication, name='storage_set_replication'), + re_path(r'^storage/rmtree$', api_public.storage_rmtree, name='storage_rmtree'), + re_path(r'^storage/trash/restore$', api_public.storage_trash_restore, name='storage_trash_restore'), + re_path(r'^storage/trash/purge$', api_public.storage_trash_purge, name='storage_trash_purge'), ] urlpatterns += [ re_path( r'^(?P.+)/analyze/(?P\w+)/(?P
\w+)(?:/(?P\w+))?/?$', api_public.analyze_table, - name='dialect_analyze_table' + name='dialect_analyze_table', ), ] @@ -124,13 +127,11 @@ urlpatterns += [ re_path(r'^connector/types/?$', api_public.get_connector_types, name='connector_get_types'), re_path(r'^connector/instances/?$', api_public.get_connectors_instances, name='connector_get_instances'), - re_path(r'^connector/instance/new/(?P[\w\-]+)/(?P[\w\-]+)$', api_public.new_connector, name='connector_new'), re_path(r'^connector/instance/get/(?P\d+)$', api_public.get_connector, name='connector_get'), re_path(r'^connector/instance/delete/?$', api_public.delete_connector, name='connector_delete'), re_path(r'^connector/instance/update/?$', api_public.update_connector, name='connector_update'), re_path(r'^connector/instance/test/?$', api_public.test_connector, name='connector_test'), - re_path(r'^connector/examples/install/?$', api_public.install_connector_examples, name='connector_install_examples'), ] @@ -141,7 +142,6 @@ re_path(r'^optimizer/top_joins/?$', api_public.top_joins, name='optimizer_top_joins'), re_path(r'^optimizer/top_filters/?$', api_public.top_filters, name='optimizer_top_filters'), re_path(r'^optimizer/top_aggs/?$', api_public.top_aggs, name='optimizer_top_aggs'), - re_path(r'^optimizer/query_risk/?$', api_public.query_risk, name='optimizer_query_risk'), re_path(r'^optimizer/predict/?$', api_public.predict, name='optimizer_predict'), re_path(r'^optimizer/query_compatibility/?$', api_public.query_compatibility, name='optimizer_query_compatibility'), @@ -155,6 +155,5 @@ urlpatterns += [ re_path(r'^iam/users/autocomplete', api_public.list_for_autocomplete, name='iam_users_list_for_autocomplete'), re_path(r'^iam/users/?$', api_public.get_users_by_id, name='iam_get_users_by_id'), - re_path(r'^iam/get_users/?', api_public.get_users, name='iam_get_users'), -] \ No newline at end of file +] diff --git a/desktop/core/src/desktop/lib/fs/proxyfs.py b/desktop/core/src/desktop/lib/fs/proxyfs.py index 3137c2d603..ad9c17309a 100644 --- a/desktop/core/src/desktop/lib/fs/proxyfs.py +++ b/desktop/core/src/desktop/lib/fs/proxyfs.py @@ -191,6 +191,9 @@ def remove(self, path, skip_trash=False): def restore(self, path): self._get_fs(path).restore(path) + def set_replication(self, src_path, replication_factor): + return self._get_fs(src_path).set_replication(src_path, replication_factor) + def create(self, path, *args, **kwargs): self._get_fs(path).create(path, *args, **kwargs)