Merge pull request #18598 from sanjaysrikakulam/webdav_as_user_file_s…

…ource

Bring your own file sources: Add the WebDAV template and configuration

client/src/api/schema/schema.ts

@@ -9035,7 +9035,7 @@ export interface components {
              * Type
              * @enum {string}
              */
-            type: "ftp" | "posix" | "s3fs" | "azure" | "onedata";
+            type: "ftp" | "posix" | "s3fs" | "azure" | "onedata" | "webdav";
             /** Variables */
             variables?:
                 | (
@@ -17366,7 +17366,7 @@ export interface components {
              * Type
              * @enum {string}
              */
-            type: "ftp" | "posix" | "s3fs" | "azure" | "onedata";
+            type: "ftp" | "posix" | "s3fs" | "azure" | "onedata" | "webdav";
             /** Uri Root */
             uri_root: string;
             /**

client/src/components/FileSources/FileSourceTypeSpan.vue

@@ -9,6 +9,7 @@ const MESSAGES = {
     azure: "This is an remote file source plugin based on the Azure service.",
     onedata: "This is an remote file source plugin based on the Onedata service.",
     ftp: "This is an remote file source plugin based on the FTP protocol.",
+    webdav: "This is an remote file source plugin based on the WebDAV protocol.",
 };
 
 interface Props {

doc/source/admin/data.md

@@ -350,6 +350,19 @@ configuration).
 
 ![](file_source_azure_configuration.png)
 
+#### `webdav`
+
+The syntax for the `configuration` section of `webdav` templates looks like this.
+
+![](file_source_webdav_configuration_template.png)
+
+At runtime, after the `configuration` template is expanded, the resulting dictionary
+passed to Galaxy's file source plugin infrastructure looks like this and should match a subset
+of what you'd be able to add directly to `file_sources_conf.yml` (Galaxy's global file source
+configuration).
+
+![](file_source_webdav_configuration.png)
+
 ### YAML Syntax
 
 ![galaxy.files.templates.models](file_source_templates.png)
@@ -398,6 +411,14 @@ and you are comfortable with it storing your user's secrets.
 :language: yaml
 ```
 
+#### Allow Users to Define WebDAV Servers as File Sources
+
+```{literalinclude} ../../../lib/galaxy/files/templates/examples/production_webdav.yml
+:language: yaml
+```
+
+![Screenshot](user_file_source_form_full_webdav.png)
+
 ## Playing Nicer with Ansible
 
 Many large instances of Galaxy are configured with Ansible and much of the existing administrator

doc/source/admin/gen_diagrams.py

@@ -15,6 +15,8 @@
     PosixFileSourceTemplateConfiguration,
     S3FSFileSourceConfiguration,
     S3FSFileSourceTemplateConfiguration,
+    WebdavFileSourceConfiguration,
+    WebdavFileSourceTemplateConfiguration,
 )
 from galaxy.objectstore.templates.models import (
     AwsS3ObjectStoreConfiguration,
@@ -53,6 +55,8 @@
     S3FSFileSourceConfiguration: "file_source_s3fs_configuration",
     FtpFileSourceTemplateConfiguration: "file_source_ftp_configuration_template",
     FtpFileSourceConfiguration: "file_source_ftp_configuration",
+    WebdavFileSourceTemplateConfiguration: "file_source_webdav_configuration_template",
+    WebdavFileSourceConfiguration: "file_source_webdav_configuration",
 }
 
 for clazz, diagram_name in class_to_diagram.items():

doc/source/admin/search_for_new_screenshots.py

@@ -22,6 +22,7 @@
     "user_file_source_form_full_aws_public.png",
     "user_file_source_form_full_azure.png",
     "user_file_source_form_full_ftp.png",
+    "user_file_source_form_full_webdav.png",
 ]
 
 

lib/galaxy/config/schemas/config_schema.yml

@@ -3746,7 +3746,7 @@ mapping:
           Workflows launched with URI/URL inputs that are not marked as 'deferred'
           are "materialized" (or undeferred) by the workflow scheduler. This might be
           a lengthy process. Setting this to 'True' will place the invocation back in
-          the queue after materialization before scheduling the workflow so it is less 
+          the queue after materialization before scheduling the workflow so it is less
           likely to starve other workflow scheduling. Ideally, Galaxy would allow more
           fine grain control of handlers but until then, this provides a way to tip the
           balance between "doing more work" and "being more fair". The default here is
@@ -4110,7 +4110,7 @@ mapping:
         required: false
         desc: |
           Directory to store temporary files for file sources. This defaults to new_file_path if not set.
-      
+
       file_source_webdav_use_temp_files:
         type: bool
         default: true

lib/galaxy/files/templates/examples/production_webdav.yml

@@ -0,0 +1,42 @@
+- id: webdav
+  version: 0
+  name: WebDAV
+  description: |
+    The WebDAV protocol is a simple way to access files over the internet. This template
+    configuration allows you to connect to a WebDAV server.
+  variables:
+    url:
+      label: Server Domain (e.g. https://myowncloud.org)
+      type: string
+      help: |
+        The domain of the WebDAV server you are connecting to. This should be the full URL
+        including the protocol (http or https) and the domain name.
+    root:
+      label: WebDAV server Path (should end with /remote.php/webdav, e.g. /a/sub/path/remote.php/webdav)
+      type: string
+      help: |
+        The full server path to the WebDAV service. Ensure the path includes /remote.php/webdav.
+    login:
+      label: Username
+      type: string
+      help: |
+        The username to use to connect to the WebDAV server. This should be the username you use
+        to log in to the WebDAV server.
+    writable:
+      label: Writable?
+      type: boolean
+      default: false
+      help: Allow Galaxy to write data to this WebDAV server.
+  secrets:
+    password:
+      label: Password
+      help: |
+        The password to use to connect to the WebDAV server. This should be the password you use
+        to log in to the WebDAV server.
+  configuration:
+    type: webdav
+    url: '{{ variables.url }}'
+    root: '{{ variables.root }}'
+    login: '{{ variables.login }}'
+    writable: '{{ variables.writable }}'
+    password: '{{ secrets.password }}'

lib/galaxy/files/templates/models.py

@@ -25,7 +25,7 @@
     UserDetailsDict,
 )
 
-FileSourceTemplateType = Literal["ftp", "posix", "s3fs", "azure", "onedata"]
+FileSourceTemplateType = Literal["ftp", "posix", "s3fs", "azure", "onedata", "webdav"]
 
 
 class PosixFileSourceTemplateConfiguration(StrictModel):
@@ -122,19 +122,41 @@ class OnedataFileSourceConfiguration(StrictModel):
     writable: bool = False
 
 
+class WebdavFileSourceTemplateConfiguration(StrictModel):
+    type: Literal["webdav"]
+    url: Union[str, TemplateExpansion]
+    root: Union[str, TemplateExpansion]
+    login: Union[str, TemplateExpansion]
+    password: Union[str, TemplateExpansion]
+    writable: Union[bool, TemplateExpansion] = False
+    template_start: Optional[str] = None
+    template_end: Optional[str] = None
+
+
+class WebdavFileSourceConfiguration(StrictModel):
+    type: Literal["webdav"]
+    url: str
+    root: str
+    login: str
+    password: str
+    writable: bool = False
+
+
 FileSourceTemplateConfiguration = Union[
     PosixFileSourceTemplateConfiguration,
     S3FSFileSourceTemplateConfiguration,
     FtpFileSourceTemplateConfiguration,
     AzureFileSourceTemplateConfiguration,
     OnedataFileSourceTemplateConfiguration,
+    WebdavFileSourceTemplateConfiguration,
 ]
 FileSourceConfiguration = Union[
     PosixFileSourceConfiguration,
     S3FSFileSourceConfiguration,
     FtpFileSourceConfiguration,
     AzureFileSourceConfiguration,
     OnedataFileSourceConfiguration,
+    WebdavFileSourceConfiguration,
 ]
 
 
@@ -196,6 +218,7 @@ def template_to_configuration(
     "s3fs": S3FSFileSourceConfiguration,
     "azure": AzureFileSourceConfiguration,
     "onedata": OnedataFileSourceConfiguration,
+    "webdav": WebdavFileSourceConfiguration,
 }