From 32ce723b7633ccd190dd30f94d7ffa5fe9b355b2 Mon Sep 17 00:00:00 2001 From: bartvaneswhiffle <129189801+bartvaneswhiffle@users.noreply.github.com> Date: Mon, 12 Aug 2024 17:11:47 +0200 Subject: [PATCH] Add tls to ftp (#1581) --- fsspec/implementations/ftp.py | 21 +++++++++---- fsspec/implementations/reference.py | 8 +++-- fsspec/implementations/tests/ftp_tls.py | 38 ++++++++++++++++++++++++ fsspec/implementations/tests/keycert.pem | 24 +++++++++++++++ fsspec/implementations/tests/test_ftp.py | 29 ++++++++++++++++-- 5 files changed, 109 insertions(+), 11 deletions(-) create mode 100644 fsspec/implementations/tests/ftp_tls.py create mode 100644 fsspec/implementations/tests/keycert.pem diff --git a/fsspec/implementations/ftp.py b/fsspec/implementations/ftp.py index 9e245d39f..e56edd2ba 100644 --- a/fsspec/implementations/ftp.py +++ b/fsspec/implementations/ftp.py @@ -2,7 +2,7 @@ import sys import uuid import warnings -from ftplib import FTP, Error, error_perm +from ftplib import FTP, FTP_TLS, Error, error_perm from typing import Any from ..spec import AbstractBufferedFile, AbstractFileSystem @@ -27,6 +27,7 @@ def __init__( tempdir=None, timeout=30, encoding="utf-8", + tls=False, **kwargs, ): """ @@ -56,28 +57,38 @@ def __init__( Timeout of the ftp connection in seconds encoding: str Encoding to use for directories and filenames in FTP connection + tls: bool + Use FTP-TLS, by default False """ super().__init__(**kwargs) self.host = host self.port = port self.tempdir = tempdir or "/tmp" - self.cred = username, password, acct + self.cred = username or "", password or "", acct or "" + print(self.cred) self.timeout = timeout self.encoding = encoding if block_size is not None: self.blocksize = block_size else: self.blocksize = 2**16 + self.tls = tls self._connect() + if self.tls: + self.ftp.prot_p() def _connect(self): + if self.tls: + ftp_cls = FTP_TLS + else: + ftp_cls = FTP if sys.version_info >= (3, 9): - self.ftp = FTP(timeout=self.timeout, encoding=self.encoding) + self.ftp = ftp_cls(timeout=self.timeout, encoding=self.encoding) elif self.encoding: warnings.warn("`encoding` not supported for python<3.9, ignoring") - self.ftp = FTP(timeout=self.timeout) + self.ftp = ftp_cls(timeout=self.timeout) else: - self.ftp = FTP(timeout=self.timeout) + self.ftp = ftp_cls(timeout=self.timeout) self.ftp.connect(self.host, self.port) self.ftp.login(*self.cred) diff --git a/fsspec/implementations/reference.py b/fsspec/implementations/reference.py index c14ee8302..5340aaf1c 100644 --- a/fsspec/implementations/reference.py +++ b/fsspec/implementations/reference.py @@ -997,9 +997,11 @@ def _process_gen(self, gens): out = {} for gen in gens: dimension = { - k: v - if isinstance(v, list) - else range(v.get("start", 0), v["stop"], v.get("step", 1)) + k: ( + v + if isinstance(v, list) + else range(v.get("start", 0), v["stop"], v.get("step", 1)) + ) for k, v in gen["dimensions"].items() } products = ( diff --git a/fsspec/implementations/tests/ftp_tls.py b/fsspec/implementations/tests/ftp_tls.py new file mode 100644 index 000000000..6d1359bfa --- /dev/null +++ b/fsspec/implementations/tests/ftp_tls.py @@ -0,0 +1,38 @@ +import os + +from pyftpdlib.authorizers import DummyAuthorizer +from pyftpdlib.handlers import TLS_FTPHandler +from pyftpdlib.servers import FTPServer + + +def ftp(): + """Script to run FTP server that accepts TLS""" + # Set up FTP server parameters + FTP_HOST = "localhost" + FTP_PORT = 2121 + FTP_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) + + # Instantiate a dummy authorizer + authorizer = DummyAuthorizer() + authorizer.add_user( + "user", + "pass", + FTP_DIRECTORY, + "elradfmwMT", + ) + authorizer.add_anonymous(FTP_DIRECTORY) + + # Instantiate TLS_FTPHandler with required parameters + handler = TLS_FTPHandler + handler.certfile = os.path.join(os.path.dirname(__file__), "keycert.pem") + handler.authorizer = authorizer + + # Instantiate FTP server with TLS handler and authorizer + server = FTPServer((FTP_HOST, FTP_PORT), handler) + server.authorizer = authorizer + + server.serve_forever() + + +if __name__ == "__main__": + ftp() diff --git a/fsspec/implementations/tests/keycert.pem b/fsspec/implementations/tests/keycert.pem new file mode 100644 index 000000000..2093f1d15 --- /dev/null +++ b/fsspec/implementations/tests/keycert.pem @@ -0,0 +1,24 @@ +-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIBTg1e61mzYYPJ+MDkOWCSevnT1HUaaK9iopgTGyDoIuoAoGCCqGSM49 +AwEHoUQDQgAEDy3E+4WgohcRUlaSZBndEZQBTyoRztCSoaDbhZkqsPFBbeaGJ5zA +E7qX+9LICDezAUsCiq2RYltOqDCsELteiQ== +-----END EC PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICdzCCAh2gAwIBAgIUNN4kmTSxbLOoQXLFiYOs2XeK1jIwCgYIKoZIzj0EAwIw +gY8xCzAJBgNVBAYTAk5MMRUwEwYDVQQIDAxadWlkLUhvbGxhbmQxDjAMBgNVBAcM +BURlbGZ0MRAwDgYDVQQKDAdXaGlmZmxlMQ0wCwYDVQQLDARERVZBMRIwEAYDVQQD +DAlCYXJ0dmFuRXMxJDAiBgkqhkiG9w0BCQEWFWJhcnQudmFuZXNAd2hpZmZsZS5u +bDAgFw0yNDA0MTgxMDI0NDFaGA8yMjk4MDIwMTEwMjQ0MVowgY8xCzAJBgNVBAYT +Ak5MMRUwEwYDVQQIDAxadWlkLUhvbGxhbmQxDjAMBgNVBAcMBURlbGZ0MRAwDgYD +VQQKDAdXaGlmZmxlMQ0wCwYDVQQLDARERVZBMRIwEAYDVQQDDAlCYXJ0dmFuRXMx +JDAiBgkqhkiG9w0BCQEWFWJhcnQudmFuZXNAd2hpZmZsZS5ubDBZMBMGByqGSM49 +AgEGCCqGSM49AwEHA0IABA8txPuFoKIXEVJWkmQZ3RGUAU8qEc7QkqGg24WZKrDx +QW3mhiecwBO6l/vSyAg3swFLAoqtkWJbTqgwrBC7XomjUzBRMB0GA1UdDgQWBBRb +1nPqritk/P2cbDzTw9SQ9vO7JDAfBgNVHSMEGDAWgBRb1nPqritk/P2cbDzTw9SQ +9vO7JDAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0gAMEUCIBcvCFS4AD3p +Ix1v8pp3hcMvGFIQLeczh4kXkPfZWvBkAiEAiTCqsdKhZi8k814H6FFkaoQVIjTe +iUtUlW6RfyDNZ9E= +-----END CERTIFICATE----- diff --git a/fsspec/implementations/tests/test_ftp.py b/fsspec/implementations/tests/test_ftp.py index 7bf0c0e5d..e480ecaff 100644 --- a/fsspec/implementations/tests/test_ftp.py +++ b/fsspec/implementations/tests/test_ftp.py @@ -2,6 +2,7 @@ import subprocess import sys import time +from ftplib import FTP, FTP_TLS import pytest @@ -17,7 +18,7 @@ def ftp(): pytest.importorskip("pyftpdlib") P = subprocess.Popen( - [sys.executable, "-m", "pyftpdlib", "-d", here], + [sys.executable, os.path.join(here, "ftp_tls.py")], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, ) @@ -29,9 +30,31 @@ def ftp(): P.wait() -def test_basic(ftp): +@pytest.mark.parametrize( + "tls,exp_cls", + ( + (False, FTP), + (True, FTP_TLS), + ), +) +def test_tls(ftp, tls, exp_cls): host, port = ftp - fs = FTPFileSystem(host, port) + fs = FTPFileSystem(host, port, tls=tls) + assert isinstance(fs.ftp, exp_cls) + + +@pytest.mark.parametrize( + "tls,username,password", + ( + (False, "", ""), + (True, "", ""), + (False, "user", "pass"), + (True, "user", "pass"), + ), +) +def test_basic(ftp, tls, username, password): + host, port = ftp + fs = FTPFileSystem(host, port, username, password, tls=tls) assert fs.ls("/", detail=False) == sorted(os.listdir(here)) out = fs.cat(f"/{os.path.basename(__file__)}") assert out == open(__file__, "rb").read()