Merge branch 'fsspec:master' into master

.github/workflows/main.yaml

@@ -25,19 +25,15 @@ jobs:
           fetch-depth: 0
 
       - name: Setup conda
-        uses: mamba-org/setup-micromamba@v1
+        uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: ci/environment-py38.yml
-          create-args: >-
-            python=${{ matrix.PY }}
+          python-version: ${{ matrix.PY }}
 
       - name: Run Tests
         shell: bash -l {0}
         run: |
-          pip install s3fs
-          pip uninstall s3fs
           pip install -e .[test_full]
-          pip install s3fs --no-deps
           pytest -v
 
   win:
@@ -54,17 +50,14 @@ jobs:
           fetch-depth: 0
 
       - name: Setup conda
-        uses: mamba-org/setup-micromamba@v1
+        uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: ci/environment-win.yml
 
       - name: Run Tests
         shell: bash -l {0}
         run: |
-          pip install s3fs
-          pip uninstall s3fs
           pip install -e .[test]
-          pip install s3fs --no-deps
           pytest -v
 
   lint:
@@ -84,7 +77,7 @@ jobs:
 #        uses: actions/checkout@v4
 #
 #      - name: Setup conda
-#        uses: mamba-org/setup-micromamba@v1
+#        uses: conda-incubator/setup-miniconda@v3
 #        with:
 #          environment-file: ci/environment-typecheck.yml
 #
@@ -104,7 +97,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup conda
-        uses: mamba-org/setup-micromamba@v1
+        uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: ci/environment-downstream.yml
 
@@ -145,7 +138,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup conda
-        uses: mamba-org/setup-micromamba@v1
+        uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: ci/environment-friends.yml
 

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
       - id: check-yaml
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.3.4
+    rev: v0.4.4
     hooks:
       # Run the linter.
       - id: ruff

README.md

@@ -4,7 +4,6 @@
 [![Anaconda-Server Badge](https://anaconda.org/conda-forge/fsspec/badges/version.svg)](https://anaconda.org/conda-forge/fsspec)
 ![Build](https://github.com/fsspec/filesystem_spec/workflows/CI/badge.svg)
 [![Docs](https://readthedocs.org/projects/filesystem-spec/badge/?version=latest)](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
-[![PyPi downloads](https://img.shields.io/pypi/dm/fsspec?label=pypi%20downloads&style=flat)](https://pepy.tech/project/fsspec)
 
 A specification for pythonic filesystems.
 
@@ -51,13 +50,13 @@ CI runtime. For local use, pick a version suitable for you.
 mamba create -n fsspec -c conda-forge  python=3.9 -y
 conda activate fsspec
 
-# Standard dev test install.
-pip install -e ".[dev,test]"
+# Standard dev install with docs and tests.
+pip install -e ".[dev,doc,test]"
 
 # Full tests except for downstream
 pip install s3fs
 pip uninstall s3fs
-pip install -e .[dev,test_full]
+pip install -e .[dev,doc,test_full]
 pip install s3fs --no-deps
 pytest -v
 

ci/environment-downstream.yml

@@ -2,6 +2,6 @@ name: test_env
 channels:
   - conda-forge
 dependencies:
-  - python=3.9
+  - python=3.11
   - pip:
       - git+https://github.com/dask/dask

ci/environment-friends.yml

@@ -2,7 +2,7 @@ name: test_env
 channels:
   - conda-forge
 dependencies:
-  - python=3.9
+  - python=3.12
   - pytest
   - pytest-asyncio !=0.22.0
   - pytest-benchmark

ci/environment-py38.yml

@@ -3,8 +3,9 @@ channels:
   - conda-forge
 dependencies:
   - pip
-  - git
+  - git <2.45.0
   - py
+  - s3fs
   - pip:
     - hadoop-test-cluster
     - smbprotocol

ci/environment-win.yml

@@ -2,4 +2,6 @@ name: test_env
 channels:
   - conda-forge
 dependencies:
-  - python=3.9
+  - python=3.11
+  - s3fs
+  - pytest

docs/environment.yml

@@ -3,9 +3,3 @@ channels:
   - defaults
 dependencies:
   - python=3.9
-  - docutils<0.17
-  - numpydoc
-  - sphinx_rtd_theme
-  - yarl
-  - pip:
-    - sphinx-design

docs/source/api.rst

@@ -226,6 +226,7 @@ Other Known Implementations
 - `s3fs`_ for Amazon S3 and other compatible stores
 - `wandbfs`_ to access Wandb run data (experimental)
 - `webdav4`_ for WebDAV
+- `xrootd`_ for xrootd, with protocol "root://"
 
 .. _abfs: https://github.com/dask/adlfs
 .. _adl: https://github.com/dask/adlfs
@@ -244,6 +245,7 @@ Other Known Implementations
 .. _s3fs: https://s3fs.readthedocs.io/en/latest/
 .. _wandbfs: https://github.com/jkulhanek/wandbfs
 .. _webdav4: https://github.com/skshetry/webdav4
+.. _xrootd: https://github.com/CoffeaTeam/fsspec-xrootd
 
 .. _readbuffering:
 

docs/source/changelog.rst

@@ -1,6 +1,66 @@
 Changelog
 =========
 
+2024.6.1
+--------
+
+Fixes
+
+- fix appending to non-dict reference sets (#1634)
+- don't let generic edit info dicts (#1633)
+- set https's loop before calling super (#1633)
+- cached write file doesn't need to update it's size on close (#1633)
+- fix JSON serialize for FSs with interior FSs (#1628, 1627)
+- option to strip "password" when pickling (#1625)
+- fix filecache write (#1622)
+
+
+2024.6.0
+--------
+
+Enhancements
+
+- allow dicts (not just bytes) for referenceFS (#1616
+- make filesystems JSON serializeable (#1612)
+- implement multifile cat() for github (#1620)
+
+Fixes
+
+- implement auto_mkdir for SMB (#1604)
+
+Other
+
+- add doc deps to pyproject (#1613)
+- re-remove test from package (#1611)
+- formatting (#1610, 1608, 1602)
+- change monthly download badge (#1607)
+
+2024.5.0
+--------
+
+Enhancements
+
+- log hits/misses in bytes cachers (#1566)
+
+Fixes
+
+- SMB flaky tests (#1597)
+- rsync: only delete files if there are some to delete (#1596)
+- don't let files equal bytes objects (#1594)
+- url_to_fs to stringify paths (#1591)
+- assert types in MemoryFS (#1574)
+- restore _strip_protocol signature for local (#1567)
+- convert list to set when loading cache metadata (#1556)
+
+Other
+
+- remove mv_file (#1585)
+- mv() should not swallow errors (#1576)
+- change versioning template, allows easier co-install of dev s3fs (#1569)
+- in ls_from_cache, avoid dounble lookup (#1561)
+- expand=True in open() (#1558)
+- build system to hatch (#1553)
+
 2024.3.1
 --------
 

docs/source/developer.rst

@@ -107,9 +107,9 @@ The following can be used to install ``fsspec`` in development mode
 
    git clone https://github.com/fsspec/filesystem_spec
    cd filesystem_spec
-   pip install -e .
+   pip install -e .[dev,doc,test]
 
-A number of additional dependencies are required to run tests, see "ci/environment*.yml", as
+A number of additional dependencies are required to run tests in full, see "ci/environment*.yml", as
 well as Docker. Most implementation-specific tests should skip if their requirements are
 not met.
 

docs/source/index.rst

@@ -69,7 +69,7 @@ The following libraries use ``fsspec`` internally for path and file handling:
 .. _xarray: https://docs.xarray.dev/
 .. _zarr: https://zarr.readthedocs.io/
 .. _DVC: https://dvc.org/
-.. _kedro: https://kedro.readthedocs.io/en/stable/01_introduction/01_introduction.html
+.. _kedro: https://docs.kedro.org/en/stable/tutorial/set_up_data.html#supported-data-locations
 .. _pyxet: https://github.com/xetdata/pyxet
 .. _Huggingface🤗 Datasets: https://github.com/huggingface/datasets
 .. _pyarrow: https://arrow.apache.org/docs/python/

fsspec/caching.py

@@ -15,6 +15,7 @@
     ClassVar,
     Generic,
     NamedTuple,
+    Optional,
     OrderedDict,
     TypeVar,
 )
@@ -574,7 +575,7 @@ def __init__(
         blocksize: int,
         fetcher: Fetcher,
         size: int,
-        data: dict[tuple[int, int], bytes] = {},
+        data: Optional[dict[tuple[int, int], bytes]] = None,
         strict: bool = True,
         **_: Any,
     ):
@@ -597,7 +598,7 @@ def __init__(
 
             self.data = dict(zip(offsets, blocks))
         else:
-            self.data = data
+            self.data = {}
 
     def _fetch(self, start: int | None, stop: int | None) -> bytes:
         if start is None:

fsspec/compression.py

@@ -139,7 +139,7 @@ def _fetch_range(self, start, end):
 try:
     import snappy
 
-    snappy.compress
+    snappy.compress(b"")
     # Snappy may use the .sz file extension, but this is not part of the
     # standard implementation.
     register_compression("snappy", SnappyFile, [])

fsspec/core.py

@@ -379,6 +379,7 @@ def url_to_fs(url, **kwargs):
     urlpath : str
         The file-systems-specific URL for ``url``.
     """
+    url = stringify_path(url)
     # non-FS arguments that appear in fsspec.open()
     # inspect could keep this in sync with open()'s signature
     known_kwargs = {
@@ -638,7 +639,7 @@ def get_fs_token_paths(
     if isinstance(urlpath, (list, tuple, set)):
         if not urlpath:
             raise ValueError("empty urlpath sequence")
-        urlpath0 = stringify_path(list(urlpath)[0])
+        urlpath0 = stringify_path(next(iter(urlpath)))
     else:
         urlpath0 = stringify_path(urlpath)
     storage_options = storage_options or {}

fsspec/fuse.py

@@ -31,8 +31,8 @@ def getattr(self, path, fh=None):
         path = "".join([self.root, path.lstrip("/")]).rstrip("/")
         try:
             info = self.fs.info(path)
-        except FileNotFoundError:
-            raise FuseOSError(ENOENT)
+        except FileNotFoundError as exc:
+            raise FuseOSError(ENOENT) from exc
 
         data = {"st_uid": info.get("uid", 1000), "st_gid": info.get("gid", 1000)}
         perm = info.get("mode", 0o777)
@@ -119,8 +119,8 @@ def unlink(self, path):
         fn = "".join([self.root, path.lstrip("/")])
         try:
             self.fs.rm(fn, False)
-        except (OSError, FileNotFoundError):
-            raise FuseOSError(EIO)
+        except (OSError, FileNotFoundError) as exc:
+            raise FuseOSError(EIO) from exc
 
     def release(self, path, fh):
         try:

fsspec/generic.py

@@ -139,7 +139,7 @@ def rsync(
         source_files, target_files = zip(*allfiles.items())
         fs.cp(source_files, target_files, **kwargs)
     logger.debug(f"{len(to_delete)} files to delete")
-    if delete_missing:
+    if delete_missing and to_delete:
         fs.rm(to_delete)
     return allfiles
 
@@ -197,6 +197,7 @@ async def _find(self, path, maxdepth=None, withdirs=False, detail=False, **kwarg
             )
         result = {}
         for k, v in out.items():
+            v = v.copy()  # don't corrupt target FS dircache
             name = fs.unstrip_protocol(k)
             v["name"] = name
             result[name] = v
@@ -210,6 +211,7 @@ async def _info(self, url, **kwargs):
             out = await fs._info(url, **kwargs)
         else:
             out = fs.info(url, **kwargs)
+        out = out.copy()  # don't edit originals
         out["name"] = fs.unstrip_protocol(out["name"])
         return out
 
@@ -224,6 +226,7 @@ async def _ls(
             out = await fs._ls(url, detail=True, **kwargs)
         else:
             out = fs.ls(url, detail=True, **kwargs)
+        out = [o.copy() for o in out]  # don't edit originals
         for o in out:
             o["name"] = fs.unstrip_protocol(o["name"])
         if detail:

fsspec/gui.py

@@ -93,8 +93,10 @@ def _repr_mimebundle_(self, *args, **kwargs):
         """Display in a notebook or a server"""
         try:
             return self.panel._repr_mimebundle_(*args, **kwargs)
-        except (ValueError, AttributeError):
-            raise NotImplementedError("Panel does not seem to be set " "up properly")
+        except (ValueError, AttributeError) as exc:
+            raise NotImplementedError(
+                "Panel does not seem to be set up properly"
+            ) from exc
 
     def connect(self, signal, slot):
         """Associate call back with given event

fsspec/implementations/arrow.py

@@ -139,8 +139,6 @@ def mv(self, path1, path2, **kwargs):
         path2 = self._strip_protocol(path2).rstrip("/")
         self.fs.move(path1, path2)
 
-    mv_file = mv
-
     @wrap_exceptions
     def rm_file(self, path):
         path = self._strip_protocol(path)