Skip to content

Commit

Permalink
fix: Remove ffspec from parquet reader (#15927)
Browse files Browse the repository at this point in the history
  • Loading branch information
ritchie46 authored Apr 27, 2024
1 parent 3564a77 commit 4e7a0e1
Show file tree
Hide file tree
Showing 2 changed files with 0 additions and 65 deletions.
45 changes: 0 additions & 45 deletions py-polars/polars/io/parquet/anonymous_scan.py

This file was deleted.

20 changes: 0 additions & 20 deletions py-polars/polars/io/parquet/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,10 @@
from polars.convert import from_arrow
from polars.dependencies import import_optional
from polars.io._utils import (
is_local_file,
is_supported_cloud,
parse_columns_arg,
parse_row_index_args,
prepare_file_arg,
)
from polars.io.parquet.anonymous_scan import _scan_parquet_fsspec

with contextlib.suppress(ImportError):
from polars.polars import PyDataFrame, PyLazyFrame
Expand Down Expand Up @@ -343,8 +340,6 @@ def scan_parquet(
Cache the result after reading.
storage_options
Options that indicate how to connect to a cloud provider.
If the cloud provider is not supported by Polars, the storage options
are passed to `fsspec.open()`.
The cloud providers currently supported are AWS, GCP, and Azure.
See supported keys here:
Expand Down Expand Up @@ -425,24 +420,9 @@ def _scan_parquet_impl(
if isinstance(source, list):
sources = source
source = None # type: ignore[assignment]
can_use_fsspec = False
else:
can_use_fsspec = True
sources = []

# try fsspec scanner
if (
can_use_fsspec
and not is_local_file(source) # type: ignore[arg-type]
and not is_supported_cloud(source) # type: ignore[arg-type]
):
scan = _scan_parquet_fsspec(source, storage_options) # type: ignore[arg-type]
if n_rows:
scan = scan.head(n_rows)
if row_index_name is not None:
scan = scan.with_row_index(row_index_name, row_index_offset)
return scan

if storage_options:
storage_options = list(storage_options.items()) # type: ignore[assignment]
else:
Expand Down

0 comments on commit 4e7a0e1

Please sign in to comment.