[FSTORE-1533] Importing hopsworks fails due to core dump while import…

…ing polars (#1379)

java/beam/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>3.8.0-RC0</version>
+    <version>3.8.0-RC1</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/flink/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>3.8.0-RC0</version>
+    <version>3.8.0-RC1</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/hsfs/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <artifactId>hsfs-parent</artifactId>
     <groupId>com.logicalclocks</groupId>
-    <version>3.8.0-RC0</version>
+    <version>3.8.0-RC1</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/pom.xml

@@ -7,7 +7,7 @@
     <groupId>com.logicalclocks</groupId>
     <artifactId>hsfs-parent</artifactId>
     <packaging>pom</packaging>
-    <version>3.8.0-RC0</version>
+    <version>3.8.0-RC1</version>
     <modules>
         <module>hsfs</module>
         <module>spark</module>

java/spark/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>hsfs-parent</artifactId>
         <groupId>com.logicalclocks</groupId>
-        <version>3.8.0-RC0</version>
+        <version>3.8.0-RC1</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

python/hsfs/core/arrow_flight_client.py

@@ -23,19 +23,23 @@
 from functools import wraps
 from typing import Any, Dict, Optional, Union
 
-import polars as pl
 import pyarrow
 import pyarrow._flight
 import pyarrow.flight
 from hsfs import client, feature_group, util
 from hsfs.client.exceptions import FeatureStoreException
 from hsfs.constructor import query
+from hsfs.core.constants import HAS_POLARS, polars_not_installed_message
 from hsfs.core.variable_api import VariableApi
 from hsfs.storage_connector import StorageConnector
 from pyarrow.flight import FlightServerError
 from retrying import retry
 
 
+if HAS_POLARS:
+    import polars as pl
+
+
 _logger = logging.getLogger(__name__)
 
 
@@ -399,6 +403,8 @@ def _get_dataset(self, descriptor, timeout=None, dataframe_type="pandas"):
         reader = self._connection.do_get(info.endpoints[0].ticket, options)
         _logger.debug("Dataset fetched. Converting to dataframe %s.", dataframe_type)
         if dataframe_type.lower() == "polars":
+            if not HAS_POLARS:
+                raise ModuleNotFoundError(polars_not_installed_message)
             return pl.from_arrow(reader.read_all())
         else:
             return reader.read_pandas()

python/hsfs/core/constants.py

@@ -0,0 +1,28 @@
+#
+#   Copyright 2024 Hopsworks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import importlib.util
+
+
+polars_not_installed_message = (
+    "Polars package not found. "
+    "If you want to use Polars with Hopsworks you can install the corresponding extras "
+    """`pip install hopsworks[polars]` or `pip install "hopsworks[polars]"` if using zsh. """
+    "You can also install polars directly in your environment e.g `pip install polars`. "
+    "You will need to restart your kernel if applicable."
+)
+
+HAS_POLARS: bool = importlib.util.find_spec("polars") is not None

python/hsfs/core/vector_server.py

@@ -20,13 +20,22 @@
 from base64 import b64decode
 from datetime import datetime, timezone
 from io import BytesIO
-from typing import Any, Callable, Dict, List, Literal, Optional, Set, Tuple, Union
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
 
 import avro.io
 import avro.schema
 import numpy as np
 import pandas as pd
-import polars as pl
 from hsfs import (
     client,
     feature_view,
@@ -49,6 +58,7 @@
 from hsfs.core import (
     transformation_function_engine as tf_engine_mod,
 )
+from hsfs.core.constants import HAS_POLARS, polars_not_installed_message
 
 
 HAS_FASTAVRO = False
@@ -59,6 +69,9 @@
 except ImportError:
     from avro.io import BinaryDecoder
 
+if HAS_POLARS:
+    import polars as pl
+
 _logger = logging.getLogger(__name__)
 
 
@@ -487,6 +500,9 @@ def handle_feature_vector_return_type(
                 return pandas_df
         elif return_type.lower() == "polars":
             _logger.debug("Returning feature vector as polars dataframe")
+            if not HAS_POLARS:
+                raise ModuleNotFoundError(polars_not_installed_message)
+
             return pl.DataFrame(
                 feature_vectorz if batch else [feature_vectorz],
                 schema=self._feature_vector_col_name if not inference_helper else None,