[hailctl] update to dataproc 2.2 and Spark 3.5.0 (#14158)

Fixes #13971 CHANGELOG: Hail now supports and primarily tests against Dataproc 2.2.5, Spark 3.5.0, and Java 11. We strongly recommend updating to Spark 3.5.0 and Java 11. You should also update your GCS connector *after installing Hail*: `curl https://broad.io/install-gcs-connector | python3`. Do not try to update before installing Hail 0.2.131. https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-release-2.2 --------- Co-authored-by: Edmund Higham <[email protected]>
hail-is · Apr 11, 2024 · bd0156d · bd0156d
1 parent dcb83f6
commit bd0156d
Show file tree

Hide file tree

Showing 29 changed files with 80 additions and 108 deletions.
diff --git a/batch/Dockerfile.worker b/batch/Dockerfile.worker
@@ -6,7 +6,7 @@ RUN hail-apt-get-install \
     iproute2 \
     iptables \
     ca-certificates-java \
-    openjdk-8-jre-headless \
+    openjdk-11-jre-headless \
     liblapack3 \
     xfsprogs \
     libyajl-dev
@@ -51,7 +51,7 @@ RUN hail-pip-install \
     -r hailtop-requirements.txt \
     -r gear-requirements.txt \
     -r batch-requirements.txt \
-    pyspark==3.3.2
+    pyspark==3.5.0
 
 ENV SPARK_HOME /usr/local/lib/python3.9/dist-packages/pyspark
 ENV PATH "$PATH:$SPARK_HOME/sbin:$SPARK_HOME/bin"

diff --git a/batch/jvm-entryway/build.gradle b/batch/jvm-entryway/build.gradle
@@ -17,7 +17,7 @@ repositories {
 }
 
 project.ext {
-    sparkVersion = System.getProperty("spark.version", "3.3.2")
+    sparkVersion = System.getProperty("spark.version", "3.5.0")
     scalaVersion = System.getProperty("scala.version", "2.12.18")
 }
 

diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base
@@ -8,7 +8,7 @@ RUN hail-apt-get-install \
     rsync \
     emacs-nox \
     xsltproc pandoc \
-    openjdk-8-jdk-headless \
+    openjdk-11-jdk-headless \
     liblapack3 \
     liblz4-dev \
     g++-10 \

diff --git a/docker/core-site.xml b/docker/core-site.xml
@@ -4,13 +4,13 @@
 <configuration>
 
   <property>
-    <name>google.cloud.auth.service.account.enable</name>
-    <value>true</value>
+    <name>google.cloud.auth.type</name>
+    <value>SERVICE_ACCOUNT_JSON_KEYFILE</value>
   </property>
-  
+
   <property>
     <name>google.cloud.auth.service.account.json.keyfile</name>
     <value>/gsa-key/key.json</value>
   </property>
-  
+
 </configuration>
diff --git a/docker/hailgenetics/hail/Dockerfile b/docker/hailgenetics/hail/Dockerfile
@@ -7,14 +7,15 @@ FROM $BASE_IMAGE
 RUN hail-apt-get-install \
     git \
     liblapack3 \
-    openjdk-8-jre-headless
+    openjdk-11-jre-headless
 
 COPY hail/python/pinned-requirements.txt requirements.txt
 RUN hail-pip-install -r requirements.txt scikit-learn ipython
 
 RUN export SPARK_HOME=$(find_spark_home.py) && \
-    curl https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop2-2.2.7.jar \
-         >$SPARK_HOME/jars/gcs-connector-hadoop2-2.2.7.jar && \
+    curl --fail --silent --show-error --location \
+         https://repo1.maven.org/maven2/com/google/cloud/bigdataoss/gcs-connector/3.0.0/gcs-connector-3.0.0-shaded.jar \
+         >${SPARK_HOME}/jars/gcs-connector-3.0.0-shaded.jar && \
     mkdir -p $SPARK_HOME/conf && \
     touch $SPARK_HOME/conf/spark-defaults.conf && \
     sed -i $SPARK_HOME/conf/spark-defaults.conf \

diff --git a/hail/Dockerfile.hail-run b/hail/Dockerfile.hail-run
@@ -18,5 +18,7 @@ ENV PATH "$PATH:$SPARK_HOME/sbin:$SPARK_HOME/bin"
 ENV PYSPARK_PYTHON python3
 ENV PYSPARK_SUBMIT_ARGS "--conf spark.kryo.registrator=is.hail.kryo.HailKryoRegistrator pyspark-shell"
 
-RUN curl >${SPARK_HOME}/jars/gcs-connector-hadoop2-2.2.7.jar https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop2-2.2.7.jar
+RUN curl --fail --silent --show-error --location \
+         https://repo1.maven.org/maven2/com/google/cloud/bigdataoss/gcs-connector/3.0.0/gcs-connector-3.0.0-shaded.jar \
+         >${SPARK_HOME}/jars/gcs-connector-3.0.0-shaded.jar
 COPY docker/core-site.xml ${SPARK_HOME}/conf/core-site.xml
diff --git a/hail/Makefile b/hail/Makefile
@@ -366,7 +366,6 @@ install-editable: $(FAST_PYTHON_JAR) $(FAST_PYTHON_JAR_EXTRA_CLASSPATH)
 .PHONY: install-for-qob
 install-for-qob: upload-qob-jar install-editable
 	! [ -z $(NAMESPACE) ]  # call this like: make install-for-qob NAMESPACE=default
-	$(JAVAC) -version 2>&1 | grep -e '1\.8\.0'  # install-for-qob requires javac version 1.8, see https://discuss.hail.is/t/on-mac-os-x-how-do-i-install-and-use-java-8-if-i-already-have-a-different-version-of-java-installed/831/2
 	hailctl config set query/backend batch
 	hailctl config set query/jar_url $$(cat upload-qob-jar)
 	hailctl dev config set default_namespace $(NAMESPACE)

diff --git a/hail/build.sc b/hail/build.sc
@@ -38,7 +38,7 @@ def javaVersion: T[String] = T.input {
 }
 
 def sparkVersion: T[String] = T.input {
-  Result.Success(T.ctx().env.getOrElse("SPARK_VERSION", "3.3.0"))
+  Result.Success(T.ctx().env.getOrElse("SPARK_VERSION", "3.5.0"))
 }
 
 def debugMode: T[Boolean] = T.input {
@@ -79,6 +79,10 @@ object Deps {
   }
 
   object Breeze {
+    // WARNING WARNING WARNING
+    // Before changing the breeze version review:
+    // - https://hail.zulipchat.com/#narrow/stream/123011-Hail-Query-Dev/topic/new.20spark.20ndarray.20failures/near/41645
+    // - https://github.com/hail-is/hail/pull/11555
     val core = ivy"org.scalanlp::breeze:1.1"
     val natives = ivy"org.scalanlp::breeze-natives:1.1"
   }
@@ -210,8 +214,8 @@ object main extends RootModule with HailScalaModule { outer =>
   override def compileIvyDeps: T[Agg[Dep]] = Agg(
     Deps.log4j,
     Deps.hadoopClient,
-    Deps.Spark.core(),
-    Deps.Spark.mllib(),
+    Deps.Spark.core().excludeOrg("org.scalanlp"),  // Hail has an explicit dependency on Breeze 1.1
+    Deps.Spark.mllib().excludeOrg("org.scalanlp"),  // Hail has an explicit dependency on Breeze 1.1
     Deps.Breeze.core,
   )
 

diff --git a/hail/python/dataproc-pre-installed-requirements.txt b/hail/python/dataproc-pre-installed-requirements.txt
@@ -1,4 +1,4 @@
-# https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-release-2.1
+# https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-release-2.2
 #
-# 2.1.33-debian11
-pyspark==3.3.2
+# 2.2.5-debian12
+pyspark==3.5.0
diff --git a/hail/python/dev/requirements.txt b/hail/python/dev/requirements.txt
@@ -14,7 +14,7 @@ pytest-instafail>=0.4.2,<1
 pytest-asyncio>=0.14.0,<0.23
 pytest-timestamper>=0.0.9,<1
 pytest-timeout>=2.1,<3
-pyright>=1.1.324<1.2
+pyright>=1.1.349<1.2
 sphinx>=6,<7
 sphinx-autodoc-typehints==1.23.0
 nbsphinx>=0.8.8,<1

diff --git a/hail/python/hail/backend/local_backend.py b/hail/python/hail/backend/local_backend.py
@@ -1,3 +1,5 @@
+import glob
+import logging
 import os
 import sys
 from contextlib import ExitStack
@@ -17,6 +19,8 @@
 from .backend import local_jar_information
 from .py4j_backend import Py4JBackend, uninstall_exception_handler
 
+log = logging.getLogger('hail.backend')
+
 
 class LocalBackend(Py4JBackend):
     def __init__(
@@ -49,12 +53,18 @@ def __init__(
         if jvm_heap_size is not None:
             jvm_opts.append(f'-Xmx{jvm_heap_size}')
 
+        py4j_jars = glob.glob(f'{spark_home}/jars/py4j-*.jar')
+        if len(py4j_jars) == 0:
+            raise ValueError(f'No py4j JAR found in {spark_home}/jars')
+        if len(py4j_jars) > 1:
+            log.warning(f'found multiple p4yj jars arbitrarily choosing the first one: {py4j_jars}')
+
         port = launch_gateway(
             redirect_stdout=sys.stdout,
             redirect_stderr=sys.stderr,
             java_path=None,
             javaopts=jvm_opts,
-            jarpath=f'{spark_home}/jars/py4j-0.10.9.5.jar',
+            jarpath=py4j_jars[0],
             classpath=extra_classpath,
             die_on_exit=True,
         )

diff --git a/hail/python/hail/docs/getting_started_developing.rst b/hail/python/hail/docs/getting_started_developing.rst
@@ -6,11 +6,8 @@ Hail is an open-source project. We welcome contributions to the repository.
 Requirements
 ~~~~~~~~~~~~
 
-- `Java 8 or 11 JDK <https://adoptopenjdk.net/index.html>`_ . If you have a Mac, you must use a
-  compatible architecture (``uname -m`` prints your architecture). Moreover, you *must* use Java
-  **8** or **11**. Hail does not support other versions because `Spark does not support other
-  versions
-  <https://spark.apache.org/docs/3.4.0/#:%7E:text=Spark%20runs%20on%20Java%208,3.6%2B%20and%20R%203.5%2B.>`__.
+- `Java 11 JDK <https://adoptopenjdk.net/index.html>`_ . If you have a Mac, you must use a
+  compatible architecture (``uname -m`` prints your architecture).
 
 - The Python and non-pip installation requirements in `Getting Started <getting_started.html>`_.
   Note: These instructions install the JRE but that is not necessary as the JDK should already

diff --git a/hail/python/hail/docs/install/linux.rst b/hail/python/hail/docs/install/linux.rst
@@ -2,7 +2,7 @@
 Install Hail on GNU/Linux
 =========================
 
-- Install Java 8 or Java 11.
+- Install Java 11.
 - Install Python 3.9 or later.
 - Install a recent version of the C and C++ standard libraries. GCC 5.0, LLVM
   version 3.4, or any later versions suffice.
@@ -14,7 +14,7 @@ On a recent Debian-like system, the following should suffice:
 .. code-block:: sh
 
    apt-get install -y \
-       openjdk-8-jre-headless \
+       openjdk-11-jre-headless \
        g++ \
        python3.9 python3-pip \
        libopenblas-base liblapack3

diff --git a/hail/python/hail/docs/install/macosx.rst b/hail/python/hail/docs/install/macosx.rst
@@ -2,8 +2,8 @@
 Install Hail on Mac OS X
 ========================
 
-- Install Java 8 or 11. We recommend using a `packaged installation from Azul
-  <https://www.azul.com/downloads/?version=java-8-lts&os=macos&package=jdk&show-old-builds=true>`__
+- Install Java 11. We recommend using a `packaged installation from Azul
+  <https://www.azul.com/downloads/?version=java-11-lts&os=macos&package=jdk&show-old-builds=true>`__
   (make sure the OS version and architecture match your system) or using `Homebrew
   <https://brew.sh/>`__:
 

diff --git a/hail/python/hail/docs/install/other-cluster.rst b/hail/python/hail/docs/install/other-cluster.rst
@@ -5,12 +5,12 @@ Install Hail on a Spark Cluster
 If you are using Google Dataproc, please see `these simpler instructions <dataproc.rst>`__. If you
 are using Azure HDInsight please see `these simpler instructions <azure.rst>`__.
 
-Hail should work with any Spark 3.3.x cluster built with Scala 2.12.
+Hail should work with any Spark 3.5.x cluster built with Scala 2.12.
 
 Hail needs to be built from source on the leader node. Building Hail from source
 requires:
 
-- Java 8 or 11 JDK.
+- Java 11 JDK.
 - Python 3.9 or later.
 - A recent C and a C++ compiler, GCC 5.0, LLVM 3.4, or later versions of either
   suffice.
@@ -23,7 +23,7 @@ On a Debian-like system, the following should suffice:
 
    apt-get update
    apt-get install \
-       openjdk-8-jdk-headless \
+       openjdk-11-jdk-headless \
        g++ \
        python3 python3-pip \
        libopenblas-dev liblapack-dev \
@@ -36,7 +36,7 @@ The next block of commands downloads, builds, and installs Hail from source.
 
     git clone https://github.com/hail-is/hail.git
     cd hail/hail
-    make install-on-cluster HAIL_COMPILE_NATIVES=1 SCALA_VERSION=2.12.18 SPARK_VERSION=3.3.2
+    make install-on-cluster HAIL_COMPILE_NATIVES=1 SCALA_VERSION=2.12.18 SPARK_VERSION=3.5.0
 
 If you forget to install any of the requirements before running `make install-on-cluster`, it's possible
 to get into a bad state where `make` insists you don't have a requirement that you have in fact installed.

diff --git a/hail/python/hail/expr/expressions/typed_expressions.py b/hail/python/hail/expr/expressions/typed_expressions.py
@@ -2944,7 +2944,7 @@ def replace(self, pattern1, pattern2):
         -----
 
         The regex expressions used should follow `Java regex syntax
-        <https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html>`_. In
+        <https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/regex/Pattern.html>`_. In
         the Java regular expression syntax, a dollar sign, ``$1``, refers to the
         first group, not the canonical ``\\1``.
 
@@ -2974,7 +2974,7 @@ def split(self, delim, n=None):
         Notes
         -----
         The delimiter is a regex using the
-        `Java regex syntax <https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html>`_
+        `Java regex syntax <https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/regex/Pattern.html>`_
         delimiter. To split on special characters, escape them with double
         backslash (``\\\\``).
 
@@ -3222,7 +3222,7 @@ def matches(self, regex, full_match=False):
         The `regex` argument is a
         `regular expression <https://en.wikipedia.org/wiki/Regular_expression>`__,
         and uses
-        `Java regex syntax <https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html>`__.
+        `Java regex syntax <https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/regex/Pattern.html>`__.
 
         Parameters
         ----------

diff --git a/hail/python/hail/expr/functions.py b/hail/python/hail/expr/functions.py
@@ -6581,7 +6581,7 @@ def format(f, *args):
 
     Notes
     -----
-    See the `Java documentation <https://docs.oracle.com/javase/8/docs/api/java/lang/String.html#format-java.lang.String-java.lang.Object...->`__
+    See the `Java documentation <https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/Formatter.html#syntax>`__
     for valid format specifiers and arguments.
 
     Missing values are printed as ``'null'`` except when using the
@@ -6590,7 +6590,7 @@ def format(f, *args):
     Parameters
     ----------
     f : :class:`.StringExpression`
-        Java `format string <https://docs.oracle.com/javase/8/docs/api/java/util/Formatter.html#syntax>`__.
+        Java `format string <https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/Formatter.html#syntax>`__.
     args : variable-length arguments of :class:`.Expression`
         Arguments to format.
 

diff --git a/hail/python/hail/methods/impex.py b/hail/python/hail/methods/impex.py
@@ -1033,7 +1033,7 @@ def grep(regex, path, max_count=100, *, show: bool = True, force: bool = False,
     convenience to those in the statistical genetics community who often
     search enormous text files like VCFs. Hail uses `Java regular expression
     patterns
-    <https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html>`__.
+    <https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/regex/Pattern.html>`__.
     The `RegExr sandbox <http://regexr.com/>`__ may be helpful.
 
     Parameters
@@ -1747,7 +1747,7 @@ def import_table(
     find_replace : (:class:`str`, :obj:`str`)
         Line substitution regex. Functions like ``re.sub``, but obeys the exact
         semantics of Java's
-        `String.replaceAll <https://docs.oracle.com/javase/8/docs/api/java/lang/String.html#replaceAll-java.lang.String-java.lang.String->`__.
+        `String.replaceAll <https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/String.html#replaceAll(java.lang.String,java.lang.String)>`__.
     force : :obj:`bool`
         If ``True``, load gzipped files serially on one core. This should
         be used only when absolutely necessary, as processing time will be
@@ -3030,7 +3030,7 @@ def import_vcf(
     find_replace : (:class:`str`, :obj:`str`)
         Line substitution regex. Functions like ``re.sub``, but obeys the exact
         semantics of Java's
-        `String.replaceAll <https://docs.oracle.com/javase/8/docs/api/java/lang/String.html#replaceAll-java.lang.String-java.lang.String->`__.
+        `String.replaceAll <https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/String.html#replaceAll(java.lang.String,java.lang.String)>`__.
     n_partitions : :obj:`int`, optional
         Number of partitions.  If both `n_partitions` and `block_size`
         are specified, `n_partitions` will be used.
@@ -3517,7 +3517,7 @@ def import_csv(
     find_replace : (:class:`str`, :obj:`str`)
         Line substitution regex. Functions like ``re.sub``, but obeys the exact
         semantics of Java's
-        `String.replaceAll <https://docs.oracle.com/javase/8/docs/api/java/lang/String.html#replaceAll-java.lang.String-java.lang.String->`__.
+        `String.replaceAll <https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/String.html#replaceAll(java.lang.String,java.lang.String)>`__.
     force : :obj:`bool`
         If ``True``, load gzipped files serially on one core. This should
         be used only when absolutely necessary, as processing time will be

diff --git a/hail/python/hailtop/hailctl/dataproc/start.py b/hail/python/hailtop/hailctl/dataproc/start.py
@@ -129,7 +129,7 @@ class VepVersion(str, Enum):
 
 ANNOTATION_DB_BUCKETS = ["hail-datasets-us-central1", "hail-datasets-europe-west1"]
 
-IMAGE_VERSION = '2.1.33-debian11'
+IMAGE_VERSION = '2.2.5-debian12'
 
 
 def start(

diff --git a/hail/python/pinned-requirements.txt b/hail/python/pinned-requirements.txt
@@ -211,7 +211,12 @@ portalocker==2.8.2
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   msal-extensions
-py4j==0.10.9.5
+protobuf==3.20.2
+    # via
+    #   -c hail/hail/python/hailtop/pinned-requirements.txt
+    #   -r hail/hail/python/hailtop/requirements.txt
+    #   -r hail/hail/python/requirements.txt
+py4j==0.10.9.7
     # via pyspark
 pyasn1==0.5.1
     # via
@@ -238,7 +243,7 @@ pyjwt[crypto]==2.8.0
     # via
     #   -c hail/hail/python/hailtop/pinned-requirements.txt
     #   msal
-pyspark==3.3.2
+pyspark==3.5.0
     # via
     #   -c hail/hail/python/dataproc-pre-installed-requirements.txt
     #   -r hail/hail/python/requirements.txt

diff --git a/hail/python/requirements.txt b/hail/python/requirements.txt
@@ -10,6 +10,7 @@ numpy<2
 pandas>=2,<3
 parsimonious<1
 plotly>=5.18.0,<6
-pyspark>=3.3.2,<3.4
+protobuf==3.20.2
+pyspark>=3.5.0,<3.6
 requests>=2.31.0,<3
 scipy>1.2,<1.12
diff --git a/hail/python/test/hail/test_exceptions_from_workers_have_stack_traces.py b/hail/python/test/hail/test_exceptions_from_workers_have_stack_traces.py
@@ -11,10 +11,12 @@
 @qobtest
 def test_exceptions_from_workers_have_stack_traces():
     ht = hl.utils.range_table(10, n_partitions=10)
-    ht = ht.annotate(x=hl.int(1) // hl.int(hl.rand_norm(0, 0.1)))
+    ht = ht.annotate(x=hl.int(1) // hl.int(hl.rand_unif(0, 1)))
     pattern = (
         '.*'
-        + re.escape('java.lang.Math.floorDiv(Math.java:1052)')
+        + re.escape('java.lang.Math.floorDiv(Math.java:')
+        + '[0-9]+'
+        + re.escape(')')
         + '.*'
         + re.escape('(BackendUtils.scala:')
         + '[0-9]+'