From d34f5c8039c46f80390cd8d058f9cb9f63443004 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 27 May 2024 15:02:48 +0200 Subject: [PATCH 1/3] Remove support for Julia REQUIRE it doesn't work anymore because build dependencies of the old ZMQ.jl have been removed. --- docs/source/architecture.md | 2 +- docs/source/config_files.rst | 10 +- docs/source/faq.rst | 3 +- docs/source/specification.rst | 4 +- .../julia/install-repo-dependencies.jl | 36 ---- repo2docker/buildpacks/julia/julia_require.py | 198 +----------------- .../julia/require-1-requirements-file/REQUIRE | 5 - .../requirements.txt | 1 - .../julia/require-1-requirements-file/verify | 34 --- tests/julia/require/REQUIRE | 2 - tests/julia/require/verify | 19 -- 11 files changed, 18 insertions(+), 296 deletions(-) delete mode 100644 repo2docker/buildpacks/julia/install-repo-dependencies.jl delete mode 100644 tests/julia/require-1-requirements-file/REQUIRE delete mode 100644 tests/julia/require-1-requirements-file/requirements.txt delete mode 100755 tests/julia/require-1-requirements-file/verify delete mode 100644 tests/julia/require/REQUIRE delete mode 100755 tests/julia/require/verify diff --git a/docs/source/architecture.md b/docs/source/architecture.md index 0153c8ae4..c145010ca 100644 --- a/docs/source/architecture.md +++ b/docs/source/architecture.md @@ -87,7 +87,7 @@ the `build` method of the `BuildPack` base class. The **assemble** stage builds the specific environment that is requested by the repository. This usually means installing required libraries specified in a format native to the language -(`requirements.txt`, `environment.yml`, `REQUIRE`, `install.R`, etc). +(`requirements.txt`, `environment.yml`, `Project.toml`, `install.R`, etc). Most of this work is done in `get_assemble_scripts` method. It can return arbitrary bash script lines that can be run as different users, and has access to the repository contents (unlike diff --git a/docs/source/config_files.rst b/docs/source/config_files.rst index 88392eb84..975f82c85 100644 --- a/docs/source/config_files.rst +++ b/docs/source/config_files.rst @@ -102,14 +102,8 @@ of the Julia packages that are installed. ``REQUIRE`` - Install a Julia environment (legacy) ================================================== -A ``REQUIRE`` file can specify both the version of Julia to be used and -which Julia packages should be used. The use of ``REQUIRE`` is only -recommended for pre 1.0 Julia versions. The recommended way of installing -a Julia environment that uses Julia 1.0 or newer is to use a ``Project.toml`` -file. If both a ``REQUIRE`` and a ``Project.toml`` file are detected, -the ``REQUIRE`` file is ignored. To see an example of a Julia repository -with ``REQUIRE`` and ``environment.yml``, visit -`binder-examples/julia-python `_. +``REQUIRE`` files no longer work, and are no longer supported. +The recommended way of installing a Julia environment is to use a ``Project.toml`` file. .. _install.R: diff --git a/docs/source/faq.rst b/docs/source/faq.rst index e2c9800db..042a3ecb6 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -57,7 +57,8 @@ Julia All Julia versions since Julia 0.7.0 are supported via a :ref:`Project.toml ` file, and this is the recommended way to install Julia environments. -Julia versions 0.6.x and earlier are supported via a :ref:`REQUIRE ` file. + +The older Julia REQUIRE file is no longer supported because required infrastructure has been removed. R ~ diff --git a/docs/source/specification.rst b/docs/source/specification.rst index a3fef2d39..28226845e 100644 --- a/docs/source/specification.rst +++ b/docs/source/specification.rst @@ -5,14 +5,14 @@ The Reproducible Execution Environment Specification ==================================================== repo2docker scans a repository for particular :ref:`config-files`, such -as ``requirements.txt`` or ``REQUIRE``. The collection of files, their contents, +as ``requirements.txt`` or ``Project.toml``. The collection of files, their contents, and the resulting actions that repo2docker takes is known as the **Reproducible Execution Environment Specification** (or REES). The goal of the REES is to automate and encourage existing community best practices for reproducible computational environments. This includes installing pacakges using community-standard specification files and their corresponding tools, -such as ``requirements.txt`` (with ``pip``), ``REQUIRE`` (with Julia), or +such as ``requirements.txt`` (with ``pip``), ``Project.toml`` (with Julia), or ``apt.txt`` (with ``apt``). While repo2docker automates the creation of the environment, a human should be able to look at a REES-compliant repository and reproduce the environment using common, clear steps without diff --git a/repo2docker/buildpacks/julia/install-repo-dependencies.jl b/repo2docker/buildpacks/julia/install-repo-dependencies.jl deleted file mode 100644 index 21321c369..000000000 --- a/repo2docker/buildpacks/julia/install-repo-dependencies.jl +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/julia - -# Path to potential REQUIRE file is passed on the command line. -require_file = ARGS[1] - -if VERSION < v"0.7-" - pkg_dir = "$(ENV["JULIA_PKGDIR"])/v$(VERSION.major).$(VERSION.minor)" - open("$pkg_dir/REQUIRE", "a") do io - write(io, read(require_file)) - end - Pkg.resolve(); - Reqs = Pkg.Reqs -else - using Pkg - Reqs = Pkg.Pkg2.Reqs - emptyversionlower = v"0.0.0-" - for reqline in Reqs.read(require_file) - if reqline isa Reqs.Requirement - pkg = String(reqline.package) - if pkg == "julia" continue end - version = try; reqline.versions.intervals[1].lower; catch; emptyversionlower; end - if version != emptyversionlower - Pkg.add(PackageSpec(name=pkg, version=version)) - else - Pkg.add(pkg) - end - end - end -end -# Precompile the packages -for reqline in Reqs.read(require_file) - if reqline isa Reqs.Requirement - pkg = reqline.package - pkg != "julia" && eval(:(using $(Symbol(pkg)))) - end -end diff --git a/repo2docker/buildpacks/julia/julia_require.py b/repo2docker/buildpacks/julia/julia_require.py index cfea13440..69fd79dda 100644 --- a/repo2docker/buildpacks/julia/julia_require.py +++ b/repo2docker/buildpacks/julia/julia_require.py @@ -1,205 +1,29 @@ -"""Generates a Dockerfile based on an input matrix with REQUIRE for legacy Julia""" +""" +DEPRECATED - Dependencies of REQUIRE have been removed +""" import os -from functools import lru_cache -from ...semver import parse_version as V from ..python import PythonBuildPack class JuliaRequireBuildPack(PythonBuildPack): """ Julia build pack which uses conda and REQUIRE. - """ - - minor_julias = {"0.6": "0.6.4", "0.7": "0.7.0", "1.0": "1.0.4", "1.1": "1.1.1"} - major_julias = {"1": "1.1.1"} - - @property - def python_version(self): - # IJulia doesn't build on julia 0.6 - # due to old incompatibilities with Jupyter-core >= 4.5, - # so use the similarly-old Python 3.5 base environment - if V(self.julia_version) < V("0.7"): - return "3.5" - else: - return super().python_version - - @property - def julia_version(self): - require = self.binder_path("REQUIRE") - try: - with open(require) as f: - julia_version_line = ( - f.readline().strip() - ) # First line is optionally a julia version - except FileNotFoundError: - julia_version_line = "" - - if not julia_version_line.startswith("julia "): - # not a Julia version line. - # use the default Julia. - self._julia_version = self.minor_julias["0.6"] - return self._julia_version - - julia_version_info = julia_version_line.split(" ", 1)[1].split(".") - julia_version = "" - if len(julia_version_info) == 1: - julia_version = self.major_julias[julia_version_info[0]] - elif len(julia_version_info) == 2: - # get major.minor - julia_version = self.minor_julias[".".join(julia_version_info)] - else: - # use supplied julia version - julia_version = ".".join(julia_version_info) - self._julia_version = julia_version - return self._julia_version - - @lru_cache() - def get_build_env(self): - """Get additional environment settings for Julia and Jupyter - - Returns: - an ordered list of environment setting tuples - The tuples contain a string of the environment variable name and - a string of the environment setting: - - `JULIA_PATH`: base path where all Julia Binaries and libraries - will be installed - - `JULIA_HOME`: path where all Julia Binaries will be installed - - `JULIA_PKGDIR`: path where all Julia libraries will be installed - - `JULIA_DEPOT_PATH`: path where Julia libraries are installed. - Similar to JULIA_PKGDIR, used in 1.x. - - `JULIA_VERSION`: default version of julia to be installed - - `JULIA_ARCH`: machine architecture used in Julia download URLs - - `JULIA_ARCH_SHORT`: machine architecture used in Julia download URLs - - `JUPYTER`: environment variable required by IJulia to point to - the `jupyter` executable - - For example, a tuple may be `('JULIA_VERSION', '0.6.0')`. - - """ - if self.platform == "linux/arm64": - julia_arch = julia_arch_short = "aarch64" - else: - julia_arch = "x86_64" - julia_arch_short = "x64" - - if V(self.julia_version) < V("0.7"): - # IJulia with Julia 0.6 isn't compatible with more recent jupyter-core - # point it to the one in the kernel env - # I _think_ this is only relevant during installation - jupyter = "${KERNEL_PYTHON_PREFIX}/bin/jupyter" - else: - jupyter = "${NB_PYTHON_PREFIX}/bin/jupyter" - return super().get_build_env() + [ - ("JULIA_PATH", "${APP_BASE}/julia"), - ("JULIA_HOME", "${JULIA_PATH}/bin"), # julia <= 0.6 - ("JULIA_BINDIR", "${JULIA_HOME}"), # julia >= 0.7 - ("JULIA_PKGDIR", "${JULIA_PATH}/pkg"), - ("JULIA_DEPOT_PATH", "${JULIA_PKGDIR}"), # julia >= 0.7 - ("JULIA_VERSION", self.julia_version), - ("JULIA_ARCH", julia_arch), - ("JULIA_ARCH_SHORT", julia_arch_short), - ("JUPYTER", jupyter), - ] - - @lru_cache() - def get_path(self): - """Adds path to Julia binaries to user's PATH. - - Returns: - an ordered list of path strings. The path to the Julia - executable is added to the list. - - """ - return super().get_path() + ["${JULIA_HOME}"] - - @lru_cache() - def get_build_scripts(self): - """ - Return series of build-steps common to "ALL" Julia repositories - - All scripts found here should be independent of contents of a - particular repository. - - This creates a directory with permissions for installing julia packages - (from get_assemble_scripts). - - """ - return super().get_build_scripts() + [ - ( - "root", - r""" - mkdir -p ${JULIA_PATH} && \ - curl -sSL "https://julialang-s3.julialang.org/bin/linux/${JULIA_ARCH_SHORT}/${JULIA_VERSION%[.-]*}/julia-${JULIA_VERSION}-linux-${JULIA_ARCH}.tar.gz" | tar -xz -C ${JULIA_PATH} --strip-components 1 - """, - ), - ( - "root", - r""" - mkdir -p ${JULIA_PKGDIR} && \ - chown ${NB_USER}:${NB_USER} ${JULIA_PKGDIR} - """, - ), - ( - "${NB_USER}", - # HACK: Can't seem to tell IJulia to install in sys-prefix - # FIXME: Find way to get it to install under /srv and not $HOME? - r""" - julia -e 'if (VERSION > v"0.7-") using Pkg; else Pkg.init(); end; Pkg.add("IJulia"); using IJulia;' && \ - mv ${HOME}/.local/share/jupyter/kernels/julia-${JULIA_VERSION%[.-]*} ${NB_PYTHON_PREFIX}/share/jupyter/kernels/julia-${JULIA_VERSION%[.-]*} - """, - ), - ] - - @lru_cache() - def get_assemble_scripts(self): - """ - Return series of build-steps specific to "this" Julia repository - - Precompile all Julia libraries found in the repository's REQUIRE - file. The parent, CondaBuildPack, will add the build steps for - any needed Python packages found in environment.yml. - - """ - require = self.binder_path("REQUIRE") - return super().get_assemble_scripts() + [ - ( - "${NB_USER}", - # Install and pre-compile all libraries if they've opted into it. - # In v0.6, Pkg.resolve() installs all the packages, but in v0.7+, we - # have to manually Pkg.add() each of them (since the REQUIRES file - # format is deprecated). - # The precompliation is done via `using {libraryname}`. - r""" - julia /tmp/install-repo-dependencies.jl "%(require)s" - """ - % {"require": require}, - # TODO: For some reason, `rm`ing the file fails with permission denied. - # && rm /tmp/install-repo-dependencies.jl - ) - ] + Now just an informative error message. + """ - @lru_cache() - def get_build_script_files(self): - files = { - "julia/install-repo-dependencies.jl": "/tmp/install-repo-dependencies.jl" - } - files.update(super().get_build_script_files()) - return files + def build(self, *args, **kwargs): + raise ValueError( + "Julia REQUIRE no longer supported due to removed infrastructure. Use Project.toml." + ) def detect(self): """ - Check if current repo should be built with the Julia Legacy Build pack - - super().detect() is not called in this function - it would return - false unless an `environment.yml` is present and we do not want to - require the presence of a `environment.yml` to use Julia. - - Instead we just check if the path to `REQUIRE` exists and that there is - no julia 1.0 style environment + Check if current repo exects tp be built with the Julia Legacy Build pack + This no longer works, but try to raise an informative error. """ return os.path.exists(self.binder_path("REQUIRE")) and not ( os.path.exists(self.binder_path("Project.toml")) diff --git a/tests/julia/require-1-requirements-file/REQUIRE b/tests/julia/require-1-requirements-file/REQUIRE deleted file mode 100644 index 1d95307f2..000000000 --- a/tests/julia/require-1-requirements-file/REQUIRE +++ /dev/null @@ -1,5 +0,0 @@ -julia 1 - -# Julia packages: -PyCall -PyPlot diff --git a/tests/julia/require-1-requirements-file/requirements.txt b/tests/julia/require-1-requirements-file/requirements.txt deleted file mode 100644 index 6ccafc3f9..000000000 --- a/tests/julia/require-1-requirements-file/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -matplotlib diff --git a/tests/julia/require-1-requirements-file/verify b/tests/julia/require-1-requirements-file/verify deleted file mode 100755 index 3e55195de..000000000 --- a/tests/julia/require-1-requirements-file/verify +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env julia -using PyCall -using PyPlot - -if ! (VERSION >= v"1" && VERSION < v"2") - println("Julia version was: ", VERSION) - println("Julia version expected to be 1.x when pinned to 1 in a REQUIRE file") - exit(1) -end - -# Make sure we are re-using the same conda python -if ! startswith(PyCall.libpython, ENV["NB_PYTHON_PREFIX"] * "/lib") - println("Not re-using conda python! Using " * PyCall.libpython * " instead") - exit(1) -end - -# We check that PyPlot is installed inside ${JULIA_DEPOT_PATH} -if ! isdir(ENV["JULIA_DEPOT_PATH"] * "/packages/PyPlot") - println("PyPlot not installed under JULIA_DEPOT_PATH") - exit(1) -end - -# FIXME: Julia code that worked before v1, but now fails with: -# -# ERROR: LoadError: UndefVarError: linspace not defined -# -# x = linspace(0,2*pi,1000); y = sin(3*x + 4*cos(2*x)) -# plot(x, y, color="red", linewidth=2.0, linestyle="--") -# savefig("graph.png") -# if isfile("graph.png") -# exit(0) -# else -# exit(1) -# end diff --git a/tests/julia/require/REQUIRE b/tests/julia/require/REQUIRE deleted file mode 100644 index f77f67e60..000000000 --- a/tests/julia/require/REQUIRE +++ /dev/null @@ -1,2 +0,0 @@ -# Julia packages: -Compat diff --git a/tests/julia/require/verify b/tests/julia/require/verify deleted file mode 100755 index 73851db68..000000000 --- a/tests/julia/require/verify +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env julia - -if VERSION != v"0.6.4" - println("Julia version was: ", VERSION) - println("Default Julia version expected to be 0.6.4 when unspecified in a REQUIRE file") - exit(1) -end - -try - # Test that the package was installed. - using Compat - - # Verify that the environment variables are set correctly for julia < 0.7 - @assert "julia" ∈ readdir(Base.JULIA_HOME) -catch - exit(1) -end - -exit(0) From 48f894dbe18c2f9f65014ceba6c5f0330087ca8b Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 27 May 2024 15:32:46 +0200 Subject: [PATCH 2/3] julia 1.3 is the oldest version that can build ZMQ.jl now --- docs/source/faq.rst | 4 ++-- .../.binder/Project.toml | 2 +- .../Project.toml | 0 .../{project-1.0.2-binder-dir => project-binder-dir}/verify | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) rename tests/julia/{project-1.0.2-binder-dir => project-binder-dir}/.binder/Project.toml (83%) rename tests/julia/{project-1.0.2-binder-dir => project-binder-dir}/Project.toml (100%) rename tests/julia/{project-1.0.2-binder-dir => project-binder-dir}/verify (87%) diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 042a3ecb6..d643264b8 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -55,10 +55,10 @@ We **strongly** recommend specifying a Python version (in environment.yml, runti Julia ~~~~~ -All Julia versions since Julia 0.7.0 are supported via a :ref:`Project.toml ` +All Julia versions since Julia 1.3 are supported via a :ref:`Project.toml ` file, and this is the recommended way to install Julia environments. -The older Julia REQUIRE file is no longer supported because required infrastructure has been removed. +Julia < 1.3 and the older Julia REQUIRE file is no longer supported because required infrastructure has been removed. R ~ diff --git a/tests/julia/project-1.0.2-binder-dir/.binder/Project.toml b/tests/julia/project-binder-dir/.binder/Project.toml similarity index 83% rename from tests/julia/project-1.0.2-binder-dir/.binder/Project.toml rename to tests/julia/project-binder-dir/.binder/Project.toml index c2e6033b1..1aae75bd3 100644 --- a/tests/julia/project-1.0.2-binder-dir/.binder/Project.toml +++ b/tests/julia/project-binder-dir/.binder/Project.toml @@ -2,4 +2,4 @@ IteratorInterfaceExtensions = "82899510-4779-5014-852e-03e436cf321d" [compat] -julia = "=1.0.2" +julia = "=1.3" diff --git a/tests/julia/project-1.0.2-binder-dir/Project.toml b/tests/julia/project-binder-dir/Project.toml similarity index 100% rename from tests/julia/project-1.0.2-binder-dir/Project.toml rename to tests/julia/project-binder-dir/Project.toml diff --git a/tests/julia/project-1.0.2-binder-dir/verify b/tests/julia/project-binder-dir/verify similarity index 87% rename from tests/julia/project-1.0.2-binder-dir/verify rename to tests/julia/project-binder-dir/verify index 4d339a9c1..e9705456d 100755 --- a/tests/julia/project-1.0.2-binder-dir/verify +++ b/tests/julia/project-binder-dir/verify @@ -1,7 +1,7 @@ #!/usr/bin/env julia -if VERSION != v"1.0.2" - println("Julia version should be 1.0.2") +if VERSION != v"1.3.1" + println("Julia version should be 1.3.1") exit(1) end From c130e87aaaee55b7109f21b0c7ff387740f96cff Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 29 May 2024 12:24:24 +0200 Subject: [PATCH 3/3] fix verify in Julia 1.3 --- tests/julia/project-binder-dir/verify | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/julia/project-binder-dir/verify b/tests/julia/project-binder-dir/verify index e9705456d..b8e42890b 100755 --- a/tests/julia/project-binder-dir/verify +++ b/tests/julia/project-binder-dir/verify @@ -1,7 +1,7 @@ #!/usr/bin/env julia -if VERSION != v"1.3.1" - println("Julia version should be 1.3.1") +if VERSION != v"1.3" + println("Julia version should be 1.3, got $VERSION") exit(1) end