From 44844ee29bf680c8376befb397e5bc1b0efba7f5 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Sun, 14 May 2023 19:24:02 +0200 Subject: [PATCH] Make CUDA and Term weak dependencies on Julia >= 1.9 (#176) * Make CUDA and Term weak dependencies on Julia >= 1.9 * Fix typo * More updates --- .github/workflows/ci.yml | 1 + Project.toml | 15 +++- docs/src/features.md | 15 ++-- ext/XGBoostCUDAExt.jl | 19 +++++ ext/XGBoostTermExt.jl | 131 +++++++++++++++++++++++++++++++++ src/XGBoost.jl | 6 +- src/cuda.jl | 1 - src/dmatrix.jl | 50 ++++--------- src/show.jl | 153 +++------------------------------------ test/runtests.jl | 3 +- 10 files changed, 205 insertions(+), 189 deletions(-) create mode 100644 ext/XGBoostCUDAExt.jl create mode 100644 ext/XGBoostTermExt.jl delete mode 100644 src/cuda.jl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 259f68e..33950a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,7 @@ jobs: version: - '1.6' - '1' # automatically expands to the latest stable 1.x release of Julia + - 'nightly' os: - ubuntu-latest arch: diff --git a/Project.toml b/Project.toml index 21f7d6d..a269424 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "XGBoost" uuid = "009559a3-9522-5dbb-924b-0b6ed2b22bb9" -version = "2.2.5" +version = "2.3.0" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" @@ -14,9 +14,16 @@ SparseMatricesCSR = "a0a7dd2c-ebf4-11e9-1f05-cf50bc540ca1" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Term = "22787eb5-b846-44ae-b979-8e399b8463ab" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" XGBoost_jll = "a5c6f535-4255-5ca2-a466-0e519f119c46" +[weakdeps] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +Term = "22787eb5-b846-44ae-b979-8e399b8463ab" + +[extensions] +XGBoostCUDAExt = "CUDA" +XGBoostTermExt = "Term" + [compat] AbstractTrees = "0.4" CEnum = "0.4" @@ -30,8 +37,10 @@ XGBoost_jll = "1.7.2" julia = "1.6" [extras] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Term = "22787eb5-b846-44ae-b979-8e399b8463ab" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Random", "Test"] +test = ["CUDA", "Random", "Term", "Test"] diff --git a/docs/src/features.md b/docs/src/features.md index f0a4332..494d5fa 100644 --- a/docs/src/features.md +++ b/docs/src/features.md @@ -8,8 +8,8 @@ CurrentModule = XGBoost ## Introspection ### Feature Importance -This package contains a number of methods for inspecting the results of training and displaying the -results in a legible way with [Term.jl](https://github.com/FedeClaudi/Term.jl). + +This package contains a number of methods for inspecting the results of training and displaying the results. Feature importances can be computed explicitly using [`importance`](@ref) @@ -22,6 +22,7 @@ bst = xgboost(X, y) imp = DataFrame(importancetable(bst)) ``` +XGBoost also supports rich terminal output with [Term.jl](https://github.com/FedeClaudi/Term.jl). A convenient visualization of this table can also be seen with [`importancereport`](@ref). These will use assigned feature names, for example ```julia @@ -41,7 +42,7 @@ julia> df = DataFrame(randn(10,3), ["kirk", "spock", "bones"]) 9 │ -1.26053 -1.60734 2.21421 10 │ 0.30378 -0.299256 0.384029 -julia> bst = xgboost((df, randn(10)), num_round=10) +julia> bst = xgboost((df, randn(10)), num_round=10); [ Info: XGBoost: starting training. [ Info: [1] train-rmse:0.57998637329114211 [ Info: [2] train-rmse:0.48232409595403752 @@ -54,6 +55,8 @@ julia> bst = xgboost((df, randn(10)), num_round=10) [ Info: [9] train-rmse:0.15198720040980171 [ Info: [10] train-rmse:0.12906074380448287 [ Info: Training rounds complete. + +julia> using Term; Panel(bst) ╭──── XGBoost.Booster ─────────────────────────────────────────────────────────────────╮ │ Features: ["kirk", "spock", "bones"] │ │ │ @@ -97,7 +100,7 @@ julia> ts = trees(bst) XGBoost.Node(split_feature="bones") XGBoost.Node(split_feature="bones") -julia> ts[1] +julia> using Term; Panel(ts[1]) ╭──── XGBoost.Node (id=0, depth=0) ────────────────────────────────────────────────────╮ │ │ │ split_condition yes no nmissing gain cover │ @@ -190,7 +193,7 @@ will fit a random forest according to a Poisson likelihood fit with 12 trees. ## GPU Support XGBoost supports GPU-assisted training on Nvidia GPU's with CUDA via -[CUDA.jl](https://github.com/JuliaGPU/CUDA.jl). To utilize the GPU, one has to construct a +[CUDA.jl](https://github.com/JuliaGPU/CUDA.jl). To utilize the GPU, one has to load CUDA and construct a `DMatrix` object from GPU arrays. There are two ways of doing this: - Pass a `CuArray` as the training matrix (conventionally `X`, the first argument to `DMatrix`). - Pass a table with *all* columns as `CuVector`s. @@ -210,6 +213,8 @@ normally directly to `xgboost` or `Booster`, as long as that data consists of `C ### Example ```julia +using CUDA + X = cu(randn(1000, 3)) y = randn(1000) diff --git a/ext/XGBoostCUDAExt.jl b/ext/XGBoostCUDAExt.jl new file mode 100644 index 0000000..ce73e7a --- /dev/null +++ b/ext/XGBoostCUDAExt.jl @@ -0,0 +1,19 @@ +module XGBoostCUDAExt + +using XGBoost +using XGBoost: DMatrixHandle, XGDMatrixCreateFromCudaArrayInterface, numpy_json_info, xgbcall +using CUDA: CuMatrix, CuVector + +function XGBoost._dmatrix(x::CuMatrix{T}; missing_value::Float32=NaN32, kw...) where {T<:Real} + o = Ref{DMatrixHandle}() + cfg = "{\"missing\": $missing_value}" + GC.@preserve x begin + info = numpy_json_info(x) + xgbcall(XGDMatrixCreateFromCudaArrayInterface, info, cfg, o) + end + DMatrix(o[]; is_gpu=true, kw...) +end + +XGBoost.isa_cuvector(::CuVector) = true + +end # module diff --git a/ext/XGBoostTermExt.jl b/ext/XGBoostTermExt.jl new file mode 100644 index 0000000..cb5d4da --- /dev/null +++ b/ext/XGBoostTermExt.jl @@ -0,0 +1,131 @@ +module XGBoostTermExt + +using XGBoost: XGBoost, OrderedDict, children +import Term + +function _features_display_string(fs, n) + str = "{bold yellow}Features:{/bold yellow} " + if isempty(fs) + str*"$n (unknown names)" + else + string(str, fs) + end +end + +function Term.Panel(dm::XGBoost.DMatrix) + str = if !XGBoost.hasdata(dm) + "{dim}(values not allocated){/dim}" + else + repr(MIME("text/plain"), dm.data; context=:compact=>true) + end + subtitle = sprint(dm) do io, dm + print(io, "(nrows=", XGBoost.nrows(dm), ", ncols=", XGBoost.ncols(dm), ")") + if XGBoost.isgpu(dm) + print(io, " {bold green}(GPU){/bold green}") + end + end + Term.Panel(_features_display_string(XGBoost.getfeaturenames(dm), size(dm,2)), + str; + style="magenta", + title="XGBoost.DMatrix", + title_style="bold cyan", + subtitle, + subtitle_style="blue", + ) +end + +function Term.Panel(b::XGBoost.Booster) + info = if isempty(b.params) + () + else + (paramspanel(b.params; header_style="bold green", columns_style=["bold yellow", "default"], box=:SIMPLE,),) + end + Term.Panel(_features_display_string(b.feature_names, XGBoost.nfeatures(b)), + info...; + style="magenta", + title="XGBoost.Booster", + title_style="bold cyan", + subtitle="boosted rounds: $(XGBoost.getnrounds(b))", + subtitle_style="blue", + ) +end +function paramspanel(params::AbstractDict; kwargs...) + names = sort!(collect(keys(params))) + vals = map(k -> params[k], names) + Term.Table(OrderedDict(:Parameter=>names, :Value=>vals), kwargs...) +end + +function Term.Tree( + node::XGBoost.Node; + title="XGBoost Tree (from this node)", + title_style="bold green", + kwargs..., +) + td = isempty(children(node)) ? Dict(repr(node)=>"leaf") : _tree_display(node) + Term.Tree(td; title, title_style, kwargs...) +end + +function Term.Panel(node::XGBoost.Node) + subtitle = if isempty(children(node)) + "{bold green}leaf{/bold green}" + else + string(length(children(node)), " children") + end + + Term.Panel(paramstable(node; header_style="bold yellow", box=:SIMPLE), + Term.Tree(node); + style="magenta", + title="XGBoost.Node {italic blue}(id=$(node.id), depth=$(node.depth)){/italic blue}", + title_style="bold cyan", + subtitle, + subtitle_style="blue", + ) +end +function paramstable(node::XGBoost.Node; kwargs...) + if isempty(children(node)) + _paramstable(node, :cover, :leaf; kwargs...) + else + _paramstable(node, :split_condition, :yes, :no, :nmissing, :gain, :cover; kwargs...) + end +end +function _paramstable(node::XGBoost.Node, names::Symbol...; kwargs...) + vals = mapreduce(Base.Fix1(getproperty, node), hcat, names) + Term.Table(vals; header=map(string, names), kwargs...) +end + +function XGBoost.importancereport(b::XGBoost.Booster) + if XGBoost.getnrounds(b) == 0 + Panel("{red}(booster not trained){/red}", + title="XGBoost Feature Importance", + style="magenta", + ) + else + tbl = XGBoost.importancetable(b) + tbl = OrderedDict(k=>_importance_number_string.(getproperty(tbl, k)) for k ∈ propertynames(tbl)) + Term.Table(tbl, + header_style="bold green", + columns_style=["bold yellow"; fill("default", 5)], + box=:ROUNDED, + ) + end +end +_importance_number_string(imp) = repr(imp, context=:compact=>true) +_importance_number_string(::Missing) = "{dim}missing{/dim}" + +function _tree_display(node::XGBoost.Node) + ch = children(node) + if isempty(ch) + repr(node; context=:compact=>true) + else + OrderedDict(_tree_display_branch_string(node, ch[j].id)=>_tree_display(ch[j]) for j ∈ 1:length(ch)) + end +end +function _tree_display_branch_string(node, child_id::Integer) + if node.yes == child_id + string(node.split, " < ", round(node.split_condition, digits=3)) + else + string(node.split, " ≥ ", round(node.split_condition, digits=3)) + end +end + +end # module diff --git a/src/XGBoost.jl b/src/XGBoost.jl index ff05e53..856cecd 100644 --- a/src/XGBoost.jl +++ b/src/XGBoost.jl @@ -9,8 +9,6 @@ using AbstractTrees using OrderedCollections using JSON3 using Tables -using Term -using CUDA using Statistics: mean, std using Base: @propagate_inbounds @@ -49,5 +47,9 @@ include("introspection.jl") include("show.jl") include("defaultparams.jl") +if !isdefined(Base, :get_extension) + include("../ext/XGBoostCUDAExt.jl") + include("../ext/XGBoostTermExt.jl") +end end # module XGBoost diff --git a/src/cuda.jl b/src/cuda.jl deleted file mode 100644 index 8b13789..0000000 --- a/src/cuda.jl +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/dmatrix.jl b/src/dmatrix.jl index 655d2fb..21aa91d 100644 --- a/src/dmatrix.jl +++ b/src/dmatrix.jl @@ -184,33 +184,13 @@ function _dmatrix(x::AbstractMatrix{T}; missing_value::Float32=NaN32, kw...) whe DMatrix(o[]; kw...) end -# sadly we have to copy CuArray because of incompatible column convention -function _transposed_cuda_dmatrix(x::CuArray{T}; missing_value::Float32=NaN32, kw...) where {T<:Real} - o = Ref{DMatrixHandle}() - cfg = "{\"missing\": $missing_value}" - GC.@preserve x begin - info = numpy_json_info(x) - xgbcall(XGDMatrixCreateFromCudaArrayInterface, info, cfg, o) - end - DMatrix(o[]; is_gpu=true, kw...) -end - -DMatrix(x::Transpose{T,<:CuArray}; kw...) where {T<:Real} = _transposed_cuda_dmatrix(parent(x); kw...) -DMatrix(x::Adjoint{T,<:CuArray}; kw...) where {T<:Real} = _transposed_cuda_dmatrix(parent(x); kw...) - -function DMatrix(x::CuArray; kw...) - x′ = CuArray(transpose(x)) - _transposed_cuda_dmatrix(x′; kw...) -end - function DMatrix(x::AbstractMatrix{T}; kw...) where {T<:Real} # sadly, this copying is unavoidable - _dmatrix(convert(Matrix{Float32}, transpose(x)); kw...) + _dmatrix(permutedims(x); kw...) end # ideally these would be recursive but can't be bothered -DMatrix(x::Transpose{T}; kw...) where {T<:Real} = _dmatrix(parent(x); kw...) -DMatrix(x::Adjoint{T}; kw...) where {T<:Real} = _dmatrix(parent(x); kw...) +DMatrix(x::LinearAlgebra.AdjOrTransAbsMat{T}; kw...) where {T<:Real} = _dmatrix(parent(x); kw...) function DMatrix(x::AbstractMatrix{Union{Missing,T}}; kw...) where {T<:Real} # we try to make it so that we only have to copy once @@ -270,12 +250,6 @@ DMatrix(Xy::Tuple; kw...) = DMatrix(Xy[1], Xy[2]; kw...) DMatrix(dm::DMatrix) = dm -function _check_gpu_table(tbl) - cols = Tables.Columns(tbl) - isgpu = all(x -> x isa CuArray, cols) - (isgpu, cols) -end - function _dmatrix_gpu_table(cols::Tables.Columns; missing_value::Float32=NaN32, kw...) o = Ref{DMatrixHandle}() cfg = "{\"missing\": $missing_value}" @@ -286,27 +260,31 @@ function _dmatrix_gpu_table(cols::Tables.Columns; missing_value::Float32=NaN32, DMatrix(o[]; is_gpu=true, kw...) end +isa_cuvector(x) = false + function DMatrix(tbl; - feature_names::AbstractVector{<:AbstractString}=collect(string.(Tables.columnnames(tbl))), + feature_names::Union{Nothing,AbstractVector{<:AbstractString}}=nothing, kw... ) - if !Tables.istable(tbl) - throw(ArgumentError("DMatrix requires either an AbstractMatrix or table satisfying the Tables.jl interface")) + cols = Tables.columns(tbl) + if feature_names === nothing + feature_names = [string(x) for x in Tables.columnnames(cols)] end - (isgpu, cols) = _check_gpu_table(tbl) + isgpu = all(isa_cuvector, cols) if isgpu _dmatrix_gpu_table(cols; feature_names, kw...) else - DMatrix(Tables.matrix(tbl); feature_names, kw...) + DMatrix(Tables.matrix(cols); feature_names, kw...) end end DMatrix(tbl, y::AbstractVector; kw...) = DMatrix(tbl; label=y, kw...) function DMatrix(tbl, ycol::Symbol; kw...) - Xcols = [n for n ∈ Tables.columnnames(tbl) if n ≠ ycol] - tbl′ = NamedTuple(n=>Tables.getcolumn(tbl, n) for n ∈ Xcols) - DMatrix(tbl′, Tables.getcolumn(tbl, ycol); kw...) + cols = Tables.columns(tbl) + Xcols = [n for n ∈ Tables.columnnames(cols) if n ≠ ycol] + tbl′ = NamedTuple(n=>Tables.getcolumn(cols, n) for n ∈ Xcols) + DMatrix(tbl′, Tables.getcolumn(cols, ycol); kw...) end """ diff --git a/src/show.jl b/src/show.jl index 4464707..fb8c7df 100644 --- a/src/show.jl +++ b/src/show.jl @@ -4,160 +4,31 @@ function Base.show(io::IO, dm::DMatrix) print(io, "(", size(dm,1), ", ", size(dm,2), ")") end -function _features_display_string(fs, n) - str = "{bold yellow}Features:{/bold yellow} " - if isempty(fs) - str*"$n (unknown names)" - else - string(str, fs) - end -end - -function Base.show(io::IO, mime::MIME"text/plain", dm::DMatrix) - str = if !hasdata(dm) - "{dim}(values not allocated){/dim}" - else - sprint((io, x) -> show(io, MIME"text/plain"(), x), dm.data, - context=:compact=>true, - ) - end - subtitle = "(nrows=$(nrows(dm)), ncols=$(ncols(dm)))" - isgpu(dm) && (subtitle *= " {bold green}(GPU){/bold green}") - p = Panel(_features_display_string(getfeaturenames(dm), size(dm,2)), - str; - style="magenta", - title="XGBoost.DMatrix", - title_style="bold cyan", - subtitle, - subtitle_style="blue", - ) - show(io, mime, p) -end - function Base.show(io::IO, b::Booster) show(io, typeof(b)) print(io, "()") end - -function paramspanel(params::AbstractDict) - names = sort!(collect(keys(params))) - vals = map(k -> params[k], names) - Term.Table(OrderedDict(:Parameter=>names, :Value=>vals), - header_style="bold green", - columns_style=["bold yellow", "default"], - box=:SIMPLE, - ) -end -paramspanel(b::Booster) = paramspanel(b.params) - -function Term.Panel(b::Booster) - info = isempty(b.params) ? () : (paramspanel(b),) - Panel(_features_display_string(b.feature_names, nfeatures(b)), - info...; - style="magenta", - title="XGBoost.Booster", - title_style="bold cyan", - subtitle="boosted rounds: $(getnrounds(b))", - subtitle_style="blue", - ) -end - -Base.show(io::IO, mime::MIME"text/plain", b::Booster) = show(io, mime, Panel(b)) - -_importance_number_string(imp) = repr(imp, context=:compact=>true) -_importance_number_string(::Missing) = "{dim}missing{/dim}" - """ importancereport(b::Booster) -Show a convenient text display of the table output by [`importancetable`](@ref). This is -intended entirely for display purposes, see [`importance`](@ref) for how to retrieve -feature importance statistics directly. -""" -function importancereport(b::Booster) - if getnrounds(b) == 0 - Panel("{red}(booster not trained){/red}", - title="XGBoost Feature Importance", - style="magenta", - ) - else - tbl = importancetable(b) - tbl = OrderedDict(k=>_importance_number_string.(getproperty(tbl, k)) for k ∈ propertynames(tbl)) - Term.Table(tbl, - header_style="bold green", - columns_style=["bold yellow"; fill("default", 5)], - box=:ROUNDED, - ) - end -end - +Show a convenient text display of the table output by [`importancetable`](@ref). -function _tree_display_branch_string(node, child_id::Integer) - if node.yes == child_id - string(node.split, " < ", round(node.split_condition, digits=3)) - else - string(node.split, " ≥ ", round(node.split_condition, digits=3)) - end -end - -function _tree_display(node::Node) - ch = children(node) - if isempty(ch) - sprint(show, node) - else - OrderedDict(_tree_display_branch_string(node, ch[j].id)=>_tree_display(ch[j]) for j ∈ 1:length(ch)) - end -end - -function Term.Tree(node::Node) - td = isempty(children(node)) ? Dict(sprint(show, node)=>"leaf") : _tree_display(node) - Term.Tree(td; - title="XGBoost Tree (from this node)", - title_style="bold green", - ) -end - -function _paramstable(node::Node, names::AbstractVector) - vals = [getproperty(node, n) for n ∈ permutedims(names)] - Term.Table(vals; - header=string.(names), - header_style="bold yellow", - box=:SIMPLE - ) -end - -_paramstable(node::Node) = _paramstable(node, [:split_condition, :yes, :no, :nmissing, :gain, :cover]) - -_paramstable_leaf(node::Node) = _paramstable(node, [:cover, :leaf]) - -paramstable(node::Node) = length(children(node)) == 0 ? _paramstable_leaf(node) : _paramstable(node) - -function Term.Panel(node::Node) - subtitle = if isempty(children(node)) - "{bold green}leaf{/bold green}" - else - string(length(children(node)), " children") - end +This is intended entirely for display purposes, see [`importance`](@ref) for how to retrieve +feature importance statistics directly. - Panel(paramstable(node), - Term.Tree(node); - style="magenta", - title="XGBoost.Node {italic blue}(id=$(node.id), depth=$(node.depth)){/italic blue}", - title_style="bold cyan", - subtitle, - subtitle_style="blue", - ) -end +!!! note + In Julia >= 1.9, you have to load Term.jl to be able to use this functionality. +""" +function importancereport end function Base.show(io::IO, node::Node) show(io, typeof(node)) - str = if isempty(children(node)) - "leaf=$(node.leaf)" + print(io, "(") + if isempty(children(node)) + print(io, "leaf=", node.leaf) else - "split_feature=$(sprint(show, node.split))" + print(io, "split_feature=", node.split) end - print(io, "(", str, ")") + print(io, ")") end - -Base.show(io::IO, mime::MIME"text/plain", node::Node) = show(io, mime, Panel(node)) diff --git a/test/runtests.jl b/test/runtests.jl index b805af0..d5b9870 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,6 @@ using XGBoost using CUDA: has_cuda, cu +import Term using Random, SparseArrays using Test @@ -160,7 +161,7 @@ end @test tbl.feature[1] == 29 @test XGBoost.Tables.columnnames(tbl) == (:feature, :gain, :weight, :cover, :total_gain, :total_cover) - @test typeof(importancereport(bst)) <: XGBoost.Term.Tables.Table + @test typeof(importancereport(bst)) <: Term.Tables.Table end @testset "Booster Save/Load/Serialize" begin