Generate complete bindings to libzmq (#232)

* Generate bindings for zmq.h using Clang.jl

Along with some extra methods we need for the Message type.

* Replace explicit ccall's with calls to the generated bindings

* Make the _Message API require using Ref{_Message}'s

Using plain _Message's with most of these functions is likely incorrect.

* fixup! Generate bindings for zmq.h using Clang.jl

* fixup! Replace explicit ccall's with calls to the generated bindings

* fixup! Generate bindings for zmq.h using Clang.jl

* fixup! Generate bindings for zmq.h using Clang.jl

docs/make.jl

@@ -20,6 +20,7 @@ makedocs(
             "man/examples.md",
         ],
         "Reference" => "reference.md",
+        "Bindings" => "bindings.md",
         "Changelog" => "changelog.md"
     ],
     format = Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true")

docs/src/_changelog.md

@@ -12,6 +12,7 @@ Changelog](https://keepachangelog.com).
 ### Added
 - Support for creating [`Message`](@ref)'s from the new `Memory` type in Julia
   1.11 ([#244]).
+- Full [Bindings](@ref) to libzmq ([#232]).
 
 ### Fixed
 - Fixed [`isfreed()`](@ref), which would previously return the wrong values

docs/src/bindings.md

@@ -0,0 +1,15 @@
+# Bindings
+
+This page documents the low-level bindings to libzmq that were automatically
+generated. Where possible, the docstrings link to the [upstream
+documentation](https://libzmq.readthedocs.io). Bindings have not been generated
+for deprecated functions.
+
+!!! danger
+    These bindings are unsafe, do not use them unless you know what you're doing.
+
+---
+
+```@autodocs
+Modules = [ZMQ.lib]
+```

docs/src/index.md

@@ -10,6 +10,6 @@
 
 The [Guide](@ref) provides a tutorial explaining how to get started using ZMQ.jl.
 
-Some examples of packages using Documenter can be found on the [Examples](@ref) page.
+Some examples are linked on the [Examples](@ref) page.
 
-See the [Reference](@ref) for the complete list of documented functions and types.
+See the [Reference](@ref) for the complete list of wrapped functions and types.

gen/Project.toml

@@ -0,0 +1,4 @@
+[deps]
+Clang = "40e3b903-d033-50b4-a0cc-940c62c95e31"
+MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+ZeroMQ_jll = "8f1865be-045e-5c20-9c9f-bfbfb0764568"

gen/gen.jl

@@ -0,0 +1,106 @@
+import Clang
+import Clang.Generators: FunctionProto
+import ZeroMQ_jll
+import MacroTools: @capture, postwalk, prettify
+
+
+# Helper function to look through all the generated bindings and create new
+# zmq_msg_* methods for the Message type. We need to create these overloads
+# because the Message type relies on _Message (i.e. lib.zmq_msg_t) under the
+# hood, and _Message is an immutable type so it doesn't have a stable address,
+# and so cannot safely be passed to a ccall.
+#
+# We get around this for _Message by always using a Ref{_Message}, but Message
+# is a mutable struct with a _Message as its first field. It's safe to pass a
+# pointer to a Message to libzmq because the address of the Message is the same
+# as its first field, the _Message. But to do that we need to create methods to
+# ccall libzmq with the Message type instead of lib.zmq_msg_t (_Message).
+function get_msg_methods(ctx, module_name)
+    methods = Expr[]
+
+    for node in ctx.dag.nodes
+        for i in eachindex(node.exprs)
+            expr = node.exprs[i]
+
+            # Check if this is a function
+            if @capture(expr, function name_(arg1_, args__) body_ end)
+                # Check if it's a zmq_msg_* function
+                if startswith(string(name), "zmq_msg_")
+                    # Replace occurrences of `arg::Ptr{zmq_msg_t}` with
+                    # `arg::Ref{Message}`.
+                    new_body = postwalk(body) do x
+                        if @capture(x, Ptr{T_}) && T == :(zmq_msg_t)
+                            :(Ref{Message})
+                        else
+                            x
+                        end
+                    end
+
+                    # Create the new method
+                    new_method = quote
+                        function $module_name.$name($arg1::Message, $(args...))
+                            $new_body
+                        end
+                    end
+
+                    push!(methods, prettify(new_method))
+                end
+            end
+        end
+    end
+
+    return methods
+end
+
+# See:
+# https://github.com/zeromq/libzmq/blob/c2fae81460d9d39a896da7b3f72484d23a172fa7/include/zmq.h#L582-L611
+const undocumented_functions = [:zmq_stopwatch_start,
+                                :zmq_stopwatch_intermediate,
+                                :zmq_stopwatch_stop,
+                                :zmq_sleep,
+                                :zmq_threadstart,
+                                :zmq_threadclose]
+function get_docs(node, doc)
+    # Only add docstrings for functions
+    if !(node.type isa FunctionProto)
+        return doc
+    end
+
+    url_prefix = "https://libzmq.readthedocs.io/en/latest"
+
+    # The timer functions are all documented on a single page
+    if startswith(string(node.id), "zmq_timers")
+        return ["[Upstream documentation]($(url_prefix)/zmq_timers.html)."]
+    elseif node.id in undocumented_functions
+        return ["This is an undocumented function, not part of the formal ZMQ API."]
+    else
+        # For all the others, generate the URL from the function name
+        return ["[Upstream documentation]($(url_prefix)/$(node.id).html)."]
+    end
+end
+
+cd(@__DIR__) do
+    # Set the options
+    options = Clang.load_options(joinpath(@__DIR__, "generator.toml"))
+    options["general"]["callback_documentation"] = get_docs
+    header = joinpath(ZeroMQ_jll.artifact_dir, "include", "zmq.h")
+    args = Clang.get_default_args()
+
+    # Generate the generic bindings
+    ctx = Clang.create_context([header], args, options)
+    Clang.build!(ctx)
+
+    # Generate the Message methods we need
+    module_name = Symbol(options["general"]["module_name"])
+    msg_methods = get_msg_methods(ctx, module_name)
+    output_file = joinpath(@__DIR__, "../src/msg_bindings.jl")
+    open(output_file; write=true) do io
+        # Import symbols required by the bindings
+        write(io, "import ZeroMQ_jll: libzmq\n")
+        write(io, "import .lib: zmq_free_fn\n\n")
+
+        for expr in msg_methods
+            write(io, string(expr), "\n\n")
+        end
+    end
+end

gen/generator.toml

@@ -0,0 +1,15 @@
+[general]
+library_name = "libzmq"
+module_name = "lib"
+output_file_path = "../src/bindings.jl"
+print_using_CEnum = false
+output_ignorelist = ["ZMQ_VERSION", # This macro cannot be parsed by Clang.jl
+                     # These functions/types are deprecated
+                     "zmq_init", "zmq_term", "zmq_ctx_destroy",
+                     "zmq_device", "zmq_sendmsg", "zmq_recvmsg",
+                     "iovec", "zmq_sendiov", "zmq_recviov"]
+prologue_file_path = "./prologue.jl"
+
+auto_mutability = true
+auto_mutability_with_new = false
+auto_mutability_includelist = ["zmq_pollitem_t"]

gen/prologue.jl

@@ -0,0 +1 @@
+import ZeroMQ_jll: libzmq

src/ZMQ.jl

@@ -1,7 +1,6 @@
 # Support for ZeroMQ, a network and interprocess communication library
 
 module ZMQ
-import ZeroMQ_jll: libzmq
 
 using Base.Libc: EAGAIN
 using FileWatching: UV_READABLE, uv_pollcb, FDWatcher
@@ -19,14 +18,15 @@ export
     #Sockets
     connect, bind, send, recv
 
-
+include("bindings.jl")
 include("constants.jl")
 include("optutil.jl")
 include("error.jl")
 include("context.jl")
 include("socket.jl")
 include("sockopts.jl")
 include("message.jl")
+include("msg_bindings.jl")
 include("comm.jl")
 
 """
@@ -38,7 +38,7 @@ function lib_version()
     major = Ref{Cint}()
     minor = Ref{Cint}()
     patch = Ref{Cint}()
-    ccall((:zmq_version, libzmq), Cvoid, (Ptr{Cint}, Ptr{Cint}, Ptr{Cint}), major, minor, patch)
+    lib.zmq_version(major, minor, patch)
     return VersionNumber(major[], minor[], patch[])
 end
 

src/_message.jl

@@ -3,34 +3,33 @@
 
 # Low-level message type, matching the declaration of
 # zmq_msg_t in the header: char _[64];
-primitive type _Message 64 * 8 end
+const _Message = lib.zmq_msg_t
 
-const _MessageOrRef = Union{_Message,Base.RefValue{_Message}}
+const _MessageRef = Base.RefValue{_Message}
 
 function msg_init()
     zmsg = Ref{_Message}()
-    rc = ccall((:zmq_msg_init, libzmq), Cint, (Ref{_Message},), zmsg)
+    rc = lib.zmq_msg_init(zmsg)
     rc != 0 && throw(StateError(jl_zmq_error_str()))
     return zmsg
 end
 
 function msg_init(nbytes::Int)
     zmsg = Ref{_Message}()
-    rc = ccall((:zmq_msg_init_size, libzmq), Cint, (Ref{_Message}, Csize_t), zmsg, nbytes % Csize_t)
+    rc = lib.zmq_msg_init_size(zmsg, nbytes % Csize_t)
     rc != 0 && throw(StateError(jl_zmq_error_str()))
     return zmsg
 end
 
 # note: no finalizer for _Message, so we need to call close manually!
-function Base.close(zmsg::_MessageOrRef)
-    rc = ccall((:zmq_msg_close, libzmq), Cint, (Ref{_Message},), zmsg)
+function Base.close(zmsg::_MessageRef)
+    rc = lib.zmq_msg_close(zmsg)
     rc != 0 && throw(StateError(jl_zmq_error_str()))
     return nothing
 end
 
-Base.length(zmsg::_MessageOrRef) = ccall((:zmq_msg_size, libzmq), Csize_t, (Ref{_Message},), zmsg) % Int
-Base.unsafe_convert(::Type{Ptr{UInt8}}, zmsg::_MessageOrRef) =
-    ccall((:zmq_msg_data, libzmq), Ptr{UInt8}, (Ref{_Message},), zmsg)
+Base.length(zmsg::Base.RefValue{_Message}) = lib.zmq_msg_size(zmsg) % Int
+Base.unsafe_convert(::Type{Ptr{UInt8}}, zmsg::_MessageRef) = Ptr{UInt8}(lib.zmq_msg_data(zmsg))
 
 # isbits data, vectors thereof, and strings can be converted to/from _Message
 
@@ -57,7 +56,7 @@ function _MessageRef(x::String)
     return zmsg
 end
 
-function unsafe_copy(::Type{Vector{T}}, zmsg::_MessageOrRef) where {T}
+function unsafe_copy(::Type{Vector{T}}, zmsg::_MessageRef) where {T}
     isbitstype(T) || throw(MethodError(unsafe_copy, (T, zmsg,)))
     n = length(zmsg)
     len, remainder = divrem(n, sizeof(T))
@@ -67,16 +66,16 @@ function unsafe_copy(::Type{Vector{T}}, zmsg::_MessageOrRef) where {T}
     return a
 end
 
-function unsafe_copy(::Type{T}, zmsg::_MessageOrRef) where {T}
+function unsafe_copy(::Type{T}, zmsg::_MessageRef) where {T}
     isbitstype(T) || throw(MethodError(unsafe_copy, (T, zmsg,)))
     n = length(zmsg)
     n == sizeof(T) || error("message length $n ≠ sizeof($T)")
     return @preserve zmsg unsafe_load(Ptr{T}(Base.unsafe_convert(Ptr{UInt8}, zmsg)))
 end
 
-function unsafe_copy(::Type{String}, zmsg::_MessageOrRef)
+function unsafe_copy(::Type{String}, zmsg::_MessageRef)
     n = length(zmsg)
     return @preserve zmsg unsafe_string(Base.unsafe_convert(Ptr{UInt8}, zmsg), n)
 end
 
-unsafe_copy(::Type{IOBuffer}, zmsg::_MessageOrRef) = IOBuffer(unsafe_copy(Vector{UInt8}, zmsg))
+unsafe_copy(::Type{IOBuffer}, zmsg::_MessageRef) = IOBuffer(unsafe_copy(Vector{UInt8}, zmsg))