How to build librdkafka

[UebelAndre]

I saw this incredible thread in the slack channel and thought I'd re-post it here to make it more discoverable

David Lopez:

has someone been able to build the the rdkafka library using the cmake-build feature using bazel? Im getting panics after the following warnings:

Building and linking librdkafka statically
CMake Error: Could not find CMAKE_ROOT !!!
CMake has most likely not been installed correctly.
Modules directory not found in

I defined CMAKE_ROOT and CMAKE_MODULE_PATH in my shell but seems like bazel is ignoring it and it is using a different cmake version that the one that I have installed in my system (edited)

This then lead to a hero's reponse

Brian Myers:

Here's what my librdkafka looks like to help get you started. The others (zstd, lz4, zlib, openssl) should have examples if you google for them. I don't recall seeing any good librdkafka examples, so pasting here

load("@bazel_skylib//rules:write_file.bzl", "write_file")
load("@rules_cc//cc:defs.bzl", "cc_library")

alias(
    name = "librdkafka",
    actual = "rdkafka",
    visibility = ["//visibility:public"],
)

alias(
    name = "librdkafka++",
    actual = "rdkafka++",
    visibility = ["//visibility:public"],
)

write_file(
    name = "config_dot_h",
    out = "config.h",
    content = [
        "#define WITHOUT_OPTIMIZATION 0",
        "#define ENABLE_DEVEL 0",
        "#define ENABLE_REFCNT_DEBUG 0",
        "#define HAVE_ATOMICS_32 1",
        "#define HAVE_ATOMICS_32_SYNC 0",
        "#if (HAVE_ATOMICS_32)",
        "# if (HAVE_ATOMICS_32_SYNC)",
        "#  define ATOMIC_OP32(OP1,OP2,PTR,VAL) __sync_ ## OP1 ## _and_ ## OP2(PTR, VAL)",
        "# else",
        "#  define ATOMIC_OP32(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)",
        "# endif",
        "#endif",
        "#define HAVE_ATOMICS_64 1",
        "#define HAVE_ATOMICS_64_SYNC 0",
        "#if (HAVE_ATOMICS_64)",
        "# if (HAVE_ATOMICS_64_SYNC)",
        "#  define ATOMIC_OP64(OP1,OP2,PTR,VAL) __sync_ ## OP1 ## _and_ ## OP2(PTR, VAL)",
        "# else",
        "#  define ATOMIC_OP64(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)",
        "# endif",
        "#endif",
        "#define HAVE_RAND_R 1",
        "#define HAVE_REGEX 1",
        "#define HAVE_STRNDUP 1",
        "#define WITH_C11THREADS 0",
        "#define WITH_CRC32C_HW 0",
        "#define WITH_HDRHISTOGRAM 1",
        "#define WITH_LIBDL 1",
        "#define WITH_LZ4_EXT 1",
        "#define WITH_PKGCONFIG 1",
        "#define WITH_PLUGINS 1",
        "#define WITH_SASL 0",
        "#define WITH_SASL_CYRUS 0",
        "#define WITH_SASL_OAUTHBEARER 0",
        "#define WITH_SASL_SCRAM 0",
        "#define WITH_SNAPPY 1",
        "#define WITH_SOCKEM 1",
        "#define WITH_SSL 1",
        "#define WITH_ZLIB 1",
        "#define WITH_ZSTD 1",
    ] + select({
        "@//bazel/constraints/os:darwin": [
            "#define HAVE_PTHREAD_SETNAME_DARWIN 1",
            "#define HAVE_PTHREAD_SETNAME_FREEBSD 0",
            "#define HAVE_PTHREAD_SETNAME_GNU 0",
            "#define SOLIB_EXT \".dylib\"",
            # The compiler here is incorrect, but I don't think it matters
            "#define BUILT_WITH  \"CMAKE AppleClang AppleClang PKGCONFIG HDRHISTOGRAM ZLIB ZSTD LIBDL PLUGINS SSL LZ4_EXT SNAPPY SOCKEM\"",
        ],
        "@//bazel/constraints/os:linux": [
            "#define HAVE_PTHREAD_SETNAME_DARWIN 0",
            "#define HAVE_PTHREAD_SETNAME_FREEBSD 0",
            "#define HAVE_PTHREAD_SETNAME_GNU 1",
            "#define SOLIB_EXT \".so\"",
            "#define BUILT_WITH  \"CMAKE GNU GNU PKGCONFIG HDRHISTOGRAM ZLIB ZSTD LIBDL PLUGINS SSL LZ4_EXT SNAPPY SOCKEM\"",
        ],
        "//conditions:default": [],
    }),
)

cc_library(
    name = "config_header",
    hdrs = [
        "config.h",
    ],
)

cc_library(
    name = "rdkafka_header",
    hdrs = [
        "src/rdkafka.h",
    ],
)

cc_library(
    name = "rdkafka",
    srcs = glob(
        include = [
            "src/*.c",
            "src/*.h",
        ],
        exclude = [
            # don't use sasl
            "src/rdkafka_sasl_cyrus.c",
            "src/rdkafka_sasl_oauthbearer.c",
            "src/rdkafka_sasl_scram.c",
            "src/rdkafka_sasl_win32.c",
            # no need for win32
            "src/rdwin32.h",
            "src/win32_config.h",
            # don't use built-in laz4
            "src/lz4.c",
            "src/lz4frame.c",
            "src/lz4hc.c",
        ],
    ),
    hdrs = [
        "src/rdkafka.h",
    ],
    copts = [
        "-fPIC",
        "-Wall",
        # Keep log spew down when compiling with LLVM
        "-Wno-unused-but-set-variable",
    ],
    include_prefix = "librdkafka",
    strip_include_prefix = "src",
    deps = [
        ":config_header",
        "@com_github__facebook__zstd//:libzstd",
        "@com_github__lz4__lz4//:liblz4",
        "@com_github__openssl__openssl//:openssl",
        "@net_zlib__zlib//:libz",
    ],
)

cc_library(
    name = "rdkafka++",
    srcs = glob([
        "src-cpp/*.cpp",
        "src-cpp/*.h",
    ]),
    hdrs = [
        "src-cpp/rdkafkacpp.h",
    ],
    strip_include_prefix = "src-cpp",
    deps = [
        ":config_header",
        ":rdkafka_header",
    ],
)

I only have to support macos and linux by the way. Good luck if you care about windows
Oh, and make sure you pull the version of librdkafka that rdkafka-sys uses. rdkafka 0.28.0 uses https://github.com/benesch/librdkafka/archive/30633e59a9072b7b9c76411c281578e0125d5f65.tar.gz

Huge thanks to Brian for sharing!