diff --git a/cloud/blockstore/libs/nbd/netlink_device.cpp b/cloud/blockstore/libs/nbd/netlink_device.cpp index 6e9bd52779..b7ac8f01a2 100644 --- a/cloud/blockstore/libs/nbd/netlink_device.cpp +++ b/cloud/blockstore/libs/nbd/netlink_device.cpp @@ -3,6 +3,7 @@ #include "utils.h" #include +#include #include @@ -19,177 +20,17 @@ namespace { using namespace NThreading; -using TResponseHandler = std::function; - //////////////////////////////////////////////////////////////////////////////// constexpr TStringBuf NBD_DEVICE_SUFFIX = "/dev/nbd"; //////////////////////////////////////////////////////////////////////////////// -class TNetlinkSocket -{ -private: - nl_sock* Socket; - int Family; - -public: - TNetlinkSocket() - { - Socket = nl_socket_alloc(); - - if (Socket == nullptr) { - throw TServiceError(E_FAIL) << "unable to allocate netlink socket"; - } - - if (int err = genl_connect(Socket)) { - nl_socket_free(Socket); - throw TServiceError(E_FAIL) - << "unable to connect to generic netlink socket: " - << nl_geterror(err); - } - - Family = genl_ctrl_resolve(Socket, "nbd"); - - if (Family < 0) { - nl_socket_free(Socket); - throw TServiceError(E_FAIL) - << "unable to resolve nbd netlink family: " - << nl_geterror(Family); - } - } - - ~TNetlinkSocket() - { - nl_socket_free(Socket); - } - - int GetFamily() const - { - return Family; - } - - template - void SetCallback(nl_cb_type type, F func) - { - auto arg = std::make_unique(std::move(func)); - - if (int err = nl_socket_modify_cb( - Socket, - type, - NL_CB_CUSTOM, - TNetlinkSocket::ResponseHandler, - arg.get())) - { - throw TServiceError(E_FAIL) - << "unable to set socket callback: " << nl_geterror(err); - } - arg.release(); - } - - static int ResponseHandler(nl_msg* msg, void* arg) - { - auto func = std::unique_ptr( - static_cast(arg)); - - return (*func)(static_cast(nlmsg_data(nlmsg_hdr(msg)))); - } - - void Send(nl_msg* message) - { - if (int err = nl_send_auto(Socket, message); err < 0) { - throw TServiceError(E_FAIL) - << "send error: " << nl_geterror(err); - } - if (int err = nl_wait_for_ack(Socket)) { - // this is either recv error, or an actual error message received - // from the kernel - throw TServiceError(E_FAIL) - << "recv error: " << nl_geterror(err); - } - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TNestedAttribute -{ -private: - nl_msg* Message; - nlattr* Attribute; - -public: - TNestedAttribute(nl_msg* message, int attribute) - : Message(message) - { - Attribute = nla_nest_start(message, attribute); - if (!Attribute) { - throw TServiceError(E_FAIL) << "unable to nest attribute"; - } - } - - ~TNestedAttribute() - { - nla_nest_end(Message, Attribute); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TNetlinkMessage -{ -private: - nl_msg* Message; - -public: - TNetlinkMessage(int family, int command) - { - Message = nlmsg_alloc(); - if (Message == nullptr) { - throw TServiceError(E_FAIL) << "unable to allocate message"; - } - genlmsg_put( - Message, - NL_AUTO_PORT, - NL_AUTO_SEQ, - family, - 0, // hdrlen - 0, // flags - command, - 0); // version - } - - ~TNetlinkMessage() - { - nlmsg_free(Message); - } - - operator nl_msg*() const - { - return Message; - } - - template - void Put(int attribute, T data) - { - if (int err = nla_put(Message, attribute, sizeof(T), &data)) { - throw TServiceError(E_FAIL) << "unable to put attribute " - << attribute << ": " << nl_geterror(err); - } - } - - TNestedAttribute Nest(int attribute) - { - return TNestedAttribute(Message, attribute); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - class TNetlinkDevice final : public IDevice , public std::enable_shared_from_this { + using TNetlinkMessage = NCloud::NNetlink::TNetlinkMessage; private: const ILoggingServicePtr Logging; const TNetworkAddress ConnectAddress; @@ -231,7 +72,7 @@ class TNetlinkDevice final void Disconnect(); void DoConnect(bool connected); - int StatusHandler(genlmsghdr* header); + int StatusHandler(nl_msg* nlmsg); }; //////////////////////////////////////////////////////////////////////////////// @@ -309,8 +150,8 @@ TFuture TNetlinkDevice::Stop(bool deleteDevice) TFuture TNetlinkDevice::Resize(ui64 deviceSizeInBytes) { try { - TNetlinkSocket socket; - TNetlinkMessage message(socket.GetFamily(), NBD_CMD_RECONFIGURE); + auto socket = NCloud::NNetlink::CreateNetlinkSocket("nbd"); + TNetlinkMessage message(socket->GetFamily(), NBD_CMD_RECONFIGURE); message.Put(NBD_ATTR_INDEX, DeviceIndex); message.Put(NBD_ATTR_SIZE_BYTES, deviceSizeInBytes); @@ -321,7 +162,7 @@ TFuture TNetlinkDevice::Resize(ui64 deviceSizeInBytes) message.Put(NBD_SOCK_FD, static_cast(Socket)); } - socket.Send(message); + socket->Send(message); } catch (const TServiceError& e) { return MakeFuture(MakeError( @@ -373,26 +214,25 @@ void TNetlinkDevice::DisconnectSocket() // or reconfigure (if Reconfigure == true) specified device void TNetlinkDevice::Connect() { - TNetlinkSocket socket; - socket.SetCallback( + auto socket = NCloud::NNetlink::CreateNetlinkSocket("nbd"); + socket->SetCallback( NL_CB_VALID, - [device = shared_from_this()] (auto* header) { - return device->StatusHandler(header); - }); + [device = shared_from_this()](nl_msg* nlmsg) + { return device->StatusHandler(nlmsg); }); - TNetlinkMessage message(socket.GetFamily(), NBD_CMD_STATUS); + TNetlinkMessage message(socket->GetFamily(), NBD_CMD_RECONFIGURE); message.Put(NBD_ATTR_INDEX, DeviceIndex); - socket.Send(message); + socket->Send(message); } void TNetlinkDevice::Disconnect() { STORAGE_INFO("disconnect " << DeviceName); - TNetlinkSocket socket; - TNetlinkMessage message(socket.GetFamily(), NBD_CMD_DISCONNECT); + auto socket = NCloud::NNetlink::CreateNetlinkSocket("nbd"); + TNetlinkMessage message(socket->GetFamily(), NBD_CMD_DISCONNECT); message.Put(NBD_ATTR_INDEX, DeviceIndex); - socket.Send(message); + socket->Send(message); StopResult.SetValue(MakeError(S_OK)); } @@ -410,8 +250,8 @@ void TNetlinkDevice::DoConnect(bool connected) STORAGE_INFO("connect " << DeviceName); } - TNetlinkSocket socket; - TNetlinkMessage message(socket.GetFamily(), command); + auto socket = NCloud::NNetlink::CreateNetlinkSocket("nbd"); + TNetlinkMessage message(socket->GetFamily(), command); const auto& info = Handler->GetExportInfo(); message.Put(NBD_ATTR_INDEX, DeviceIndex); @@ -437,7 +277,7 @@ void TNetlinkDevice::DoConnect(bool connected) message.Put(NBD_SOCK_FD, static_cast(Socket)); } - socket.Send(message); + socket->Send(message); StartResult.SetValue(MakeError(S_OK)); } catch (const TServiceError& e) { @@ -448,8 +288,9 @@ void TNetlinkDevice::DoConnect(bool connected) } } -int TNetlinkDevice::StatusHandler(genlmsghdr* header) +int TNetlinkDevice::StatusHandler(nl_msg* nlmsg) { + auto header = static_cast(nlmsg_data(nlmsg_hdr(nlmsg))); nlattr* attr[NBD_ATTR_MAX + 1] = {}; nlattr* deviceItem[NBD_DEVICE_ITEM_MAX + 1] = {}; nlattr* device[NBD_DEVICE_ATTR_MAX + 1] = {}; diff --git a/cloud/blockstore/libs/nbd/ya.make b/cloud/blockstore/libs/nbd/ya.make index 01d3adf187..4fd90be936 100644 --- a/cloud/blockstore/libs/nbd/ya.make +++ b/cloud/blockstore/libs/nbd/ya.make @@ -30,6 +30,7 @@ PEERDIR( cloud/blockstore/libs/diagnostics cloud/blockstore/libs/service cloud/storage/core/libs/coroutine + cloud/storage/core/libs/netlink contrib/libs/linux-headers contrib/restricted/libnl/lib/nl-3 contrib/restricted/libnl/lib/nl-genl-3 diff --git a/cloud/storage/core/libs/netlink/netlink_message.h b/cloud/storage/core/libs/netlink/netlink_message.h new file mode 100644 index 0000000000..421adaad90 --- /dev/null +++ b/cloud/storage/core/libs/netlink/netlink_message.h @@ -0,0 +1,83 @@ +#include + + #include + #include + + namespace NCloud::NNetlink { + + //////////////////////////////////////////////////////////////////////////////// + + class TNestedAttribute + { + private: + nl_msg* Message; + nlattr* Attribute; + + public: + TNestedAttribute(nl_msg* message, int attribute) + : Message(message) + { + Attribute = nla_nest_start(message, attribute); + if (!Attribute) { + throw TServiceError(E_FAIL) << "unable to nest attribute"; + } + } + + ~TNestedAttribute() + { + nla_nest_end(Message, Attribute); + } + }; + + //////////////////////////////////////////////////////////////////////////////// + + class TNetlinkMessage + { + private: + nl_msg* Message; + + public: + TNetlinkMessage(int family, int command, int flags = 0, int version = 0) + { + Message = nlmsg_alloc(); + if (Message == nullptr) { + throw TServiceError(E_FAIL) << "unable to allocate message"; + } + genlmsg_put( + Message, + NL_AUTO_PORT, + NL_AUTO_SEQ, + family, + 0, // hdrlen + flags, + command, + version); + } + + ~TNetlinkMessage() + { + nlmsg_free(Message); + } + + operator nl_msg*() const + { + return Message; + } + + template + void Put(int attribute, T data) + { + if (int err = nla_put(Message, attribute, sizeof(T), &data)) { + throw TServiceError(E_FAIL) + << "unable to put attribute " << attribute << ": " + << nl_geterror(err); + } + } + + TNestedAttribute Nest(int attribute) + { + return TNestedAttribute(Message, attribute); + } + }; + } // namespace NCloud::NNetlink + diff --git a/cloud/storage/core/libs/netlink/netlink_socket.cpp b/cloud/storage/core/libs/netlink/netlink_socket.cpp new file mode 100644 index 0000000000..a007144d2c --- /dev/null +++ b/cloud/storage/core/libs/netlink/netlink_socket.cpp @@ -0,0 +1,98 @@ +#include "netlink_socket.h" + +#include +#include + +namespace NCloud::NNetlink { + +namespace { + +using TResponseHandler = std::function; + +class TNetlinkSocket: public INetlinkSocket +{ +private: + nl_sock* Socket; + int Family; + +public: + explicit TNetlinkSocket(TString netlinkFamily) + { + Socket = nl_socket_alloc(); + + if (Socket == nullptr) { + throw TServiceError(E_FAIL) << "unable to allocate netlink socket"; + } + + if (int err = genl_connect(Socket)) { + nl_socket_free(Socket); + throw TServiceError(E_FAIL) + << "unable to connect to generic netlink socket: " + << nl_geterror(err); + } + + Family = genl_ctrl_resolve(Socket, netlinkFamily.c_str()); + + if (Family < 0) { + nl_socket_free(Socket); + throw TServiceError(E_FAIL) + << "unable to resolve netlink family: " << nl_geterror(Family); + } + } + + ~TNetlinkSocket() override + { + nl_socket_free(Socket); + } + + [[nodiscard]] int GetFamily() const override + { + return Family; + } + + void SetCallback(nl_cb_type type, TNetlinkSocketCallback func) override + { + auto arg = std::make_unique(std::move(func)); + + if (int err = nl_socket_modify_cb( + Socket, + type, + NL_CB_CUSTOM, + TNetlinkSocket::ResponseHandler, + arg.get())) + { + throw TServiceError(E_FAIL) + << "unable to set socket callback: " << nl_geterror(err); + } + arg.release(); + } + + static int ResponseHandler(nl_msg* msg, void* arg) + { + auto func = std::unique_ptr( + static_cast(arg)); + + return (*func)(msg); + } + + void Send(nl_msg* message) override + { + if (int err = nl_send_auto(Socket, message); err < 0) { + throw TServiceError(E_FAIL) << "send error: " << nl_geterror(err); + } + if (int err = nl_wait_for_ack(Socket)) { + // this is either recv error, or an actual error message received + // from the kernel + throw TServiceError(E_FAIL) << "recv error: " << nl_geterror(err); + } + } +}; + +} // namespace + +INetlinkSocketPtr CreateNetlinkSocket(TString netlinkFamily) +{ + return std::make_unique(std::move(netlinkFamily)); +} + +} // namespace NCloud::NNetlink diff --git a/cloud/storage/core/libs/netlink/netlink_socket.h b/cloud/storage/core/libs/netlink/netlink_socket.h new file mode 100644 index 0000000000..0d672bb02e --- /dev/null +++ b/cloud/storage/core/libs/netlink/netlink_socket.h @@ -0,0 +1,31 @@ +#pragma once + + #include "netlink_message.h" + + #include + #include + + #include + + namespace NCloud::NNetlink { + + class INetlinkSocket { + public: + using TNetlinkSocketCallback = std::function; + + virtual ~INetlinkSocket() = default; + + [[nodiscard]] virtual int GetFamily() const = 0; + + virtual void SetCallback(nl_cb_type type, TNetlinkSocketCallback func) = 0; + + virtual void Send(nl_msg* message) = 0; + }; + + + using INetlinkSocketPtr = std::unique_ptr; + + + INetlinkSocketPtr CreateNetlinkSocket(TString netlinkFamily); + + } // namespace NCloud::NNetlink diff --git a/cloud/storage/core/libs/netlink/ya.make b/cloud/storage/core/libs/netlink/ya.make new file mode 100644 index 0000000000..e666c66246 --- /dev/null +++ b/cloud/storage/core/libs/netlink/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + + SRCS( + netlink_socket.cpp + ) + + PEERDIR( + cloud/storage/core/libs/common + contrib/restricted/libnl/lib/nl-3 + contrib/restricted/libnl/lib/nl-genl-3 + ) + + END() + diff --git a/cloud/storage/core/libs/ya.make b/cloud/storage/core/libs/ya.make index 2056c32117..65a50cbab8 100644 --- a/cloud/storage/core/libs/ya.make +++ b/cloud/storage/core/libs/ya.make @@ -13,6 +13,7 @@ RECURSE( hive_proxy http kikimr + netlink ss_proxy tablet throttling