From 7338f6b2b27c40ef8748551bed3e89905322f151 Mon Sep 17 00:00:00 2001 From: Alejandro Pedraza Date: Fri, 5 Apr 2024 10:04:58 -0500 Subject: [PATCH] fix(transport): Handle IPv4-mapped IPv6 addrs in SO_ORIGINAL_DST (#2841) In 9ee109a954cf5e267d19e52bb7dd73b0a26d79ae we generalized the `so_original_dst` function to first retrieve the socket's `SO_DOMAIN` in order to determine what constants to use to retrieve the original destination, that would work for both IP families. This works fine when the socket is bound to `0.0.0.0` and receives a connection to the pod's IPv4 address, or when bound to `::` and receives a connection to the pod's IPv6 address. When the cluster is in dual-stack mode and we bind the socket to `::`, it properly receives connections targeted at both the pod's IPv4 and IPv6 addresses, but in the former case the `getsockopt` call to retrieve the original destination fails with: ``` WARN ThreadId(01) inbound: linkerd_app_core::serve: Server failed to accept connection error=No such file or directory (os error 2) ``` The problem is that in this case we're dealing with an IPv4-mapped IPv6 address so `SO_DOMAIN` returns `AF_INET6` but apparently in this case we require `AF_INET` when interrogating `getsockopt` for the original destination. To fix, instead of asking for `SO_DOMAIN` we determine the peer's IP family and use that to set the appropriate constants for `getsocketopt`. --- linkerd/proxy/transport/src/orig_dst.rs | 75 ++++++++++++++----------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/linkerd/proxy/transport/src/orig_dst.rs b/linkerd/proxy/transport/src/orig_dst.rs index 746bacdf38..aaaa3f0c91 100644 --- a/linkerd/proxy/transport/src/orig_dst.rs +++ b/linkerd/proxy/transport/src/orig_dst.rs @@ -6,7 +6,7 @@ use futures::prelude::*; use linkerd_error::Result; use linkerd_io as io; use linkerd_stack::Param; -use std::pin::Pin; +use std::{net::SocketAddr, pin::Pin}; use tokio::net::TcpStream; #[derive(Copy, Clone, Debug, Default)] @@ -70,6 +70,7 @@ impl From for BindWithOrigDst { impl Bind for BindWithOrigDst where B: Bind + 'static, + B::Addrs: Param>, { type Addrs = Addrs; type Io = TcpStream; @@ -81,7 +82,18 @@ where let incoming = incoming.map(|res| { let (inner, tcp) = res?; - let orig_dst = orig_dst_addr(&tcp)?; + let is_ipv4 = match inner.param() { + // used when binding to 0.0.0.0 + Remote(ClientAddr(SocketAddr::V4(_))) => true, + // used when binding to :: + Remote(ClientAddr(SocketAddr::V6(a))) => a.ip().to_ipv4_mapped().is_some(), + }; + let orig_dst = if is_ipv4 { + orig_dst_addr_v4(&tcp)? + } else { + orig_dst_addr_v6(&tcp)? + }; + let orig_dst = OrigDstAddr(orig_dst); let addrs = Addrs { inner, orig_dst }; Ok((addrs, tcp)) }); @@ -92,16 +104,32 @@ where #[cfg(target_os = "linux")] #[allow(unsafe_code)] -fn orig_dst_addr(sock: &TcpStream) -> io::Result { +fn orig_dst_addr_v4(sock: &TcpStream) -> io::Result { use std::os::unix::io::AsRawFd; let fd = sock.as_raw_fd(); - let r = unsafe { linux::so_original_dst(fd) }; - r.map(OrigDstAddr) + unsafe { linux::so_original_dst_v4(fd) } +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn orig_dst_addr_v6(sock: &TcpStream) -> io::Result { + use std::os::unix::io::AsRawFd; + + let fd = sock.as_raw_fd(); + unsafe { linux::so_original_dst_v6(fd) } +} + +#[cfg(not(target_os = "linux"))] +fn orig_dst_addr_v4(_: &TcpStream) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Other, + "SO_ORIGINAL_DST not supported on this operating system", + )) } #[cfg(not(target_os = "linux"))] -fn orig_dst_addr(_: &TcpStream) -> io::Result { +fn orig_dst_addr_v6(_: &TcpStream) -> io::Result { Err(io::Error::new( io::ErrorKind::Other, "SO_ORIGINAL_DST not supported on this operating system", @@ -115,35 +143,10 @@ mod linux { use std::os::unix::io::RawFd; use std::{io, mem}; - pub unsafe fn so_original_dst(fd: RawFd) -> io::Result { - let mut sockdomain: i32 = 0; - let mut sockdomain_len: libc::socklen_t = 32; - + pub unsafe fn so_original_dst(fd: RawFd, level: i32, optname: i32) -> io::Result { let mut sockaddr: libc::sockaddr_storage = mem::zeroed(); let mut sockaddr_len: libc::socklen_t = mem::size_of::() as u32; - let ret = libc::getsockopt( - fd, - libc::SOL_SOCKET, - libc::SO_DOMAIN, - &mut sockdomain as *mut _ as *mut _, - &mut sockdomain_len as *mut _ as *mut _, - ); - if ret != 0 { - return Err(io::Error::last_os_error()); - } - - let (level, optname) = match sockdomain { - libc::AF_INET => (libc::SOL_IP, libc::SO_ORIGINAL_DST), - libc::AF_INET6 => (libc::SOL_IPV6, libc::IP6T_SO_ORIGINAL_DST), - x => { - return Err(io::Error::new( - io::ErrorKind::Unsupported, - format!("unknown SO_DOMAIN: {x}"), - )); - } - }; - let ret = libc::getsockopt( fd, level, @@ -158,6 +161,14 @@ mod linux { mk_addr(&sockaddr, sockaddr_len) } + pub unsafe fn so_original_dst_v4(fd: RawFd) -> io::Result { + so_original_dst(fd, libc::SOL_IP, libc::SO_ORIGINAL_DST) + } + + pub unsafe fn so_original_dst_v6(fd: RawFd) -> io::Result { + so_original_dst(fd, libc::SOL_IPV6, libc::IP6T_SO_ORIGINAL_DST) + } + // Borrowed with love from net2-rs // https://github.com/rust-lang-nursery/net2-rs/blob/1b4cb4fb05fbad750b271f38221eab583b666e5e/src/socket.rs#L103 //