fix(transport): Handle IPv4-mapped IPv6 addrs in SO_ORIGINAL_DST (#2841)

In 9ee109a we generalized the `so_original_dst` function to first retrieve the socket's `SO_DOMAIN` in order to determine what constants to use to retrieve the original destination, that would work for both IP families.

This works fine when the socket is bound to `0.0.0.0` and receives a connection to the pod's IPv4 address, or when bound to `::` and receives a connection to the pod's IPv6 address.
When the cluster is in dual-stack mode and we bind the socket to `::`, it properly receives connections targeted at both the pod's IPv4 and IPv6 addresses, but in the former case the `getsockopt` call to retrieve the original destination fails with:

```
WARN ThreadId(01) inbound: linkerd_app_core::serve: Server failed to accept connection error=No such file or directory (os error 2)
```

The problem is that in this case we're dealing with an IPv4-mapped IPv6 address so `SO_DOMAIN` returns `AF_INET6` but apparently in this case we require `AF_INET` when interrogating `getsockopt` for the original destination.

To fix, instead of asking for `SO_DOMAIN` we determine the peer's IP family and use that to set the appropriate constants for `getsocketopt`.

linkerd/proxy/transport/src/orig_dst.rs

@@ -6,7 +6,7 @@ use futures::prelude::*;
 use linkerd_error::Result;
 use linkerd_io as io;
 use linkerd_stack::Param;
-use std::pin::Pin;
+use std::{net::SocketAddr, pin::Pin};
 use tokio::net::TcpStream;
 
 #[derive(Copy, Clone, Debug, Default)]
@@ -70,6 +70,7 @@ impl<B> From<B> for BindWithOrigDst<B> {
 impl<T, B> Bind<T> for BindWithOrigDst<B>
 where
     B: Bind<T, Io = TcpStream> + 'static,
+    B::Addrs: Param<Remote<ClientAddr>>,
 {
     type Addrs = Addrs<B::Addrs>;
     type Io = TcpStream;
@@ -81,7 +82,18 @@ where
 
         let incoming = incoming.map(|res| {
             let (inner, tcp) = res?;
-            let orig_dst = orig_dst_addr(&tcp)?;
+            let is_ipv4 = match inner.param() {
+                // used when binding to 0.0.0.0
+                Remote(ClientAddr(SocketAddr::V4(_))) => true,
+                // used when binding to ::
+                Remote(ClientAddr(SocketAddr::V6(a))) => a.ip().to_ipv4_mapped().is_some(),
+            };
+            let orig_dst = if is_ipv4 {
+                orig_dst_addr_v4(&tcp)?
+            } else {
+                orig_dst_addr_v6(&tcp)?
+            };
+            let orig_dst = OrigDstAddr(orig_dst);
             let addrs = Addrs { inner, orig_dst };
             Ok((addrs, tcp))
         });
@@ -92,16 +104,32 @@ where
 
 #[cfg(target_os = "linux")]
 #[allow(unsafe_code)]
-fn orig_dst_addr(sock: &TcpStream) -> io::Result<OrigDstAddr> {
+fn orig_dst_addr_v4(sock: &TcpStream) -> io::Result<SocketAddr> {
     use std::os::unix::io::AsRawFd;
 
     let fd = sock.as_raw_fd();
-    let r = unsafe { linux::so_original_dst(fd) };
-    r.map(OrigDstAddr)
+    unsafe { linux::so_original_dst_v4(fd) }
+}
+
+#[cfg(target_os = "linux")]
+#[allow(unsafe_code)]
+fn orig_dst_addr_v6(sock: &TcpStream) -> io::Result<SocketAddr> {
+    use std::os::unix::io::AsRawFd;
+
+    let fd = sock.as_raw_fd();
+    unsafe { linux::so_original_dst_v6(fd) }
+}
+
+#[cfg(not(target_os = "linux"))]
+fn orig_dst_addr_v4(_: &TcpStream) -> io::Result<OrigDstAddr> {
+    Err(io::Error::new(
+        io::ErrorKind::Other,
+        "SO_ORIGINAL_DST not supported on this operating system",
+    ))
 }
 
 #[cfg(not(target_os = "linux"))]
-fn orig_dst_addr(_: &TcpStream) -> io::Result<OrigDstAddr> {
+fn orig_dst_addr_v6(_: &TcpStream) -> io::Result<OrigDstAddr> {
     Err(io::Error::new(
         io::ErrorKind::Other,
         "SO_ORIGINAL_DST not supported on this operating system",
@@ -115,35 +143,10 @@ mod linux {
     use std::os::unix::io::RawFd;
     use std::{io, mem};
 
-    pub unsafe fn so_original_dst(fd: RawFd) -> io::Result<SocketAddr> {
-        let mut sockdomain: i32 = 0;
-        let mut sockdomain_len: libc::socklen_t = 32;
-
+    pub unsafe fn so_original_dst(fd: RawFd, level: i32, optname: i32) -> io::Result<SocketAddr> {
         let mut sockaddr: libc::sockaddr_storage = mem::zeroed();
         let mut sockaddr_len: libc::socklen_t = mem::size_of::<libc::sockaddr_storage>() as u32;
 
-        let ret = libc::getsockopt(
-            fd,
-            libc::SOL_SOCKET,
-            libc::SO_DOMAIN,
-            &mut sockdomain as *mut _ as *mut _,
-            &mut sockdomain_len as *mut _ as *mut _,
-        );
-        if ret != 0 {
-            return Err(io::Error::last_os_error());
-        }
-
-        let (level, optname) = match sockdomain {
-            libc::AF_INET => (libc::SOL_IP, libc::SO_ORIGINAL_DST),
-            libc::AF_INET6 => (libc::SOL_IPV6, libc::IP6T_SO_ORIGINAL_DST),
-            x => {
-                return Err(io::Error::new(
-                    io::ErrorKind::Unsupported,
-                    format!("unknown SO_DOMAIN: {x}"),
-                ));
-            }
-        };
-
         let ret = libc::getsockopt(
             fd,
             level,
@@ -158,6 +161,14 @@ mod linux {
         mk_addr(&sockaddr, sockaddr_len)
     }
 
+    pub unsafe fn so_original_dst_v4(fd: RawFd) -> io::Result<SocketAddr> {
+        so_original_dst(fd, libc::SOL_IP, libc::SO_ORIGINAL_DST)
+    }
+
+    pub unsafe fn so_original_dst_v6(fd: RawFd) -> io::Result<SocketAddr> {
+        so_original_dst(fd, libc::SOL_IPV6, libc::IP6T_SO_ORIGINAL_DST)
+    }
+
     // Borrowed with love from net2-rs
     // https://github.com/rust-lang-nursery/net2-rs/blob/1b4cb4fb05fbad750b271f38221eab583b666e5e/src/socket.rs#L103
     //