Add rackscale ethernet multiclient integration test

vmware-labs · Jul 6, 2023 · b822e1b · b822e1b
1 parent 0091044
commit b822e1b
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 17 deletions.
diff --git a/kernel/src/arch/x86_64/rackscale/client_state.rs b/kernel/src/arch/x86_64/rackscale/client_state.rs
@@ -16,6 +16,7 @@ use rpc::client::Client;
 use rpc::rpc::RPCError;
 
 use crate::arch::kcb::try_per_core_mem;
+use crate::arch::rackscale::controller::CONTROLLER_PORT_BASE;
 use crate::arch::rackscale::fileops::rw::{RW_SHMEM_BUF, RW_SHMEM_BUF_LEN};
 use crate::arch::rackscale::FrameCacheBase;
 use crate::arch::MAX_MACHINES;
@@ -47,7 +48,7 @@ impl ClientState {
             Arc::new(Mutex::new(
                 crate::transport::ethernet::init_ethernet_rpc(
                     smoltcp::wire::IpAddress::v4(172, 31, 0, 11),
-                    6970,
+                    CONTROLLER_PORT_BASE + (*crate::environment::MACHINE_ID as u16 - 1),
                     true,
                 )
                 .expect("Failed to initialize ethernet RPC"),

diff --git a/kernel/src/arch/x86_64/rackscale/controller.rs b/kernel/src/arch/x86_64/rackscale/controller.rs
@@ -19,7 +19,7 @@ use crate::transport::shmem::create_shmem_transport;
 
 use super::*;
 
-const PORT: u16 = 6970;
+pub(crate) const CONTROLLER_PORT_BASE: u16 = 6970;
 
 /// Controller main method
 pub(crate) fn run() {
@@ -51,19 +51,21 @@ pub(crate) fn run() {
         .get()
         .map_or(false, |c| c.transport == Transport::Ethernet)
     {
-        if num_clients > 1 {
-            panic!("Ethernet transport only supports on client, currently");
-        }
-
-        let transport = Box::try_new(
-            TCPTransport::new(None, PORT, Arc::clone(&ETHERNET_IFACE))
+        for mid in 0..num_clients {
+            let transport = Box::try_new(
+                TCPTransport::new(
+                    None,
+                    CONTROLLER_PORT_BASE + mid as u16,
+                    Arc::clone(&ETHERNET_IFACE),
+                )
                 .expect("Failed to create TCP transport"),
-        )
-        .expect("Out of memory during init");
-        let mut server: Box<dyn RPCServer<ControllerState>> =
-            Box::try_new(Server::new(transport)).expect("Out of memory during init");
-        register_rpcs(&mut server);
-        servers.push(server);
+            )
+            .expect("Out of memory during init");
+            let mut server: Box<dyn RPCServer<ControllerState>> =
+                Box::try_new(Server::new(transport)).expect("Out of memory during init");
+            register_rpcs(&mut server);
+            servers.push(server);
+        }
     } else if crate::CMDLINE
         .get()
         .map_or(false, |c| c.transport == Transport::Shmem)

diff --git a/kernel/tests/s06_rackscale_tests.rs b/kernel/tests/s06_rackscale_tests.rs
@@ -942,9 +942,20 @@ fn rackscale_userspace_multicore_test(is_shmem: bool) {
     controller_ret.unwrap();
 }
 
+#[cfg(not(feature = "baremetal"))]
+#[test]
+fn s06_rackscale_ethernet_userspace_multicore_multiclient() {
+    rackscale_userspace_multicore_multiclient(false);
+}
+
 #[cfg(not(feature = "baremetal"))]
 #[test]
 fn s06_rackscale_shmem_userspace_multicore_multiclient() {
+    rackscale_userspace_multicore_multiclient(true);
+}
+
+#[cfg(not(feature = "baremetal"))]
+fn rackscale_userspace_multicore_multiclient(is_shmem: bool) {
     use std::sync::Arc;
     use std::thread::sleep;
     use std::time::Duration;
@@ -993,9 +1004,14 @@ fn s06_rackscale_shmem_userspace_multicore_multiclient() {
     let controller = std::thread::Builder::new()
         .name("Controller".to_string())
         .spawn(move || {
+            let controller_cmd = if is_shmem {
+                "mode=controller transport=shmem"
+            } else {
+                "mode=controller transport=ethernet"
+            };
             let cmdline_controller = RunnerArgs::new_with_build("userspace-smp", &controller_build)
                 .timeout(timeout)
-                .cmd("mode=controller transport=shmem")
+                .cmd(controller_cmd)
                 .shmem_size(vec![SHMEM_SIZE as usize; 3])
                 .shmem_path(shmem_sockets)
                 .tap("tap0")
@@ -1047,10 +1063,15 @@ fn s06_rackscale_shmem_userspace_multicore_multiclient() {
     let client = std::thread::Builder::new()
         .name("Client1".to_string())
         .spawn(move || {
+            let client_cmd = if is_shmem {
+                "mode=client transport=shmem"
+            } else {
+                "mode=client transport=ethernet"
+            };
             sleep(Duration::from_millis(CLIENT_BUILD_DELAY));
             let cmdline_client = RunnerArgs::new_with_build("userspace-smp", &client1_build)
                 .timeout(timeout)
-                .cmd("mode=client transport=shmem")
+                .cmd(client_cmd)
                 .shmem_size(vec![SHMEM_SIZE as usize; 3])
                 .shmem_path(shmem_sockets)
                 .tap("tap2")
@@ -1103,10 +1124,15 @@ fn s06_rackscale_shmem_userspace_multicore_multiclient() {
     let client2 = std::thread::Builder::new()
         .name("Client2".to_string())
         .spawn(move || {
+            let client_cmd = if is_shmem {
+                "mode=client transport=shmem"
+            } else {
+                "mode=client transport=ethernet"
+            };
             sleep(Duration::from_millis(CLIENT_BUILD_DELAY * 2));
             let cmdline_client = RunnerArgs::new_with_build("userspace-smp", &client2_build)
                 .timeout(timeout)
-                .cmd("mode=client transport=shmem")
+                .cmd(client_cmd)
                 .shmem_size(vec![SHMEM_SIZE as usize; 3])
                 .shmem_path(shmem_sockets)
                 .tap("tap4")