From 91afc4374876b2f4d87017f5c4ae07b44ae228a7 Mon Sep 17 00:00:00 2001 From: Longxiang Lyu <35479537+lolyu@users.noreply.github.com> Date: Tue, 7 Nov 2023 06:42:56 +0800 Subject: [PATCH] [dualtor_neighbor_check] Adjust zero-mac check condition (#3034) * [dualtor_neighbor_check] Adjust zero-mac check condition --- scripts/dualtor_neighbor_check.py | 15 ++++--- tests/dualtor_neighbor_check_test.py | 66 ++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/scripts/dualtor_neighbor_check.py b/scripts/dualtor_neighbor_check.py index 161177008a..39de3c676f 100755 --- a/scripts/dualtor_neighbor_check.py +++ b/scripts/dualtor_neighbor_check.py @@ -155,7 +155,7 @@ DB_READ_SCRIPT_CONFIG_DB_KEY = "_DUALTOR_NEIGHBOR_CHECK_SCRIPT_SHA1" ZERO_MAC = "00:00:00:00:00:00" -NEIGHBOR_ATTRIBUTES = ["NEIGHBOR", "MAC", "PORT", "MUX_STATE", "IN_MUX_TOGGLE", "NEIGHBOR_IN_ASIC", "TUNNERL_IN_ASIC", "HWSTATUS"] +NEIGHBOR_ATTRIBUTES = ["NEIGHBOR", "MAC", "PORT", "MUX_STATE", "IN_MUX_TOGGLE", "NEIGHBOR_IN_ASIC", "TUNNEL_IN_ASIC", "HWSTATUS"] NOT_AVAILABLE = "N/A" @@ -400,9 +400,12 @@ def check_neighbor_consistency(neighbors, mux_states, hw_mux_states, mac_to_port continue check_result["NEIGHBOR_IN_ASIC"] = neighbor_ip in asic_neighs - check_result["TUNNERL_IN_ASIC"] = neighbor_ip in asic_route_destinations + check_result["TUNNEL_IN_ASIC"] = neighbor_ip in asic_route_destinations if is_zero_mac: - check_result["HWSTATUS"] = ((not check_result["NEIGHBOR_IN_ASIC"]) and check_result["TUNNERL_IN_ASIC"]) + # NOTE: for zero-mac neighbors, two situations: + # 1. new neighbor just learnt, no neighbor entry in ASIC, tunnel route present in ASIC. + # 2. neighbor expired, neighbor entry still present in ASIC, no tunnel route in ASIC. + check_result["HWSTATUS"] = check_result["NEIGHBOR_IN_ASIC"] or check_result["TUNNEL_IN_ASIC"] else: port_name = mac_to_port_name_map[mac] # NOTE: mux server ips are always fixed to the mux port @@ -415,9 +418,9 @@ def check_neighbor_consistency(neighbors, mux_states, hw_mux_states, mac_to_port check_result["IN_MUX_TOGGLE"] = mux_state != hw_mux_state if mux_state == "active": - check_result["HWSTATUS"] = (check_result["NEIGHBOR_IN_ASIC"] and (not check_result["TUNNERL_IN_ASIC"])) + check_result["HWSTATUS"] = (check_result["NEIGHBOR_IN_ASIC"] and (not check_result["TUNNEL_IN_ASIC"])) elif mux_state == "standby": - check_result["HWSTATUS"] = ((not check_result["NEIGHBOR_IN_ASIC"]) and check_result["TUNNERL_IN_ASIC"]) + check_result["HWSTATUS"] = ((not check_result["NEIGHBOR_IN_ASIC"]) and check_result["TUNNEL_IN_ASIC"]) else: # skip as unknown mux state continue @@ -442,7 +445,7 @@ def parse_check_results(check_results): if not is_zero_mac: check_result["IN_MUX_TOGGLE"] = bool_to_yes_no[in_toggle] check_result["NEIGHBOR_IN_ASIC"] = bool_to_yes_no[check_result["NEIGHBOR_IN_ASIC"]] - check_result["TUNNERL_IN_ASIC"] = bool_to_yes_no[check_result["TUNNERL_IN_ASIC"]] + check_result["TUNNEL_IN_ASIC"] = bool_to_yes_no[check_result["TUNNEL_IN_ASIC"]] check_result["HWSTATUS"] = bool_to_consistency[hwstatus] if (not hwstatus): if is_zero_mac: diff --git a/tests/dualtor_neighbor_check_test.py b/tests/dualtor_neighbor_check_test.py index 1a0a7f5f5a..5916a183a0 100644 --- a/tests/dualtor_neighbor_check_test.py +++ b/tests/dualtor_neighbor_check_test.py @@ -611,3 +611,69 @@ def test_check_neighbor_consistency_zero_mac_neighbor(self, mock_log_functions): assert res is True mock_log_warn.assert_has_calls(expected_log_warn_calls) mock_log_error.assert_not_called() + + def test_check_neighbor_consistency_zero_mac_expired_neighbor(self, mock_log_functions): + mock_log_error, mock_log_warn, _, _ = mock_log_functions + neighbors = {"192.168.0.102": "00:00:00:00:00:00"} + mux_states = {"Ethernet4": "active"} + hw_mux_states = {"Ethernet4": "active"} + mac_to_port_name_map = {"ee:86:d8:46:7d:01": "Ethernet4"} + asic_route_table = [] + asic_neigh_table = ["{\"ip\":\"192.168.0.102\",\"rif\":\"oid:0x6000000000671\",\"switch_id\":\"oid:0x21000000000000\"}"] + mux_server_to_port_map = {"192.168.0.2": "Ethernet4"} + expected_output = ["192.168.0.102", "00:00:00:00:00:00", "N/A", "N/A", "N/A", "yes", "no", "consistent"] + expected_log_output = tabulate.tabulate( + [expected_output], + headers=dualtor_neighbor_check.NEIGHBOR_ATTRIBUTES, + tablefmt="simple" + ).split("\n") + expected_log_warn_calls = [call(line) for line in expected_log_output] + + check_results = dualtor_neighbor_check.check_neighbor_consistency( + neighbors, + mux_states, + hw_mux_states, + mac_to_port_name_map, + asic_route_table, + asic_neigh_table, + mux_server_to_port_map + ) + res = dualtor_neighbor_check.parse_check_results(check_results) + + assert res is True + mock_log_warn.assert_has_calls(expected_log_warn_calls) + mock_log_error.assert_not_called() + + def test_check_neighbor_consistency_inconsistent_zero_mac_neighbor(self, mock_log_functions): + mock_log_error, mock_log_warn, _, _ = mock_log_functions + neighbors = {"192.168.0.102": "00:00:00:00:00:00"} + mux_states = {"Ethernet4": "active"} + hw_mux_states = {"Ethernet4": "active"} + mac_to_port_name_map = {"ee:86:d8:46:7d:01": "Ethernet4"} + asic_route_table = [] + asic_neigh_table = [] + mux_server_to_port_map = {"192.168.0.2": "Ethernet4"} + expected_output = ["192.168.0.102", "00:00:00:00:00:00", "N/A", "N/A", "N/A", "no", "no", "inconsistent"] + expected_log_output = tabulate.tabulate( + [expected_output], + headers=dualtor_neighbor_check.NEIGHBOR_ATTRIBUTES, + tablefmt="simple" + ).split("\n") + expected_log_warn_calls = [call(line) for line in expected_log_output] + expected_log_error_calls = [call("Found neighbors that are inconsistent with mux states: %s", ["192.168.0.102"])] + expected_log_error_calls.extend([call(line) for line in expected_log_output]) + + check_results = dualtor_neighbor_check.check_neighbor_consistency( + neighbors, + mux_states, + hw_mux_states, + mac_to_port_name_map, + asic_route_table, + asic_neigh_table, + mux_server_to_port_map + ) + res = dualtor_neighbor_check.parse_check_results(check_results) + + assert res is False + mock_log_warn.assert_has_calls(expected_log_warn_calls) + mock_log_error.assert_has_calls(expected_log_error_calls)