From 546f545e27652d87cd9136bf51475bc66e80bc7d Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Fri, 22 Oct 2021 19:28:03 +0200 Subject: [PATCH] vpp: cnat performance Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 30 ++++++------ calico-vpp-agent/cni/cni_server.go | 7 ++- vpplink/cnat.go | 17 +++++-- vpplink/generated/generate.log | 9 +++- ...002-cnat-WIP-no-k8s-maglev-from-pods.patch | 49 ++++++------------- vpplink/generated/vpp_clone_current.sh | 11 +++++ 6 files changed, 68 insertions(+), 55 deletions(-) diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index 150aa370..ce322d7c 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -55,13 +55,15 @@ var ( log *logrus.Logger ) -func Go(f func(t *tomb.Tomb) error) { +func Go(f func(t *tomb.Tomb) error, name string) { if t.Alive() { + log.Infof("STARTING %s", name) t.Go(func() error { err := f(&t) if err != nil { log.Warnf("Tomb function errored with %s", err) } + log.Infof("STOPPED %s", name) return err }) } @@ -160,7 +162,7 @@ func main() { serviceServer.SetBGPConf(bgpConf) watchDog := watchdog.NewWatchDog(log.WithFields(logrus.Fields{"component": "watchDog"}), &t) - Go(policyServer.ServePolicy) + Go(policyServer.ServePolicy, "policyServer.ServePolicy") felixConfig := watchDog.Wait(policyServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod") ourBGPSpec := watchDog.Wait(policyServer.GotOurNodeBGPchan, "Waiting for bgp spec to be provided on node add") if ourBGPSpec != nil { @@ -173,7 +175,7 @@ func main() { } if *config.GetCalicoVppFeatureGates().MultinetEnabled { - Go(netWatcher.WatchNetworks) + Go(netWatcher.WatchNetworks, "netWatcher.WatchNetworks") watchDog.Wait(netWatcher.InSync, "Waiting for networks to be listed and synced") } @@ -182,20 +184,20 @@ func main() { connectivityServer.SetFelixConfig(felixConfig.(*felixconfig.Config)) } - Go(routeWatcher.WatchRoutes) - Go(linkWatcher.WatchLinks) - Go(bgpConfigurationWatcher.WatchBGPConfiguration) - Go(prefixWatcher.WatchPrefix) - Go(peerWatcher.WatchBGPPeers) - Go(connectivityServer.ServeConnectivity) - Go(routingServer.ServeRouting) - Go(serviceServer.ServeService) - Go(cniServer.ServeCNI) - Go(prometheusServer.ServePrometheus) + Go(routeWatcher.WatchRoutes, "routeWatcher.WatchRoutes") + Go(linkWatcher.WatchLinks, "linkWatcher.WatchLinks") + Go(bgpConfigurationWatcher.WatchBGPConfiguration, "bgpConfigurationWatcher.WatchBGPConfiguration") + Go(prefixWatcher.WatchPrefix, "prefixWatcher.WatchPrefix") + Go(peerWatcher.WatchBGPPeers, "peerWatcher.WatchBGPPeers") + Go(connectivityServer.ServeConnectivity, "connectivityServer.ServeConnectivity") + Go(routingServer.ServeRouting, "routingServer.ServeRouting") + Go(serviceServer.ServeService, "serviceServer.ServeService") + Go(cniServer.ServeCNI, "cniServer.ServeCNI") + Go(prometheusServer.ServePrometheus, "prometheusServer.ServePrometheus") // watch LocalSID if SRv6 is enabled if *config.GetCalicoVppFeatureGates().SRv6Enabled { - Go(localSIDWatcher.WatchLocalSID) + Go(localSIDWatcher.WatchLocalSID, "localSIDWatcher.WatchLocalSID") } log.Infof("Agent started") diff --git a/calico-vpp-agent/cni/cni_server.go b/calico-vpp-agent/cni/cni_server.go index 435194bd..f17dbbea 100644 --- a/calico-vpp-agent/cni/cni_server.go +++ b/calico-vpp-agent/cni/cni_server.go @@ -373,7 +373,8 @@ func (s *Server) cniServerEventLoop(t *tomb.Tomb) error { for { select { case <-t.Dying(): - break + s.log.Warnf("CNI server asked to stop") + return nil case evt := <-s.cniEventChan: switch evt.Type { case common.FelixConfChanged: @@ -482,12 +483,14 @@ func (s *Server) ServeCNI(t *tomb.Tomb) error { } s.log.Infof("CNI Server returned") + s.grpcServer.Stop() + s.log.Infof("GRPC stopped") - s.grpcServer.GracefulStop() err = syscall.Unlink(config.CNIServerSocket) if err != nil { return err } + s.log.Infof("Socket CNI unlink") return nil } diff --git a/vpplink/cnat.go b/vpplink/cnat.go index 772f7252..c0796eab 100644 --- a/vpplink/cnat.go +++ b/vpplink/cnat.go @@ -18,6 +18,7 @@ package vpplink import ( "fmt" "net" + "github.com/pkg/errors" "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/cnat" "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/interface_types" @@ -25,9 +26,11 @@ import ( ) const ( - FeatureArcCnatInput = "ip?-unicast cnat-input-ip?" - FeatureArcCnatOutput = "ip?-output cnat-output-ip?" - FeatureArcSnat = "ip?-unicast cnat-snat-ip?" + FeatureArcCnatLookup = "ip?-unicast cnat-lookup-ip?" + FeatureArcCnatInput = "ip?-unicast cnat-input-ip?" + FeatureArcCnatOutput = "ip?-output cnat-output-ip?" + FeatureArcCnatWriteBack = "ip?-output cnat-writeback-ip?" + FeatureArcSnat = "ip?-unicast cnat-snat-ip?" ) const InvalidID = ^uint32(0) @@ -124,6 +127,14 @@ func (v *VppLink) CnatDelSnatPrefix(prefix *net.IPNet) error { } func (v *VppLink) CnatEnableFeatures(swIfIndex uint32) (err error) { + err = v.EnableFeatureArc46(swIfIndex, FeatureArcCnatLookup) + if err != nil { + return errors.Wrap(err, "Error enabling arc dnat in") + } + err = v.EnableFeatureArc46(swIfIndex, FeatureArcCnatWriteBack) + if err != nil { + return errors.Wrap(err, "Error enabling arc dnat out") + } err = v.EnableFeatureArc46(swIfIndex, FeatureArcCnatInput) if err != nil { return fmt.Errorf("enabling arc dnat input failed: %w", err) diff --git a/vpplink/generated/generate.log b/vpplink/generated/generate.log index 9aab921b..3b82e407 100755 --- a/vpplink/generated/generate.log +++ b/vpplink/generated/generate.log @@ -1,10 +1,15 @@ -VPP Version : 23.06-rc0~206-gbc006bb5a +VPP Version : 23.06-rc0~211-gf559a7816 Binapi-generator version : govpp v0.8.0-dev -VPP Base commit : 28d74a396 gerrit:34726/3 interface: add buffer stats api +VPP Base commit : 004ebac6a gerrit:34726/3 interface: add buffer stats api ------------------ Cherry picked commits -------------------- capo: Calico Policies plugin acl: acl-plugin custom policies cnat: [WIP] no k8s maglev from pods pbl: Port based balancer +gerrit:34552/8 cnat: add single lookup +gerrit:29748/5 cnat: remove rwlock on ts +gerrit:32821/6 cnat: add ip/client bihash +gerrit:34108/5 cnat: flag to disable rsession +gerrit:31449/10 cnat: dont compute offloaded cksums gerrit:34726/3 interface: add buffer stats api ------------------------------------------------------------- diff --git a/vpplink/generated/patches/0002-cnat-WIP-no-k8s-maglev-from-pods.patch b/vpplink/generated/patches/0002-cnat-WIP-no-k8s-maglev-from-pods.patch index 510b5f52..16249965 100644 --- a/vpplink/generated/patches/0002-cnat-WIP-no-k8s-maglev-from-pods.patch +++ b/vpplink/generated/patches/0002-cnat-WIP-no-k8s-maglev-from-pods.patch @@ -1,49 +1,30 @@ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak -Date: Mon, 8 Mar 2021 19:00:04 +0100 +Date: Fri, 7 Apr 2023 16:57:30 +0200 Subject: [PATCH 2/4] cnat: [WIP] no k8s maglev from pods -Type: improvement - -Change-Id: If0702dbc51c308f0bb0ed16149c293d7adf9a984 +Change-Id: Id262a97986b6de01a42019287377486787f2e606 Signed-off-by: Nathan Skrzypczak --- - src/plugins/cnat/cnat_node_feature.c | 8 +++++++- - 1 file changed, 7 insertions(+), 1 deletion(-) + src/plugins/cnat/cnat_node_feature.c | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plugins/cnat/cnat_node_feature.c b/src/plugins/cnat/cnat_node_feature.c -index 76aa89398..fd3b90a1a 100644 +index f6d4850f8..f2b65b2ff 100644 --- a/src/plugins/cnat/cnat_node_feature.c +++ b/src/plugins/cnat/cnat_node_feature.c -@@ -43,6 +43,7 @@ cnat_input_feature_fn (vlib_main_t *vm, vlib_node_runtime_t *node, - int session_not_found, cnat_session_t *session) - { - vlib_combined_counter_main_t *cntm = &cnat_translation_counters; -+ cnat_snat_policy_main_t *cpm = &cnat_snat_policy_main; - const cnat_translation_t *ct = NULL; - ip4_header_t *ip4 = NULL; - ip_protocol_t iproto; -@@ -53,6 +54,9 @@ cnat_input_feature_fn (vlib_main_t *vm, vlib_node_runtime_t *node, - index_t cti; - u8 trace_flags = 0; +@@ -112,7 +112,10 @@ cnat_input_feature_new_flow_inline (vlib_main_t *vm, vlib_buffer_t *b, + clib_host_to_net_u16 (trk0->ct_ep[VLIB_TX].ce_port) : + rw->tuple.port[VLIB_TX]; +- if (trk0->ct_flags & CNAT_TRK_FLAG_NO_NAT) + u32 in_if = vnet_buffer (b)->sw_if_index[VLIB_RX]; -+ int ispod; -+ - /* By default follow arc default next */ - vnet_feature_next (&next0, b); - -@@ -127,7 +131,9 @@ cnat_input_feature_fn (vlib_main_t *vm, vlib_node_runtime_t *node, - session->value.cs_port[VLIB_RX] = udp0->src_port; - session->value.flags = 0; - -- if (trk0->ct_flags & CNAT_TRK_FLAG_NO_NAT) -+ ispod = clib_bitmap_get ( -+ cpm->interface_maps[CNAT_SNAT_IF_MAP_INCLUDE_POD], in_if); -+ if (trk0->ct_flags & CNAT_TRK_FLAG_NO_NAT && !ispod) - { - const dpo_id_t *dpo0; - const load_balance_t *lb1; ++ int ispod = clib_bitmap_get ( ++ cnat_snat_policy_main.interface_maps[CNAT_SNAT_IF_MAP_INCLUDE_POD], in_if); ++ if (trk0->ct_flags & CNAT_TRK_FLAG_NO_NAT && !ispod) + { + const dpo_id_t *dpo0; + const load_balance_t *lb1; -- 2.39.2 diff --git a/vpplink/generated/vpp_clone_current.sh b/vpplink/generated/vpp_clone_current.sh index 5745dee8..65af6f11 100755 --- a/vpplink/generated/vpp_clone_current.sh +++ b/vpplink/generated/vpp_clone_current.sh @@ -96,9 +96,20 @@ git_clone_cd_and_reset "$1" 156d452750ab37a51984938bf4a8ab0e16650d82 git_cherry_pick refs/changes/26/34726/3 # 34726: interface: add buffer stats api | https://gerrit.fd.io/r/c/vpp/+/34726 +git_cherry_pick refs/changes/49/31449/10 # 31449: cnat: dont compute offloaded cksums | https://gerrit.fd.io/r/c/vpp/+/31449 +git_cherry_pick refs/changes/08/34108/5 # 34108: cnat: flag to disable rsession | https://gerrit.fd.io/r/c/vpp/+/34108 +git_cherry_pick refs/changes/21/32821/6 # 32821: cnat: add ip/client bihash | https://gerrit.fd.io/r/c/vpp/+/32821 +git_cherry_pick refs/changes/48/29748/5 # 29748: cnat: remove rwlock on ts | https://gerrit.fd.io/r/c/vpp/+/29748 +git_cherry_pick refs/changes/52/34552/8 # 34552: cnat: add single lookup | https://gerrit.fd.io/r/c/vpp/+/34552 + # --------------- private plugins --------------- # Generated with 'git format-patch --zero-commit -o ./patches/ HEAD^^^' git_apply_private 0001-pbl-Port-based-balancer.patch git_apply_private 0002-cnat-WIP-no-k8s-maglev-from-pods.patch git_apply_private 0003-acl-acl-plugin-custom-policies.patch git_apply_private 0004-capo-Calico-Policies-plugin.patch + + +# git cp 236423e261385e58556b75c9f9eb56006a2b9d67 # perf WIP +# git cp 7603fd367739e74c47349a1dd9662c78dcf9377d # capo+cnat +