diff --git a/.wordlist.txt b/.wordlist.txt
index 4c4418298..8e91327b3 100644
--- a/.wordlist.txt
+++ b/.wordlist.txt
@@ -730,3 +730,6 @@ Gan
Ze
AppDBMemoryEstimation
Ivantsiv
+Zhixiong
+Niu
+validonly
diff --git a/dash-pipeline/Makefile b/dash-pipeline/Makefile
index 89622a1dd..fe7c42680 100644
--- a/dash-pipeline/Makefile
+++ b/dash-pipeline/Makefile
@@ -177,8 +177,9 @@ sai: sai-headers sai-meta libsai
sai-headers: p4 docker-saithrift-bldr-image-exists | SAI/SAI
@echo "Generate SAI library headers and implementation..."
- # Once the specs are checked in, we can use this to revert any local changes before generating the new specs.
- # git checkout SAI/specs/*
+ # Revert any local changes before generating the new specs.
+ git clean -xf SAI/specs
+ git checkout SAI/specs/*
mkdir -p SAI/lib
diff --git a/dash-pipeline/SAI/sai_api_gen.py b/dash-pipeline/SAI/sai_api_gen.py
index af8098b48..ceb574d42 100755
--- a/dash-pipeline/SAI/sai_api_gen.py
+++ b/dash-pipeline/SAI/sai_api_gen.py
@@ -68,7 +68,8 @@
print("Outputting new SAI spec to " + sai_spec_dir)
yaml_inc_ctor.autoload = False
new_sai_spec = dash_sai_exts.to_sai()
- new_sai_spec.serialize(sai_spec_dir)
+ sai_spec.merge(new_sai_spec)
+ sai_spec.serialize(sai_spec_dir)
# Generate and update all SAI files
SAIGenerator(dash_sai_exts).generate()
diff --git a/dash-pipeline/SAI/specs/dash_outbound_ca_to_pa.yaml b/dash-pipeline/SAI/specs/dash_outbound_ca_to_pa.yaml
index 34a98a03f..4d46396fe 100644
--- a/dash-pipeline/SAI/specs/dash_outbound_ca_to_pa.yaml
+++ b/dash-pipeline/SAI/specs/dash_outbound_ca_to_pa.yaml
@@ -145,6 +145,18 @@ sai_apis:
allow_null: false
valid_only: null
deprecated: null
+ - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+ name: SAI_OUTBOUND_CA_TO_PA_ENTRY_OVERLAY_SIP_MASK
+ description: Action parameter OVERLAY_SIP_MASK
+ type: sai_ip_address_t
+ attr_value_field: ipaddr
+ default: 0.0.0.0
+ isresourcetype: false
+ flags: CREATE_AND_SET
+ object_name: null
+ allow_null: false
+ valid_only: null
+ deprecated: null
- !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
name: SAI_OUTBOUND_CA_TO_PA_ENTRY_OVERLAY_DIP
description: Action parameter OVERLAY_DIP
@@ -157,6 +169,18 @@ sai_apis:
allow_null: false
valid_only: null
deprecated: null
+ - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+ name: SAI_OUTBOUND_CA_TO_PA_ENTRY_OVERLAY_DIP_MASK
+ description: Action parameter OVERLAY_DIP_MASK
+ type: sai_ip_address_t
+ attr_value_field: ipaddr
+ default: 0.0.0.0
+ isresourcetype: false
+ flags: CREATE_AND_SET
+ object_name: null
+ allow_null: false
+ valid_only: null
+ deprecated: null
- !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
name: SAI_OUTBOUND_CA_TO_PA_ENTRY_DASH_ENCAPSULATION
description: Action parameter DASH_ENCAPSULATION
@@ -205,4 +229,16 @@ sai_apis:
allow_null: false
valid_only: null
deprecated: null
+ - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+ name: SAI_OUTBOUND_CA_TO_PA_ENTRY_DASH_TUNNEL_ID
+ description: Action parameter DASH_TUNNEL_ID
+ type: sai_object_id_t
+ attr_value_field: u16
+ default: SAI_NULL_OBJECT_ID
+ isresourcetype: false
+ flags: CREATE_AND_SET
+ object_name: SAI_OBJECT_TYPE_DASH_TUNNEL
+ allow_null: true
+ valid_only: null
+ deprecated: null
stats: []
diff --git a/dash-pipeline/SAI/specs/dash_outbound_routing.yaml b/dash-pipeline/SAI/specs/dash_outbound_routing.yaml
index a1ff21683..df80ae432 100644
--- a/dash-pipeline/SAI/specs/dash_outbound_routing.yaml
+++ b/dash-pipeline/SAI/specs/dash_outbound_routing.yaml
@@ -273,4 +273,16 @@ sai_apis:
allow_null: false
valid_only: null
deprecated: null
+ - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+ name: SAI_OUTBOUND_ROUTING_ENTRY_DASH_TUNNEL_ID
+ description: Action parameter DASH_TUNNEL_ID
+ type: sai_object_id_t
+ attr_value_field: u16
+ default: SAI_NULL_OBJECT_ID
+ isresourcetype: false
+ flags: CREATE_AND_SET
+ object_name: SAI_OBJECT_TYPE_DASH_TUNNEL
+ allow_null: true
+ valid_only: null
+ deprecated: null
stats: []
diff --git a/dash-pipeline/SAI/specs/sai_spec.yaml b/dash-pipeline/SAI/specs/sai_spec.yaml
index 6138104be..d7503a876 100644
--- a/dash-pipeline/SAI/specs/sai_spec.yaml
+++ b/dash-pipeline/SAI/specs/sai_spec.yaml
@@ -13,6 +13,7 @@ api_types:
- SAI_API_DASH_TUNNEL
- SAI_API_ROUTE
- SAI_API_DASH_VIP
+- SAI_API_DASH_TUNNEL
object_types:
- SAI_OBJECT_TYPE_DASH_ACL_GROUP
- SAI_OBJECT_TYPE_DASH_ACL_RULE
@@ -33,6 +34,7 @@ object_types:
- SAI_OBJECT_TYPE_DASH_TUNNEL
- SAI_OBJECT_TYPE_ROUTE_ENTRY
- SAI_OBJECT_TYPE_VIP_ENTRY
+- SAI_OBJECT_TYPE_DASH_TUNNEL
object_entries:
- !!python/object:utils.sai_spec.sai_struct_entry.SaiStructEntry
name: direction_lookup_entry
@@ -99,14 +101,22 @@ enums:
name: STANDBY
description: ''
value: '2'
+- !!python/object:utils.sai_spec.sai_enum.SaiEnum
+ name: sai_dash_encapsulation_t
+ description: ''
+ members:
- !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
- name: STANDALONE
+ name: INVALID
description: ''
value: '3'
- !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
- name: SWITCHING_TO_ACTIVE
+ name: VXLAN
description: ''
- value: '4'
+ value: '1'
+ - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
+ name: NVGRE
+ description: ''
+ value: '2'
- !!python/object:utils.sai_spec.sai_enum.SaiEnum
name: sai_dash_routing_actions_t
description: ''
@@ -132,49 +142,41 @@ enums:
description: ''
value: '16'
- !!python/object:utils.sai_spec.sai_enum.SaiEnum
- name: sai_dash_direction_t
+ name: sai_dash_tunnel_dscp_mode_t
description: ''
members:
- !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
- name: INVALID
+ name: PRESERVE_MODEL
description: ''
value: '0'
- !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
- name: OUTBOUND
+ name: PIPE_MODEL
description: ''
value: '1'
- - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
- name: INBOUND
- description: ''
- value: '2'
- !!python/object:utils.sai_spec.sai_enum.SaiEnum
- name: sai_dash_encapsulation_t
+ name: sai_dash_ha_role_t
description: ''
members:
- !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
- name: INVALID
+ name: DEAD
description: ''
value: '0'
- !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
- name: VXLAN
+ name: ACTIVE
description: ''
value: '1'
- !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
- name: NVGRE
+ name: STANDBY
description: ''
value: '2'
-- !!python/object:utils.sai_spec.sai_enum.SaiEnum
- name: sai_dash_tunnel_dscp_mode_t
- description: ''
- members:
- !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
- name: PRESERVE_MODEL
+ name: STANDALONE
description: ''
- value: '0'
+ value: '3'
- !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
- name: PIPE_MODEL
+ name: SWITCHING_TO_ACTIVE
description: ''
- value: '1'
+ value: '4'
port_extenstion: !!python/object:utils.sai_spec.sai_api_extension.SaiApiExtension
attributes: []
stats:
@@ -264,3 +266,4 @@ api_groups:
- !inc '/SAI/specs/dash_tunnel.yaml'
- !inc '/SAI/specs/route.yaml'
- !inc '/SAI/specs/dash_vip.yaml'
+- !inc '/SAI/specs/dash_tunnel.yaml'
diff --git a/dash-pipeline/SAI/src/objectidmanager.cpp b/dash-pipeline/SAI/src/objectidmanager.cpp
index d9d2b2b00..0fc939dbd 100644
--- a/dash-pipeline/SAI/src/objectidmanager.cpp
+++ b/dash-pipeline/SAI/src/objectidmanager.cpp
@@ -1,12 +1,20 @@
#include "objectidmanager.h"
#include "logger.h"
+extern "C" {
+#include "saimetadata.h"
+}
+
#define SAI_OBJECT_ID_BITS_SIZE (8 * sizeof(sai_object_id_t))
static_assert(SAI_OBJECT_ID_BITS_SIZE == 64, "sai_object_id_t must have 64 bits");
static_assert(sizeof(sai_object_id_t) == sizeof(uint64_t), "SAI object ID size should be uint64_t");
-#define DASH_OID_RESERVED_BITS_SIZE ( 8 )
+#define DASH_OID_RESERVED_BITS_SIZE ( 7 )
+
+#define DASH_OBJECT_TYPE_EXTENSIONS_FLAG_BITS_SIZE ( 1 )
+#define DASH_OBJECT_TYPE_EXTENSIONS_FLAG_MAX ( (1ULL << DASH_OBJECT_TYPE_EXTENSIONS_FLAG_BITS_SIZE) - 1 )
+#define DASH_OBJECT_TYPE_EXTENSIONS_FLAG_MASK (DASH_OBJECT_TYPE_EXTENSIONS_FLAG_MAX)
#define DASH_OBJECT_TYPE_BITS_SIZE ( 8 )
#define DASH_OBJECT_TYPE_MASK ( (1ULL << DASH_OBJECT_TYPE_BITS_SIZE) - 1 )
@@ -20,7 +28,12 @@ static_assert(sizeof(sai_object_id_t) == sizeof(uint64_t), "SAI object ID size s
#define DASH_OBJECT_INDEX_MASK ( (1ULL << DASH_OBJECT_INDEX_BITS_SIZE) - 1 )
#define DASH_OBJECT_INDEX_MAX (DASH_OBJECT_INDEX_MASK)
-#define DASH_OBJECT_ID_BITS_SIZE (DASH_OID_RESERVED_BITS_SIZE + DASH_OBJECT_TYPE_BITS_SIZE + DASH_SWITCH_INDEX_BITS_SIZE + DASH_OBJECT_INDEX_BITS_SIZE)
+#define DASH_OBJECT_ID_BITS_SIZE ( \
+ DASH_OID_RESERVED_BITS_SIZE + \
+ DASH_OBJECT_TYPE_EXTENSIONS_FLAG_BITS_SIZE + \
+ DASH_OBJECT_TYPE_BITS_SIZE + \
+ DASH_SWITCH_INDEX_BITS_SIZE + \
+ DASH_OBJECT_INDEX_BITS_SIZE)
static_assert(DASH_OBJECT_ID_BITS_SIZE == SAI_OBJECT_ID_BITS_SIZE, "dash object id size must be equal to SAI object id size");
@@ -28,16 +41,15 @@ static_assert(DASH_OBJECT_TYPE_MAX == 0xff, "invalid object type max size");
static_assert(DASH_SWITCH_INDEX_MAX == 0xff, "invalid switch index max size");
static_assert(DASH_OBJECT_INDEX_MAX == 0xffffffffff, "invalid object index max");
-/*
- * This condition must be met, since we need to be able to encode SAI object
- * type in object id on defined number of bits.
- */
-static_assert(SAI_OBJECT_TYPE_EXTENSIONS_RANGE_END < DASH_OBJECT_TYPE_MAX, "dash max object type value must be greater than supported SAI max objeect type value");
+static_assert(SAI_OBJECT_TYPE_MAX < 256, "object type must be possible to encode on 1 byte");
+static_assert((SAI_OBJECT_TYPE_EXTENSIONS_RANGE_END - SAI_OBJECT_TYPE_EXTENSIONS_RANGE_START) < 256,
+ "extensions object type must be possible to encode on 1 byte");
/*
* Current OBJECT ID format:
*
- * bits 63..56 - reserved (must be zero)
+ * bits 63..57 - reserved (must be zero)
+ * bits 56..56 - object type extensions flag
* bits 55..48 - SAI object type
* bits 47..40 - switch index
* bits 39..0 - object index
@@ -56,8 +68,12 @@ static_assert(SAI_OBJECT_TYPE_EXTENSIONS_RANGE_END < DASH_OBJECT_TYPE_MAX, "dash
#define DASH_GET_OBJECT_TYPE(oid) \
( (((uint64_t)oid) >> ( DASH_SWITCH_INDEX_BITS_SIZE + DASH_OBJECT_INDEX_BITS_SIZE ) ) & ( DASH_OBJECT_TYPE_MASK ) )
+ #define DASH_GET_OBJECT_TYPE_EXTENSIONS_FLAG(oid) \
+ ( (((uint64_t)oid) >> ( DASH_OBJECT_TYPE_BITS_SIZE + DASH_SWITCH_INDEX_BITS_SIZE + DASH_OBJECT_INDEX_BITS_SIZE) ) & ( DASH_OBJECT_TYPE_EXTENSIONS_FLAG_MAX ) )
+
#define DASH_TEST_OID (0x0123456789abcdef)
+static_assert(DASH_GET_OBJECT_TYPE_EXTENSIONS_FLAG(DASH_TEST_OID) == 0x1, "object type extension flag");
static_assert(DASH_GET_OBJECT_TYPE(DASH_TEST_OID) == 0x23, "test object type");
static_assert(DASH_GET_SWITCH_INDEX(DASH_TEST_OID) == 0x45, "test switch index");
static_assert(DASH_GET_OBJECT_INDEX(DASH_TEST_OID) == 0x6789abcdef, "test object index");
@@ -103,9 +119,11 @@ sai_object_type_t ObjectIdManager::saiObjectTypeQuery(
return SAI_OBJECT_TYPE_NULL;
}
- sai_object_type_t objectType = (sai_object_type_t)(DASH_GET_OBJECT_TYPE(objectId));
+ sai_object_type_t objectType = DASH_GET_OBJECT_TYPE_EXTENSIONS_FLAG(objectId)
+ ? (sai_object_type_t)(DASH_GET_OBJECT_TYPE(objectId) + SAI_OBJECT_TYPE_EXTENSIONS_RANGE_START)
+ : (sai_object_type_t)(DASH_GET_OBJECT_TYPE(objectId));
- if (objectType == SAI_OBJECT_TYPE_NULL || objectType >= (sai_object_type_t)SAI_OBJECT_TYPE_EXTENSIONS_RANGE_END)
+ if (sai_metadata_is_object_type_valid(objectType) == false)
{
DASH_LOG_ERROR("invalid object id 0x%lx", objectId);
@@ -149,7 +167,7 @@ sai_object_id_t ObjectIdManager::allocateNewObjectId(
{
DASH_LOG_ENTER();
- if ((objectType <= SAI_OBJECT_TYPE_NULL) || (objectType >= (sai_object_type_t)SAI_OBJECT_TYPE_EXTENSIONS_RANGE_END))
+ if (sai_metadata_is_object_type_valid(objectType) == false)
{
DASH_LOG_ERROR("invalid objct type: %d", objectType);
@@ -250,7 +268,14 @@ sai_object_id_t ObjectIdManager::constructObjectId(
{
DASH_LOG_ENTER();
+ uint64_t extensionsFlag = (uint64_t)objectType >= SAI_OBJECT_TYPE_EXTENSIONS_RANGE_START;
+
+ objectType = extensionsFlag
+ ? (sai_object_type_t)(objectType - SAI_OBJECT_TYPE_EXTENSIONS_RANGE_START)
+ : objectType;
+
return (sai_object_id_t)(
+ (((uint64_t)extensionsFlag & DASH_OBJECT_TYPE_EXTENSIONS_FLAG_MASK) << ( DASH_OBJECT_TYPE_BITS_SIZE + DASH_SWITCH_INDEX_BITS_SIZE + DASH_OBJECT_INDEX_BITS_SIZE )) |
(((uint64_t)objectType & DASH_OBJECT_TYPE_MASK)<< ( DASH_SWITCH_INDEX_BITS_SIZE + DASH_OBJECT_INDEX_BITS_SIZE))|
(((uint64_t)switchIndex & DASH_SWITCH_INDEX_MASK)<< ( DASH_OBJECT_INDEX_BITS_SIZE )) |
(objectIndex & DASH_OBJECT_INDEX_MASK));
@@ -295,9 +320,11 @@ sai_object_type_t ObjectIdManager::objectTypeQuery(
return SAI_OBJECT_TYPE_NULL;
}
- sai_object_type_t objectType = (sai_object_type_t)(DASH_GET_OBJECT_TYPE(objectId));
+ sai_object_type_t objectType = DASH_GET_OBJECT_TYPE_EXTENSIONS_FLAG(objectId)
+ ? (sai_object_type_t)(DASH_GET_OBJECT_TYPE(objectId) + SAI_OBJECT_TYPE_EXTENSIONS_RANGE_START)
+ : (sai_object_type_t)(DASH_GET_OBJECT_TYPE(objectId));
- if (objectType == SAI_OBJECT_TYPE_NULL || objectType >= (sai_object_type_t)SAI_OBJECT_TYPE_EXTENSIONS_RANGE_END)
+ if (sai_metadata_is_object_type_valid(objectType) == false)
{
DASH_LOG_ERROR("invalid object id 0x%lx", objectId);
diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_api.py b/dash-pipeline/SAI/utils/sai_spec/sai_api.py
index 07334c4f9..494c11bc5 100644
--- a/dash-pipeline/SAI/utils/sai_spec/sai_api.py
+++ b/dash-pipeline/SAI/utils/sai_spec/sai_api.py
@@ -3,6 +3,7 @@
from .sai_attribute import SaiAttribute
from .sai_enum import SaiEnum
from .sai_struct import SaiStruct
+from . import sai_spec_utils
class SaiApi(SaiCommon):
@@ -17,3 +18,12 @@ def __init__(self, name: str, description: str, is_object: bool = False):
self.structs: List[SaiStruct] = []
self.attributes: List[SaiAttribute] = []
self.stats: List[SaiAttribute] = []
+
+ def merge(self, other: "SaiCommon"):
+ super().merge(other)
+
+ self.is_object = other.is_object
+ sai_spec_utils.merge_sai_common_lists(self.enums, other.enums)
+ sai_spec_utils.merge_sai_common_lists(self.structs, other.structs)
+ sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes)
+ sai_spec_utils.merge_sai_common_lists(self.stats, other.stats)
diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py b/dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py
index 3ebd9e38a..d3324d88c 100644
--- a/dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py
+++ b/dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py
@@ -1,5 +1,6 @@
from typing import List
from .sai_attribute import SaiAttribute
+from . import sai_spec_utils
class SaiApiExtension:
@@ -12,3 +13,7 @@ class SaiApiExtension:
def __init__(self):
self.attributes: List[SaiAttribute] = []
self.stats: List[SaiAttribute] = []
+
+ def merge(self, other: "SaiApiExtension"):
+ sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes)
+ sai_spec_utils.merge_sai_common_lists(self.stats, other.stats)
diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_api_group.py b/dash-pipeline/SAI/utils/sai_spec/sai_api_group.py
index 25f0b7b38..b960a9f99 100644
--- a/dash-pipeline/SAI/utils/sai_spec/sai_api_group.py
+++ b/dash-pipeline/SAI/utils/sai_spec/sai_api_group.py
@@ -1,6 +1,7 @@
from typing import List
from .sai_common import SaiCommon
from .sai_api import SaiApi
+from . import sai_spec_utils
class SaiApiGroup(SaiCommon):
@@ -11,3 +12,20 @@ class SaiApiGroup(SaiCommon):
def __init__(self, name: str, description: str):
super().__init__(name, description)
self.sai_apis: List[SaiApi] = []
+
+ def merge(self, other: "SaiCommon"):
+ super().merge(other)
+ sai_spec_utils.merge_sai_common_lists(self.sai_apis, other.sai_apis)
+
+ def deprecate(self) -> bool:
+ """
+ Deprecate API group.
+
+ When deprecating the API group, we can safely remove it from the list as the
+ net effect is the same as keeping it:
+ - The old API type, object type and object entries will not be changed.
+ - The SAI headers will not be changed, because their API groups are present.
+ - The DASH libsai will not be generated anymore, but it is ok, since we will not
+ use them in the BMv2 anyway.
+ """
+ return True
diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_api_p4_meta.py b/dash-pipeline/SAI/utils/sai_spec/sai_api_p4_meta.py
new file mode 100644
index 000000000..e1279faeb
--- /dev/null
+++ b/dash-pipeline/SAI/utils/sai_spec/sai_api_p4_meta.py
@@ -0,0 +1,19 @@
+from typing import Dict, List
+
+
+class SaiApiP4MetaAction:
+ def __init__(self, name: str, id: int):
+ self.name: str = name
+ self.id: int = id
+ self.attr_param_id: Dict[str, int] = {}
+
+
+class SaiApiP4MetaTable:
+ def __init__(self, id: int):
+ self.id: int = id
+ self.actions: Dict[str, SaiApiP4MetaAction] = {}
+
+
+class SaiApiP4Meta:
+ def __init__(self):
+ self.tables: List[SaiApiP4MetaTable] = []
diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_attribute.py b/dash-pipeline/SAI/utils/sai_spec/sai_attribute.py
index bdb680f0a..88b0033ff 100644
--- a/dash-pipeline/SAI/utils/sai_spec/sai_attribute.py
+++ b/dash-pipeline/SAI/utils/sai_spec/sai_attribute.py
@@ -31,3 +31,11 @@ def __init__(
self.allow_null = allow_null
self.valid_only = valid_only
self.deprecated = deprecated
+
+ def merge(self, other: "SaiCommon"):
+ super().merge(other)
+ self.__dict__.update(other.__dict__)
+
+ def deprecate(self) -> bool:
+ self.deprecated = True
+ return False
\ No newline at end of file
diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_common.py b/dash-pipeline/SAI/utils/sai_spec/sai_common.py
index 7f1a21332..c6ee8e8cd 100644
--- a/dash-pipeline/SAI/utils/sai_spec/sai_common.py
+++ b/dash-pipeline/SAI/utils/sai_spec/sai_common.py
@@ -6,3 +6,21 @@ class SaiCommon:
def __init__(self, name: str, description: str):
self.name: str = name
self.description: str = description
+
+ def merge(self, other: "SaiCommon"):
+ """
+ Merge the other SaiCommon object into this object.
+ """
+ if not isinstance(other, type(self)):
+ raise TypeError(f"Cannot merge {type(self)} with {type(other)}")
+
+ self.description = other.description
+
+ def deprecate(self) -> bool:
+ """
+ Deprecate this object.
+
+ If the value doesn't support deprecation marking, we don't do anything
+ but return False to keep it in the list.
+ """
+ return False
diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_enum.py b/dash-pipeline/SAI/utils/sai_spec/sai_enum.py
index bae1cff4c..621b80427 100644
--- a/dash-pipeline/SAI/utils/sai_spec/sai_enum.py
+++ b/dash-pipeline/SAI/utils/sai_spec/sai_enum.py
@@ -1,6 +1,7 @@
from typing import List
from .sai_common import SaiCommon
from .sai_enum_member import SaiEnumMember
+from . import sai_spec_utils
class SaiEnum(SaiCommon):
@@ -11,3 +12,7 @@ class SaiEnum(SaiCommon):
def __init__(self, name: str, description: str, members: List[SaiEnumMember] = []):
super().__init__(name, description)
self.members: List[SaiEnumMember] = members
+
+ def merge(self, other: "SaiCommon"):
+ super().merge(other)
+ sai_spec_utils.merge_sai_common_lists(self.members, other.members)
diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py b/dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py
index f8cfb6bd2..fe4a458c8 100644
--- a/dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py
+++ b/dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py
@@ -10,3 +10,7 @@ class SaiEnumMember(SaiCommon):
def __init__(self, name: str, description: str, value: str):
super().__init__(name, description)
self.value: str = value
+
+ def merge(self, other: "SaiCommon"):
+ super().merge(other)
+ self.value = other.value
\ No newline at end of file
diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_spec.py b/dash-pipeline/SAI/utils/sai_spec/sai_spec.py
index 74cd6ee33..e82faee03 100644
--- a/dash-pipeline/SAI/utils/sai_spec/sai_spec.py
+++ b/dash-pipeline/SAI/utils/sai_spec/sai_spec.py
@@ -6,6 +6,7 @@
from .sai_api_group import SaiApiGroup
from .sai_api_extension import SaiApiExtension
from .sai_struct_entry import SaiStructEntry
+from . import sai_spec_utils
class SaiSpec:
@@ -24,12 +25,16 @@ def __init__(self):
def serialize(self, spec_dir: str):
yaml_inc_files = []
for api_group in self.api_groups:
- sai_api_group_spec_file_path = os.path.join(spec_dir, api_group.name + ".yaml")
+ sai_api_group_spec_file_path = os.path.join(
+ spec_dir, api_group.name + ".yaml"
+ )
with open(sai_api_group_spec_file_path, "w") as f:
f.write(yaml.dump(api_group, indent=2, sort_keys=False))
-
- yaml_inc_files.append(yaml_include.Data(urlpath=sai_api_group_spec_file_path))
+
+ yaml_inc_files.append(
+ yaml_include.Data(urlpath=sai_api_group_spec_file_path)
+ )
api_groups = self.api_groups
self.api_groups = yaml_inc_files
@@ -43,3 +48,21 @@ def serialize(self, spec_dir: str):
def deserialize(spec_dir: str):
with open(os.path.join(spec_dir, "sai_spec.yaml")) as f:
return yaml.unsafe_load(f)
+
+ def merge(self, other: "SaiSpec"):
+ sai_spec_utils.merge_sai_value_lists(
+ self.api_types, other.api_types, lambda x: x
+ )
+ sai_spec_utils.merge_sai_value_lists(
+ self.object_types, other.object_types, lambda x: x
+ )
+ sai_spec_utils.merge_sai_common_lists(self.object_entries, other.object_entries)
+
+ # The global enums are generated from the P4 enum types, so we can respect whatever in the
+ # new spec and simply replace them, because:
+ # - It doesn't matter if the order of enum itself changes.
+ # - We cannot move the enum members as we want, as their order changes their values.
+ self.enums = other.enums
+
+ self.port_extenstion.merge(other.port_extenstion)
+ sai_spec_utils.merge_sai_common_lists(self.api_groups, other.api_groups)
diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_spec_utils.py b/dash-pipeline/SAI/utils/sai_spec/sai_spec_utils.py
new file mode 100644
index 000000000..747d19933
--- /dev/null
+++ b/dash-pipeline/SAI/utils/sai_spec/sai_spec_utils.py
@@ -0,0 +1,50 @@
+from typing import Any, List, Callable
+from .sai_common import SaiCommon
+
+
+def merge_sai_value_lists(
+ target: List[Any],
+ source: List[Any],
+ get_key: Callable[[Any], str],
+ on_conflict: Callable[[Any, Any], None] = lambda x, y: x,
+ on_deprecate: Callable[[Any], bool] = lambda x: False
+) -> None:
+ """
+ Merge 2 SAI value lists from source list into target.
+
+ Since we could not remove the old value or change the order of old values, the merge
+ is done as below:
+ - Any new values will be added in the end of the list.
+ - Any values that collapse with existing values will invoke on_conflict callback to resolve.
+ - Any values that needs to be removed will invoke on_deprecate function to deprecate. By default,
+ it will not be removed from the old list.
+ """
+ target_dict = {get_key(item): item for item in target}
+
+ source_keys = set()
+ for source_item in source:
+ source_key = get_key(source_item)
+ source_keys.add(source_key)
+
+ if source_key in target_dict:
+ target_item = target_dict[source_key]
+ on_conflict(target_item, source_item)
+ else:
+ target.append(source_item)
+ target_dict[source_key] = source_item
+
+ # Remove all items in target, if its key doesn't exist in source_keys and on_deprecate returns True.
+ target[:] = [item for item in target if get_key(item) in source_keys or not on_deprecate(item)]
+
+
+def merge_sai_common_lists(
+ target: List[SaiCommon],
+ source: List[SaiCommon],
+) -> None:
+ merge_sai_value_lists(
+ target,
+ source,
+ get_key=lambda x: x.name,
+ on_conflict=lambda x, y: x.merge(y),
+ on_deprecate=lambda x: x.deprecate(),
+ )
diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_struct.py b/dash-pipeline/SAI/utils/sai_spec/sai_struct.py
index b98cf3a27..025d1ecb4 100644
--- a/dash-pipeline/SAI/utils/sai_spec/sai_struct.py
+++ b/dash-pipeline/SAI/utils/sai_spec/sai_struct.py
@@ -1,7 +1,7 @@
from typing import List
from .sai_common import SaiCommon
from .sai_struct_entry import SaiStructEntry
-
+from . import sai_spec_utils
class SaiStruct(SaiCommon):
"""
@@ -11,3 +11,7 @@ class SaiStruct(SaiCommon):
def __init__(self, name: str, description: str, members: List[SaiStructEntry] = []):
super().__init__(name, description)
self.members: List[SaiStructEntry] = members
+
+ def merge(self, other: "SaiCommon"):
+ super().merge(other)
+ sai_spec_utils.merge_sai_common_lists(self.members, other.members)
\ No newline at end of file
diff --git a/dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py b/dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py
index 1f4e6ea06..ba3e42f99 100644
--- a/dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py
+++ b/dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py
@@ -19,3 +19,7 @@ def __init__(
self.type = type
self.objects = objects
self.valid_only = valid_only
+
+ def merge(self, other: "SaiCommon"):
+ super().merge(other)
+ self.__dict__.update(other.__dict__)
\ No newline at end of file
diff --git a/dash-pipeline/bmv2/dash_routing_types.p4 b/dash-pipeline/bmv2/dash_routing_types.p4
index 708981c2b..ee2d4fa3f 100644
--- a/dash-pipeline/bmv2/dash_routing_types.p4
+++ b/dash-pipeline/bmv2/dash_routing_types.p4
@@ -168,7 +168,9 @@ action set_private_link_mapping(
inout metadata_t meta,
@SaiVal[type="sai_ip_address_t"] IPv4Address underlay_dip,
IPv6Address overlay_sip,
+ IPv6Address overlay_sip_mask,
IPv6Address overlay_dip,
+ IPv6Address overlay_dip_mask,
@SaiVal[type="sai_dash_encapsulation_t"] dash_encapsulation_t dash_encapsulation,
bit<24> tunnel_key,
bit<32> meter_class_or,
@@ -197,8 +199,8 @@ action set_private_link_mapping(
push_action_nat46(hdr = hdr,
meta = meta,
dip = overlay_dip,
- dip_mask = 0xffffffffffffffffffffffff,
- sip = (overlay_sip & ~meta.eni_data.pl_sip_mask) | meta.eni_data.pl_sip | (IPv6Address)hdr.u0_ipv4.src_addr,
+ dip_mask = overlay_dip_mask,
+ sip = ((( (IPv6Address)hdr.u0_ipv4.src_addr & ~overlay_sip_mask) | overlay_sip) & ~meta.eni_data.pl_sip_mask) | meta.eni_data.pl_sip,
sip_mask = 0xffffffffffffffffffffffff);
#endif /* DISABLE_128BIT_ARITHMETIC */
diff --git a/documentation/dataplane/dash-flow-api.md b/documentation/dataplane/dash-flow-api.md
new file mode 100644
index 000000000..d699b3091
--- /dev/null
+++ b/documentation/dataplane/dash-flow-api.md
@@ -0,0 +1,661 @@
+# DASH Flow API HLD
+
+| Rev | Date | Author | Change Description |
+| --- | ---- | ------ | ------------------ |
+| 0.1 | 03/20/2024 | Zhixiong Niu | Initial version |
+
+## Table of Contents
+
+- [DASH Flow API HLD](#dash-flow-api-hld)
+ - [Table of Contents](#table-of-contents)
+ - [Introduction](#introduction)
+ - [Overview](#overview)
+ - [Flow Table APIs](#flow-table-apis)
+ - [Flow APIs](#flow-apis)
+ - [Basic flow APIs](#basic-flow-apis)
+ - [Keys of flow entry](#keys-of-flow-entry)
+ - [Attributes of flow entry](#attributes-of-flow-entry)
+ - [Flow basic metadata](#flow-basic-metadata)
+ - [Reverse flow key](#reverse-flow-key)
+ - [Flow encap related attributes](#flow-encap-related-attributes)
+ - [Flow overlay rewrite related attributes](#flow-overlay-rewrite-related-attributes)
+ - [Extra flow metadata](#extra-flow-metadata)
+ - [Flow Bulk Get Session](#flow-bulk-get-session)
+ - [Flow Bulk Get Session filter](#flow-bulk-get-session-filter)
+ - [Flow Bulk Get Session API](#flow-bulk-get-session-api)
+ - [Bulk Get Session Event Notification](#bulk-get-session-event-notification)
+ - [Protobuf-based flow programming](#protobuf-based-flow-programming)
+ - [Capability](#capability)
+ - [Examples](#examples)
+ - [Create flow table](#create-flow-table)
+ - [Create flow entry key](#create-flow-entry-key)
+ - [Create flow entry](#create-flow-entry)
+ - [Add flow entries](#add-flow-entries)
+ - [Retrieve flow entry](#retrieve-flow-entry)
+ - [Retrieve flow entries via flow entry bulk get session](#retrieve-flow-entries-via-flow-entry-bulk-get-session)
+ - [Remove flow entry](#remove-flow-entry)
+ - [Remove flow table](#remove-flow-table)
+
+## Introduction
+
+DASH supports the storage and processing of millions of flow states. To further enhance the DASH flow processing capabilities, we offer a DASH flow abstraction layer to facilitate vendor-neutral flow management. This layer ensures uniform control over flows across programmable switches, DPUs, and smart switches. The DASH flow abstraction provides concepts of flow tables and flow entries, as well as APIs to manage the flows.
+
+The DASH flow APIs enable the creation, removal, retrieval, and configuration of flow tables, entries, and bulk sync sessions with flow filters.
+
+Cloud providers can leverage DASH flow to develop services tailored to a diverse array of scenarios. Examples of achievable functionalities include:
+
+- **Dataplane Applications**: Such as cloud gateways, load balancers, etc.
+- **Flow Management**: Including flow offloading and updating, with tasks like flow redirection and resimulation, etc.
+- **Dataplane Debugging**: Diagnosing the behaviors of different flows.
+- **Foundational Flow Services**: Flow state high availability, etc.
+
+## Overview
+
+- **Flow/flow entry**: It represents a single direction of the match-action entry for a connection.
+
+- **Flow Key**: The key that is used to match the packet for finding its flow.
+
+- **Flow State**: The state of the flow, including the packet transformations and all other tracked states, such as TCP, HA, etc.
+
+- **Flow Table**: The table to store a set of flows.
+
+
+
+The figure above illustrates the flow abstraction model. We represent flows as being stored within a flow table and managed through DASH flow SAI APIs. Flow entries, which contain the state information of flows, are organized within these flow tables. The key of a flow entry is utilized to retrieve its associated flow state. It's important to note that the ENI and flow table are not directly linked; for example, a single table can contain flow entries associated with various ENIs, and flows with the same ENI may span multiple flow tables. The choice of arrangement depends on specific scenarios.
+
+Upon the arrival of new flows, whether individually or in batches, corresponding flow entries are added to the table. These entries may represent either bidirectional or unidirectional flows. For bidirectional flows, the implementation adds entries for both the original flow and its reverse, linking their reverse flow keys to each other. For unidirectional flows, the current direction is specified. If a reverse flow for a unidirectional flow is created later, the user can add reverse keys for both and link them accordingly.
+
+Flows can be modified and removed through the DASH flow SAI API and can also be aged by the hardware.
+
+For more use cases, please refer to [Smart Switch HA HLD](https://github.com/sonic-net/SONiC/blob/master/doc/smart-switch/high-availability/smart-switch-ha-hld.md).
+
+## Flow Table APIs
+
+The flow_table APIs are define as follows:
+
+| API | Description |
+| ------------------------ | :------------------------------------ |
+| create_flow_table | Create a new flow table |
+| remove_flow_table | Remove a flow table |
+| set_flow_table_attribute | Set the attributes of a flow table |
+| get_flow_table_attribute | Obtain the attributes of a flow table |
+| create_flow_tables | Create multiple flow tables in bulk |
+| remove_flow_tables | Remove multiple flow tables in bulk |
+
+The attributes of the flow_table are defined as follows:
+
+| Attribute name | Type | Description |
+|----------------------------------------|----------------------------|-------------------------------------------------|
+| SAI_FLOW_TABLE_ATTR_MAX_FLOW_COUNT | `sai_uint32_t` | Maximum number of flows allowed in the table. |
+| SAI_FLOW_TABLE_ATTR_DASH_FLOW_ENABLED_KEY | `sai_dash_flow_enabled_key_t` | Key enable mask |
+| SAI_FLOW_TABLE_ATTR_FLOW_TTL_IN_MILLISECONDS | `sai_uint32_t` | Time-to-live (TTL) for flows in milliseconds. |
+
+The sai_dash_flow_enabled_key_t is defined as follows:
+
+```c
+typedef enum _sai_dash_flow_enabled_key_t
+{
+ SAI_DASH_FLOW_ENABLED_KEY_NONE = 0,
+
+ SAI_DASH_FLOW_ENABLED_KEY_ENI_ADDR = 1 << 1,
+
+ SAI_DASH_FLOW_ENABLED_KEY_VNI = 1 << 2,
+
+ SAI_DASH_FLOW_ENABLED_KEY_PROTOCOL = 1 << 3,
+
+ SAI_DASH_FLOW_ENABLED_KEY_SRC_IP = 1 << 4,
+
+ SAI_DASH_FLOW_ENABLED_KEY_DST_IP = 1 << 5,
+
+ SAI_DASH_FLOW_ENABLED_KEY_SRC_PORT = 1 << 6,
+
+ SAI_DASH_FLOW_ENABLED_KEY_DST_PORT = 1 << 7,
+
+} sai_dash_flow_enabled_key_t;
+```
+
+## Flow APIs
+
+### Basic flow APIs
+
+The flow_entry APIs are defined as follows:
+
+| API | Description |
+| ------------------------ | :----------------------------------------------------------- |
+| create_flow_entry | Add a single new entry to a certain flow table |
+| remove_flow_entry | Remove a single entry in a certain flow table. Note that the flow removal process deletes two flows if it is a bi-directional flow. If you wish to remove a flow in only one direction, you should set the flow to be uni-directional in advance. |
+| set_flow_entry_attribute | Set attributes for a single entry in a certain flow table |
+| get_flow_entry_attribute | Get attributes of a single entry in a certain flow table |
+| create_flow_entries | Add multiple entries to a certain flow table in bulk |
+| remove_flow_entries | Remove multiple entries from a specific flow table in bulk. Note that the flow removal process deletes two flows if it is a bi-directional flow. If you wish to remove a flow in only one direction, you should set the flow to be uni-directional in advance. |
+
+### Keys of flow entry
+
+The keys for a flow entry are defined as follows:
+
+Please note that there is an attribute in the *flow_table* that can specify which of the following keys are enabled. If a key is not enabled, it will not be used in match and action.
+
+The *flow_table_id* is used to designate the flow table for the flow only, which is not used in match and action.
+
+```c
+typedef struct _sai_flow_entry_t
+{
+ /**
+ * @brief Switch ID
+ *
+ * @objects SAI_OBJECT_TYPE_SWITCH
+ */
+ sai_object_id_t switch_id;
+
+ /**
+ * @brief Exact matched key flow_table_id
+ *
+ * @objects SAI_OBJECT_TYPE_FLOW_TABLE
+ */
+ sai_object_id_t flow_table_id;
+
+ /**
+ * @brief Exact matched key eni_mac
+ */
+ sai_mac_t eni_mac;
+
+ /**
+ * @brief Exact matched key vni
+ */
+ sai_uint32_t vni;
+
+ /**
+ * @brief Exact matched key ip_protocol
+ */
+ sai_uint8_t ip_proto;
+
+ /**
+ * @brief Exact matched key src_ip
+ */
+ sai_ip_address_t src_ip;
+
+ /**
+ * @brief Exact matched key dst_ip
+ */
+ sai_ip_address_t dst_ip;
+
+ /**
+ * @brief Exact matched key src_port
+ */
+ sai_uint16_t src_port;
+
+ /**
+ * @brief Exact matched key dst_port
+ */
+ sai_uint16_t dst_port;
+
+} sai_flow_entry_t;
+```
+
+### Attributes of flow entry
+
+The attributes of the flow entry can be divided into different categories. Please see below for further details.
+
+#### Flow basic metadata
+
+These are the basic attributes of flow entry.
+
+| Attribute name | Type | Description |
+| ------------------------------------------ | ------------------------ | ------------------------------------------------------------ |
+| SAI_FLOW_ENTRY_ATTR_VERSION | `sai_uint32_t` | Version of the flow entry |
+| SAI_FLOW_ENTRY_ATTR_DASH_DIRECTION | `sai_dash_direction_t` | Direction of the DASH flow |
+| SAI_FLOW_ENTRY_ATTR_DASH_FLOW_ACTION | `sai_dash_flow_action_t` | Action to be applied on the flow |
+| SAI_FLOW_ENTRY_ATTR_METER_CLASS | `sai_uint32_t` | Meter class for flow entry, used for traffic metering and policing. |
+| SAI_FLOW_ENTRY_ATTR_IS_UNIDIRECTIONAL_FLOW | `bool` | Indicates if the flow is unidirectional |
+
+#### Reverse flow key
+
+When configuring a flow_entry, it can be specified whether it is unidirectional or bidirectional. If it is bidirectional, it can be designated as a reverse flow key, allowing for the rapid identification of the corresponding reverse flow. Of course, if a flow entry is initially established as unidirectional, its reverse flow can also be set up later, utilizing these attributes to link them together.
+
+| Attribute name | Type | Description |
+| ----------------------------------------- | ------------------ | ------------------------------------------- |
+| SAI_FLOW_ENTRY_ATTR_REVERSE_FLOW_ENI_MAC | `sai_mac_t` | Eni mac addr for the reverse flow |
+| SAI_FLOW_ENTRY_ATTR_REVERSE_FLOW_VNI | `sai_uint32_t` | VNI for reverse flow |
+| SAI_FLOW_ENTRY_ATTR_REVERSE_FLOW_IP_PROTO | `sai_uint8_t` | IP protocol number for the reverse flow |
+| SAI_FLOW_ENTRY_ATTR_REVERSE_FLOW_SRC_IP | `sai_ip_address_t` | Source IP address for the reverse flow |
+| SAI_FLOW_ENTRY_ATTR_REVERSE_FLOW_DST_IP | `sai_ip_address_t` | Destination IP address for the reverse flow |
+| SAI_FLOW_ENTRY_ATTR_REVERSE_FLOW_SRC_PORT | `sai_uint16_t` | L4 source port for the reverse flow |
+| SAI_FLOW_ENTRY_ATTR_REVERSE_FLOW_DST_PORT | `sai_uint16_t` | L4 destination port for the reverse flow |
+
+#### Flow encap related attributes
+
+These are the related attributes of flow encapsulation.
+
+| Attribute name | Type | Description |
+| ------------------------------------------------ | -------------------------- | ------------------------------------------------------------ |
+| SAI_FLOW_ENTRY_ATTR_UNDERLAY0_VNI | `sai_uint32_t` | Destination VNI in the underlay network |
+| SAI_FLOW_ENTRY_ATTR_UNDERLAY0_SIP | `sai_uint32_t` | Source IP address in the underlay network |
+| SAI_FLOW_ENTRY_ATTR_UNDERLAY0_DIP | `sai_uint32_t` | Destination IP address in the underlay network |
+| SAI_FLOW_ENTRY_ATTR_UNDERLAY0_DASH_ENCAPSULATION | `sai_dash_encapsulation_t` | Encapsulation method for DASH traffic in the underlay network |
+| SAI_FLOW_ENTRY_ATTR_UNDERLAY1_VNI | `sai_uint32_t` | Destination VNI in the 2nd underlay network |
+| SAI_FLOW_ENTRY_ATTR_UNDERLAY1_SIP | `sai_uint32_t` | Source IP address in the 2nd underlay network |
+| SAI_FLOW_ENTRY_ATTR_UNDERLAY1_DIP | `sai_uint32_t` | Destination IP address in the 2nd underlay network |
+| SAI_FLOW_ENTRY_ATTR_UNDERLAY1_SMAC | `sai_mac_t` | Source MAC address in the 2nd underlay network |
+| SAI_FLOW_ENTRY_ATTR_UNDERLAY1_DMAC | `sai_mac_t` | Destination MAC address in the 2nd underlay network |
+| SAI_FLOW_ENTRY_ATTR_UNDERLAY1_DASH_ENCAPSULATION | `sai_dash_encapsulation_t` | Encapsulation method for DASH traffic in the 2nd underlay network |
+
+#### Flow overlay rewrite related attributes
+
+These are the related attributes of flow rewrite.
+
+| Attribute name | Type | Description |
+| ---------------------------- | ------------------ | ------------------------------------------------------------ |
+| SAI_FLOW_ENTRY_ATTR_DST_MAC | `sai_mac_t` | Destination MAC address for the flow entry. |
+| SAI_FLOW_ENTRY_ATTR_SIP | `sai_ip_address_t` | Source IP address for the flow entry, supporting both IPv4 and IPv6. |
+| SAI_FLOW_ENTRY_ATTR_DIP | `sai_ip_address_t` | Destination IP address for the flow entry, supporting both IPv4 and IPv6. |
+| SAI_FLOW_ENTRY_ATTR_SIP_MASK | `sai_ip_address_t` | Subnet mask for the source IP address. |
+| SAI_FLOW_ENTRY_ATTR_DIP_MASK | `sai_ip_address_t` | Subnet mask for the destination IP address. |
+
+### Extra flow metadata
+
+Here are some extra metadata for different purposes.
+
+| Attribute name | Type | Description |
+| ----------------------------------- | --------------- | ------------------------------------------------------------ |
+| SAI_FLOW_ENTRY_ATTR_VENDOR_METADATA | `sai_u8_list_t` | Vendor-specific metadata that can be attached to the flow entry for custom processing. |
+| SAI_FLOW_ENTRY_ATTR_FLOW_DATA_PB | `sai_u8_list_t` | The flow data protocol buffer enables high-efficiency creation, retrieval, and communication for a flow entry. |
+
+### Flow Bulk Get Session
+
+#### Flow Bulk Get Session filter
+
+To manage data transfer to a server via gRPC or event notification, we introduce a flow entry bulk session that incorporates filtering capabilities to precisely define the data range for transfer. The procedure for setting up these filters is straightforward:
+
+1. Initially, create up to five flow bulk get session filters based on the specific needs for filtering the flows.
+2. Subsequently, establish a flow bulk get session filter and integrate these filters as attributes.
+
+For example, consider the scenario where it's necessary to select all flow entries with a version less than 5 and greater than version 3. In this case, two distinct filters should be defined to meet the criteria and then create the flow entry bulk session.
+
+The filter, defined as an object, is specified as follows:
+
+| Function | Description |
+| ------------------------------------------------ | ------------------------------------------------------------ |
+| create_flow_entry_bulk_get_session_filter | Add a single new filter for flow entry bulk get session feature. |
+| remove_flow_entry_bulk_get_session_filter | Remove a single filter for flow entry bulk get session feature. |
+| set_flow_entry_bulk_get_session_filter_attribute | Set attributes for a single filter in flow entry bulk get session. |
+| get_flow_entry_bulk_get_session_filter_attribute | Get attributes of a single filter in flow entry bulk get session. |
+| create_flow_entry_bulk_get_session_filters | Add multiple new filters for flow entry bulk get session feature. |
+| remove_flow_entry_bulk_get_session_filters | Remove multiple filters for flow entry bulk get session feature. |
+
+| Attribute Name | Type | Description |
+| ------------------------------------------------------------ | --------------------------------------------------- | ------------------------------------------------------------ |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY | `sai_dash_flow_entry_bulk_get_session_filter_key_t` | Key of the filter |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_DASH_FLOW_ENTRY_BULK_GET_SESSION_OP_KEY | `sai_dash_flow_entry_bulk_get_session_op_key_t` | Operation of the filter |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_INT_VALUE | `sai_uint64_t` | INT Value of the filter , ``@validonly SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY == SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_IP_PROTO || SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY == SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_SRC_PORT || SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY == SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_DST_PORT || SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY == SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_FLOW_VERSION` |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_IP_VALUE | `sai_ip_address_t` | IP Value of the filter, `@validonly SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY == SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_SRC_IP || SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY == SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_DST_IP` |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_MAC_VALUE | `sai_mac_t` | Mac Value of the filter, `@validonly SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY == SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_ENI_MAC` |
+
+```c
+typedef enum _sai_dash_flow_entry_bulk_get_session_filter_key_t
+{
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_NONE,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_ENI_MAC,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_VNI,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_IP_PROTO,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_SRC_IP,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_DST_IP,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_SRC_PORT,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_DST_PORT,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_FLOW_VERSION,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_AGED,
+
+} sai_dash_flow_entry_bulk_get_session_filter_key_t;
+```
+
+```c
+typedef enum _sai_dash_flow_entry_bulk_get_session_op_key_t
+{
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_OP_KEY_FILTER_OP_INVALID,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_OP_KEY_FILTER_OP_EQUAL_TO,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_OP_KEY_FILTER_OP_GREATER_THAN,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_OP_KEY_FILTER_OP_GREATER_THAN_OR_EQUAL_TO,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_OP_KEY_FILTER_OP_LESS_THAN,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_OP_KEY_FILTER_OP_LESS_THAN_OR_EQUAL_TO,
+
+} sai_dash_flow_entry_bulk_get_session_op_key_t;
+```
+
+#### Flow Bulk Get Session API
+
+Upon establishing the bulk get session filters, we can initiate a flow bulk get session, which is defined as follows:
+
+| Function | Description |
+| ----------------------------------------- | ----------------------------------------------------------- |
+| create_flow_entry_bulk_get_session | Add a single new session for flow entry bulk get feature |
+| remove_flow_entry_bulk_get_session | Remove a single new session for flow entry bulk get feature |
+| set_flow_entry_bulk_get_session_attribute | Set attributes for a single session |
+| get_flow_entry_bulk_get_session_attribute | Get attributes of a single session |
+| create_flow_entry_bulk_get_sessions | Add multiple new sessions for flow entry bulk get feature |
+| remove_flow_entry_bulk_get_sessions | Remove multiple sessions for flow entry bulk get feature |
+
+In the attributes, we allow specifying the gRPC server and port when the mode is gRPC. For filtering flow entries, we support up to five filters. Each filter is a bulk get session filter object, and different filters are combined using an *AND* operation. If no filters are specified, the bulk get session returns all flow entries.
+
+| Attribute Name | Type | Description |
+| ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_ATTR_BULK_GET_SESSION_FLOW_TABLE | `sai_object_id_t` | Flow table to bulk get |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_ATTR_BULK_GET_SESSION_MODE | `sai_dash_flow_entry_bulk_get_session_mode_t` | Specify bulk get mode |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_ATTR_BULK_ENTRY_LIMITATION | `sai_uint32_t` | Specify a maximum limit for the bulk get session |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_ATTR_BULK_GET_SESSION_SERVER_IP | `sai_ip_address_t` | The IP address to use for the bulk get session. |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_ATTR_BULK_GET_SESSION_SERVER_PORT | `sai_uint16_t` | The port to use for the bulk get session. |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_ATTR_FIRST_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ID | @type: `sai_object_id_t` @objects `SAI_OBJECT_TYPE_FLOW_ENTRY_BULK_GET_SESSION_FILTER` | Action set_flow_entry_bulk_get_session_attr parameter BULK_GET_SESSION_IP |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_ATTR_SECOND_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ID | @type: `sai_object_id_t` @objects `SAI_OBJECT_TYPE_FLOW_ENTRY_BULK_GET_SESSION_FILTER` | Action set_flow_entry_bulk_get_session_attr parameter BULK_GET_SESSION_PORT |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_ATTR_THIRD_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ID | @type: `sai_object_id_t` @objects `SAI_OBJECT_TYPE_FLOW_ENTRY_BULK_GET_SESSION_FILTER` | Action set_flow_entry_bulk_get_session_attr parameter FIRST_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ID |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_ATTR_FOURTH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ID | @type: `sai_object_id_t` @objects `SAI_OBJECT_TYPE_FLOW_ENTRY_BULK_GET_SESSION_FILTER` | Action set_flow_entry_bulk_get_session_attr parameter SECOND_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ID |
+| SAI_FLOW_ENTRY_BULK_GET_SESSION_ATTR_FIFTH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ID | @type: `sai_object_id_t` @objects `SAI_OBJECT_TYPE_FLOW_ENTRY_BULK_GET_SESSION_FILTER` | Action set_flow_entry_bulk_get_session_attr parameter THIRD_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ID |
+
+```c
+typedef enum _sai_dash_flow_entry_bulk_get_session_mode_t
+
+{
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_MODE_GRPC,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_MODE_EVENT,
+
+ SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_MODE_EVENT_WITHOUT_FLOW_STATE,
+
+} sai_dash_flow_entry_bulk_get_session_mode_t;
+```
+
+#### Bulk Get Session Event Notification
+
+| Attribute name | Type | Description |
+| -------------------------------------------------- | ------------------------------------------------- | ------------------------------------------------------------ |
+| SAI_SWITCH_ATTR_FLOW_BULK_GET_SESSION_EVENT_NOTIFY | `sai_flow_bulk_get_session_event_notification_fn` | The callback function for receiving events on flow bulk get session event notification. |
+
+```c
+/**
+ * @brief bulk flow get event type
+ */
+typedef enum _sai_flow_bulk_get_session_event_t
+{
+ SAI_FLOW_BULK_GET_SESSION_FINISHED,
+
+ SAI_FLOW_BULK_GET_SESSION_FLOW_ENTRY,
+
+} sai_flow_bulk_get_session_event_t;
+
+/**
+ * @brief Notification data format received from SAI HA set callback
+ *
+ * @count attr[attr_count]
+ */
+typedef struct _sai_flow_bulk_get_session_event_data_t
+{
+ sai_flow_bulk_get_session_event_t event_type;
+
+ sai_object_id_t flow_bulk_session_id;
+
+ sai_flow_entry_t *flow_entry;
+
+ uint32_t attr_count;
+
+ sai_attribute_t *attr_list;
+
+} sai_flow_bulk_get_session_event_data_t;
+
+/**
+ * @brief dash flow get bulk session notification
+ *
+ * Passed as a parameter into sai_initialize_switch()
+ *
+ * @count data[count]
+ *
+ * @param[in] count Number of notifications
+ * @param[in] data Array of flow bulk get session events
+ */
+typedef void (*sai_flow_bulk_get_session_event_notification_fn)(
+ _In_ uint32_t count,
+ _In_ const sai_flow_bulk_get_session_event_data_t *flow_bulk_get_session_event_data);
+```
+
+### Protobuf-based flow programming
+
+In addition to the flow state attributes, the flow state can be represented using protobuf for high-efficiency. The attribute of the flow entry is SAI_FLOW_ENTRY_ATTR_FLOW_DATA_PB.
+
+Although the content of both attributes and protobuf may be identical, their applications differ. Attributes enable incremental updates to individual properties, whereas protobuf necessitates a complete update.
+
+```protobuf
+syntax = "proto3";
+
+message MacAddress {
+ bytes address = 1 [(validate.rules).bytes.len = 6]; // MAC address bytes
+}
+
+message IpAddress {
+ uint16 type = 1; // IP address type (IPv4 = 2 or IPv6 = 10)
+ bytes address = 2 [(validate.rules).bytes.len = {const: 4 | const: 16}]; // IP address bytes, 4 bytes for IPv4, 16 bytes for IPv6
+}
+
+message SaiDashFlowKey {
+ bytes eni_mac = 1; // ENI MAC address, using bytes to match MacAddress structure
+ uint32 vni = 2; // VNI
+ IpAddress src_ip = 3; // Source IP address
+ IpAddress dst_ip = 4; // Destination IP address
+ uint8 ip_proto = 5; // IP Protocol
+ uint32 src_port = 6; // Source port
+ uint32 dst_port = 7; // Destination port
+}
+
+message SaiDashFlowState {
+ uint32 version = 1; // SAI_FLOW_ENTRY_ATTR_VERSION
+ uint16 dash_direction = 2; // SAI_FLOW_ENTRY_ATTR_DASH_DIRECTION
+ uint32 dash_flow_action = 3; // SAI_FLOW_ENTRY_ATTR_DASH_FLOW_ACTION
+ uint32 meter_class = 4; // SAI_FLOW_ENTRY_ATTR_METER_CLASS
+ bool is_unidirectional_flow = 5; // SAI_FLOW_ENTRY_ATTR_IS_UNIDIRECTIONAL_FLOW
+ uint32 underlay0_vni = 6; // SAI_FLOW_ENTRY_ATTR_UNDERLAY0_VNI
+ IpAddress underlay0_sip = 7; // SAI_FLOW_ENTRY_ATTR_UNDERLAY0_SIP
+ IpAddress underlay0_dip = 8; // SAI_FLOW_ENTRY_ATTR_UNDERLAY0_DIP
+ uint16 underlay0_dash_encapsulation = 9; // SAI_FLOW_ENTRY_ATTR_UNDERLAY0_DASH_ENCAPSULATION
+ uint32 underlay1_vni = 10; // SAI_FLOW_ENTRY_ATTR_UNDERLAY1_VNI
+ IpAddress underlay1_sip = 11; // SAI_FLOW_ENTRY_ATTR_UNDERLAY1_SIP
+ IpAddress underlay1_dip = 12; // SAI_FLOW_ENTRY_ATTR_UNDERLAY1_DIP
+ MacAddress underlay1_smac = 13; // SAI_FLOW_ENTRY_ATTR_UNDERLAY1_SMAC
+ MacAddress underlay1_dmac = 14; // SAI_FLOW_ENTRY_ATTR_UNDERLAY1_DMAC
+ uint16 underlay1_dash_encapsulation = 15; // SAI_FLOW_ENTRY_ATTR_UNDERLAY1_DASH_ENCAPSULATION
+ MacAddress dst_mac = 16; // SAI_FLOW_ENTRY_ATTR_DST_MAC
+ IpAddress sip = 17; // SAI_FLOW_ENTRY_ATTR_SIP
+ IpAddress dip = 18; // SAI_FLOW_ENTRY_ATTR_DIP
+ bytes sip_mask = 19; // SAI_FLOW_ENTRY_ATTR_SIP_MASK
+ bytes dip_mask = 20; // SAI_FLOW_ENTRY_ATTR_DIP_MASK
+}
+
+message SaiDashFlowEntry {
+ SaiDashFlowKey flow_key = 1;
+ SaiDashFlowKey reverse_flow_key = 2;
+ SaiDashFlowState flow_state = 3;
+}
+
+```
+
+### Capability
+
+| Attribute Name | Type | Description |
+| --------------------------------------------------- | ----------------------------- | -------------------------------------------------- |
+| SAI_SWITCH_ATTR_DASH_CAPS_MAX_FLOW_TABLE_COUNT | `sai_uint32_t` | The max number of flow tables that can be created |
+| SAI_SWITCH_ATTR_DASH_CAPS_MAX_FLOW_ENTRY_COUNT | `sai_uint32_t` | The max number of flow entries for all tables |
+| SAI_SWITCH_ATTR_DASH_CAPS_SUPPORTED_ENABLED_KEY | `sai_dash_flow_enabled_key_t` | Indicates what flow key mask can be used |
+| SAI_SWITCH_ATTR_DASH_CAPS_BULK_GET_SESSION | `bool` | Indicates if it supports bulk get sessions |
+| SAI_SWITCH_ATTR_DASH_CAPS_UNIDIRECTIONAL_FLOW_ENTRY | `bool` | Indicates if it supports unidirectional flow entry |
+| SAI_SWITCH_ATTR_DASH_CAPS_FLOW_CREATE | `bool` | Indicates if it supports flow create |
+| SAI_SWITCH_ATTR_DASH_CAPS_FLOW_REMOVE | `bool` | Indicates if it supports flow remove |
+| SAI_SWITCH_ATTR_DASH_CAPS_FLOW_SET | `bool` | Indicates if it supports flow set |
+| SAI_SWITCH_ATTR_DASH_CAPS_FLOW_GET | `bool` | Indicates if it supports flow get |
+
+## Examples
+
+When a service intends to use the DASH Flow SAI APIs, it should first establish a flow table via the `create_flow_table()` function. After the table creation, the programmer can add, delete, modify, or retrieve flow entries from the table using the DASH flow API. It can also use `flow_entry_bulk_get_session` to retrieve flows with filters, allowing it to handle flows in batches under specific conditions.
+
+
+
+For instance, the figure above shows an example of a DASH flow HA between two DPUs. First, both DPUs should create flow tables for initialization via `create_flow_table`. When performing Inline Sync, DPU1 and DPU2 use `create_flow_entry` to create new flows as new flows arrive. When DASH flow HA needs to perform bulk sync from the active DPU to the standby DPU, it should initially fetch the entry from the active DPU using `create_flow_entry_bulk_get_session` to create a bulk get session to transfer the flows to DPU2 via gRPC. Then, the standby DPU subsequently calls `create_flow_entries()` to add entries to the corresponding flow table.
+
+Below are detailed examples to use the DASH flow API.
+
+### Create flow table
+
+```c
+uint32_t attr_count = 3;
+sai_attribute_t attr_list[3];
+attr_list[0].id = SAI_FLOW_TABLE_ATTR_DASH_FLOW_ENABLED_KEY;
+attr_list[0].value = SAI_DASH_FLOW_ENABLED_KEY_PROTOCOL |
+ SAI_DASH_FLOW_ENABLED_KEY_ENI_ADDR |
+ SAI_DASH_FLOW_ENABLED_KEY_VNI |
+ SAI_DASH_FLOW_ENABLED_KEY_SRC_IP |
+ SAI_DASH_FLOW_ENABLED_KEY_DST_IP |
+ SAI_DASH_FLOW_ENABLED_KEY_SRC_PORT |
+ SAI_DASH_FLOW_ENABLED_KEY_DST_PORT;
+...
+
+sai_object_id_t flow_table_id;
+sai_status_t status = create_flow_table(&flow_table_id, switch_id, attr_count, attr_list);
+```
+
+### Create flow entry key
+
+```c
+sai_flow_entry_t flow_entry;
+
+flow_entry.flow_table_id = 0x112233;
+flow_entry.ip_proto = 6;
+flow_entry.src_ip.addr_family = SAI_IP_ADDR_FAMILY_IPV4;
+inet_pton(AF_INET, "192.168.1.1", &flow_entry.src_ip.addr.ip4);
+flow_entry.dst_ip.addr_family = SAI_IP_ADDR_FAMILY_IPV4;
+inet_pton(AF_INET, "192.168.1.2", &flow_entry.dst_ip.addr.ip4);
+flow_entry.src_port = 12345;
+flow_entry.dst_port = 80;
+```
+
+### Create flow entry
+
+```c
+SaiDashFlowEntry flow_entry_pb = SAI_DASH_FLOW_ENTRY__INIT;
+...
+
+unsigned len = sai_dash_flow_entry__get_packed_size(&flow_entry_pb);
+uint8_t *buf = malloc(len);
+sai_dash_flow_entry__pack(&flow_entry_pb, buf);
+
+sai_attribute_t sai_attrs_list[1];
+sai_attrs_list[0].id = SAI_FLOW_ENTRY_ATTR_FLOW_DATA_PB;
+sai_attr_list[0].value = buf;
+
+sai_status_t status = create_flow_entry(&flow_entry, 1, attr_list);
+
+free(buf);
+```
+
+### Add flow entries
+
+```c
+uint32_t flow_count = num_flow_states;
+const sai_dash_flow_key_t flow_key[] = ...;
+uint32_t attr_count[] = ...;
+sai_attribute_t *attr_list[] = ...;
+sai_status_t object_statuses[] = ...;
+
+status = create_flow_entries(flow_table_id, flow_count, flow_key, attr_count, attr_list, SAI_BULK_OP_ERROR_MODE_IGNORE_ERROR, object_statuses);
+```
+
+### Retrieve flow entry
+
+```c
+sai_flow_entry_t flow_entry;
+flow_entry.flow_table_id = 0x112233;
+flow_entry.ip_proto = 6;
+flow_entry.src_ip.addr_family = SAI_IP_ADDR_FAMILY_IPV4;
+inet_pton(AF_INET, "192.168.1.1", &flow_entry.src_ip_addr.addr.ip4);
+flow_entry.dst_ip.addr_family = SAI_IP_ADDR_FAMILY_IPV4;
+inet_pton(AF_INET, "192.168.1.2", &flow_entry.dst_ip_addr.addr.ip4);
+flow_entry.src_port = 12345;
+flow_entry.dst_port = 80;
+
+status = get_flow_entry_attribute(flow_entry, attr_count, attr_list);
+```
+
+### Retrieve flow entries via flow entry bulk get session
+
+Example: Retrieve flow entries by filtering for all versions greater than 3 and less than 5, and return the results via GRPC.
+
+```c
+/* Filter: Flow Entry Version > 3 */
+sai_attribute_t filter1_attr_list[3];
+filter1_attr_list[0].id = SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY;
+filter1_attr_list[0].value = SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_KEY_VERSION;
+filter1_attr_list[1].id = SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_DASH_FLOW_ENTRY_BULK_GET_SESSION_OP_KEY;
+filter1_attr_list[1].value = SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_OP_KEY_FILTER_OP_GREATER_THAN,;
+filter1_attr_list[2].id = SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_INT_VALUE;
+filter1_attr_list[2].value = 3;
+
+sai_object_id_t filter1_id;
+status = create_flow_entry_bulk_get_session_filter(&filter1_id, switch_id, 3, filter1_attr_list);
+
+/* Filter: Flow Entry Version < 5 */
+sai_attribute_t filter2_attr_list[3];
+filter2_attr_list[0].id = SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY;
+filter2_attr_list[0].value = SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_FILTER_KEY_KEY_VERSION;
+filter2_attr_list[1].id = SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_DASH_FLOW_ENTRY_BULK_GET_SESSION_OP_KEY;
+filter2_attr_list[1].value = SAI_DASH_FLOW_ENTRY_BULK_GET_SESSION_OP_KEY_FILTER_OP_LESS_THAN;
+filter2_attr_list[2].id = SAI_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ATTR_INT_VALUE;
+filter2_attr_list[2].value = 5;
+
+sai_object_id_t filter2_id;
+status = create_flow_entry_bulk_get_session_filter(&filter2_id, switch_id, 3, filter2_attr_list);
+
+/* Session */
+sai_attribute_t session_attr_list[4];
+session_attr_list[0].value.addr_family = SAI_IP_ADDR_FAMILY_IPV4;
+inet_pton(AF_INET, "10.0.0.1", &(session_attr_list[0].value.addr.ip4));
+session_attr_list[1].id = SAI_FLOW_ENTRY_BULK_GET_SESSION_ATTR_BULK_GET_SESSION_PORT;
+session_attr_list[1].value = 1000;
+session_attr_list[2].id = SAI_FLOW_ENTRY_BULK_GET_SESSION_ATTR_FIRST_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ID;
+session_attr_list[2].value = filter1_id;
+session_attr_list[3].id = SAI_FLOW_ENTRY_BULK_GET_SESSION_ATTR_SECOND_FLOW_ENTRY_BULK_GET_SESSION_FILTER_ID;
+session_attr_list[3].value = filter2_id;
+
+sai_object_id_t flow_entry_bulk_get_session_id;
+status = create_flow_entry_bulk_get_session(&flow_entry_bulk_get_session_id, switch_id, 4, session_attr_list);
+```
+
+### Remove flow entry
+
+```c
+/* Note that the flow removal process deletes two flows if it is a bi-directional flow. If you wish to remove a flow in only one direction, you should set the flow to be uni-directional in advance */
+
+sai_flow_entry_t flow_entry;
+flow_entry.flow_table_id = 0x112233;
+flow_entry.ip_proto = 6;
+flow_entry.src_ip.addr_family = SAI_IP_ADDR_FAMILY_IPV4;
+inet_pton(AF_INET, "192.168.1.1", &flow_entry.src_ip.addr.ip4);
+flow_entry.dst_ip.addr_family = SAI_IP_ADDR_FAMILY_IPV4;
+inet_pton(AF_INET, "192.168.1.2", &flow_entry.dst_ip.addr.ip4);
+flow_entry.src_port = 12345;
+flow_entry.dst_port = 80;
+
+status = remove_flow_entry(flow_entry);
+```
+
+### Remove flow table
+
+```c
+sai_object_id_t flow_table_id = 0x112233;
+status = remove_flow_table(flow_table_id);
+```
diff --git a/documentation/dataplane/dash-flow-api.pdf b/documentation/dataplane/dash-flow-api.pdf
new file mode 100644
index 000000000..89681dd19
Binary files /dev/null and b/documentation/dataplane/dash-flow-api.pdf differ
diff --git a/documentation/dataplane/images/dash-flow-api-example.svg b/documentation/dataplane/images/dash-flow-api-example.svg
new file mode 100644
index 000000000..59104b2d3
--- /dev/null
+++ b/documentation/dataplane/images/dash-flow-api-example.svg
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/documentation/dataplane/images/dash-flow-api-model.svg b/documentation/dataplane/images/dash-flow-api-model.svg
new file mode 100644
index 000000000..1a42f2b31
--- /dev/null
+++ b/documentation/dataplane/images/dash-flow-api-model.svg
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file