From 7e16e7b4c492b0cbb4ce4acf3f4bacd583bda2f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jere=20Lepp=C3=A4nen?= Date: Fri, 22 Dec 2023 13:32:57 +0200 Subject: [PATCH 001/147] linux-gen: pktio: avoid possibility of out of bounds warnings in lso_update_custom() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit odp_lso_profile_create() ensures that size is one of the allowed values. But the compiler doesn't know that, and it may generate out of bounds warnings in some cases. For example, in odp-dpdk, where odp_packet_copy_to_mem() calls rte_memcpy(), this results in: /usr/lib/gcc/x86_64-redhat-linux/13/include/emmintrin.h:742:8: warning: array subscript ‘__m128i_u[0]’ is partly outside array bounds of ‘void[8]’ [-Warray-bounds=] Set the value of size to 1 if it's not one of the allowed values. Signed-off-by: Jere Leppänen Reviewed-by: Petri Savolainen --- platform/linux-generic/odp_packet_io.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 236813e80..8283c41e6 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -3015,8 +3015,15 @@ static int lso_update_custom(lso_profile_t *lso_prof, odp_packet_t pkt, int segn ptr = &u32; else if (size == 2) ptr = &u16; - else + else { + /* + * odp_lso_profile_create() ensures that size is one of the allowed values. + * But compiler doesn't know that, so set it here to avoid possibility of + * out of bounds warnings. + */ + size = 1; ptr = &u8; + } if (odp_packet_copy_to_mem(pkt, offset, size, ptr)) { _ODP_ERR("Read from packet failed at offset %u\n", offset); From 188d737dd2300797ee89c95bf5027d5a7825041b Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Wed, 27 Dec 2023 09:58:21 +0200 Subject: [PATCH 002/147] test: timer: add timer free return value checks Add missing timer free return value checks into various timer test applications. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- example/timer/odp_timer_test.c | 5 ++++- test/performance/odp_bench_timer.c | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/example/timer/odp_timer_test.c b/example/timer/odp_timer_test.c index a397a6ac9..0be53c4cd 100644 --- a/example/timer/odp_timer_test.c +++ b/example/timer/odp_timer_test.c @@ -178,7 +178,10 @@ static void test_abs_timeouts(int thr, test_globals_t *gbls) continue; odp_event_free(ttp->ev); - odp_timer_free(ttp->tim); + + if (odp_timer_free(ttp->tim)) + ODPH_ABORT("Timer free failed (%" PRIu64 ")\n", odp_timer_to_u64(ttp->tim)); + ttp = NULL; } diff --git a/test/performance/odp_bench_timer.c b/test/performance/odp_bench_timer.c index a53671460..78dd9e2d2 100644 --- a/test/performance/odp_bench_timer.c +++ b/test/performance/odp_bench_timer.c @@ -695,8 +695,12 @@ int main(int argc, char *argv[]) if (gbl_args->pool != ODP_POOL_INVALID) odp_pool_destroy(gbl_args->pool); - if (gbl_args->timer != ODP_TIMER_INVALID) - odp_timer_free(gbl_args->timer); + if (gbl_args->timer != ODP_TIMER_INVALID) { + if (odp_timer_free(gbl_args->timer)) { + ODPH_ERR("Timer free failed\n"); + exit(EXIT_FAILURE); + } + } if (gbl_args->timer_pool != ODP_TIMER_POOL_INVALID) odp_timer_pool_destroy(gbl_args->timer_pool); From 29903c61897b8e51214246440f037ee4c61a6d5c Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Fri, 29 Dec 2023 16:47:32 +0200 Subject: [PATCH 003/147] test: bench_misc: add prefetch tests Added test cases for odp_prefetch() and odp_prefetch_store(). Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/performance/odp_bench_misc.c | 40 ++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/test/performance/odp_bench_misc.c b/test/performance/odp_bench_misc.c index 64318938a..c9c9f0d75 100644 --- a/test/performance/odp_bench_misc.c +++ b/test/performance/odp_bench_misc.c @@ -20,7 +20,7 @@ #include /* Number of API function calls per test case */ -#define REPEAT_COUNT 1000 +#define REPEAT_COUNT 1024 /* Default number of rounds per test case */ #define ROUNDS 1000u @@ -733,6 +733,42 @@ static int mb_full(void) return i; } +static int prefetch(void) +{ + uint64_t *a1 = gbl_args->a1; + uint32_t index = 0; + int i; + + for (i = 0; i < REPEAT_COUNT; i++) { + odp_prefetch(&a1[index]); + + /* Prefetch every 64B */ + index += 8; + if (odp_unlikely(index >= REPEAT_COUNT)) + index = 0; + } + + return i; +} + +static int prefetch_store(void) +{ + uint64_t *a1 = gbl_args->a1; + uint32_t index = 0; + int i; + + for (i = 0; i < REPEAT_COUNT; i++) { + odp_prefetch_store(&a1[index]); + + /* Prefetch every 64B */ + index += 8; + if (odp_unlikely(index >= REPEAT_COUNT)) + index = 0; + } + + return i; +} + bench_info_t test_suite[] = { BENCH_INFO(time_local, NULL, 0, NULL), BENCH_INFO(time_local_strict, NULL, 0, NULL), @@ -785,6 +821,8 @@ bench_info_t test_suite[] = { BENCH_INFO(mb_release, NULL, 0, NULL), BENCH_INFO(mb_acquire, NULL, 0, NULL), BENCH_INFO(mb_full, NULL, 0, NULL), + BENCH_INFO(prefetch, NULL, 0, NULL), + BENCH_INFO(prefetch_store, NULL, 0, NULL), }; /* Print usage information */ From af6d32db813f3f2dbd51e9d1e3856e8d8863c266 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Tue, 2 Jan 2024 16:22:27 +0200 Subject: [PATCH 004/147] doc: doxygen: increase dot graph size Increase Doxygen's maximum dot graph size for the default (50). For example, any new include of odp/api/std_types.h would result into too many nodes in std_type.h graph. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- doc/Doxyfile_common | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/Doxyfile_common b/doc/Doxyfile_common index 2aa54ac48..6c169b4e7 100644 --- a/doc/Doxyfile_common +++ b/doc/Doxyfile_common @@ -30,6 +30,7 @@ MACRO_EXPANSION = YES EXPAND_ONLY_PREDEF = YES INTERNAL_DOCS = YES DOT_IMAGE_FORMAT = svg +DOT_GRAPH_MAX_NODES = 60 PREDEFINED = __GNUC__ \ __attribute__(x)= \ ODP_ALIGNED(x)= \ From 462fa8fc683a0609aaafa9a4c6d815bfad76f7d3 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 18 Dec 2023 12:08:46 +0200 Subject: [PATCH 005/147] example: improve example application documentations Include short description in Doxygen output for each example application. The documentations are moved to application source files for ease of maintenance. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- doc/application-api-guide/Doxyfile | 3 +- doc/application-api-guide/Makefile.am | 1 - doc/application-api-guide/examples.dox | 100 ------------------ example/classifier/odp_classifier.c | 8 ++ example/cli/odp_cli.c | 6 +- example/debug/odp_debug.c | 8 ++ example/generator/odp_generator.c | 8 ++ example/hello/odp_hello.c | 7 +- example/ipfragreass/odp_ipfragreass.c | 6 +- .../ipfragreass/odp_ipfragreass_fragment.c | 2 + .../ipfragreass/odp_ipfragreass_fragment.h | 2 + example/ipfragreass/odp_ipfragreass_helpers.c | 2 + example/ipfragreass/odp_ipfragreass_helpers.h | 2 + example/ipfragreass/odp_ipfragreass_ip.h | 2 + .../ipfragreass/odp_ipfragreass_reassemble.c | 2 + .../ipfragreass/odp_ipfragreass_reassemble.h | 2 + example/ipsec_api/odp_ipsec.c | 6 +- example/ipsec_api/odp_ipsec_cache.c | 2 + example/ipsec_api/odp_ipsec_cache.h | 2 + example/ipsec_crypto/odp_ipsec.c | 6 +- example/ipsec_crypto/odp_ipsec_cache.c | 2 + example/ipsec_crypto/odp_ipsec_cache.h | 2 + example/ipsec_crypto/odp_ipsec_fwd_db.c | 2 + example/ipsec_crypto/odp_ipsec_fwd_db.h | 2 + example/ipsec_crypto/odp_ipsec_misc.h | 2 + example/ipsec_crypto/odp_ipsec_sa_db.c | 2 + example/ipsec_crypto/odp_ipsec_sa_db.h | 2 + example/ipsec_crypto/odp_ipsec_sp_db.c | 2 + example/ipsec_crypto/odp_ipsec_sp_db.h | 2 + example/ipsec_crypto/odp_ipsec_stream.c | 2 + example/ipsec_crypto/odp_ipsec_stream.h | 2 + example/l2fwd_simple/odp_l2fwd_simple.c | 8 ++ example/l3fwd/odp_l3fwd.c | 8 ++ example/l3fwd/odp_l3fwd_db.c | 2 + example/l3fwd/odp_l3fwd_db.h | 2 + example/l3fwd/odp_l3fwd_lpm.c | 3 + example/l3fwd/odp_l3fwd_lpm.h | 2 + example/packet/odp_packet_dump.c | 8 ++ example/packet/odp_pktio.c | 8 ++ example/ping/odp_ping.c | 10 ++ example/simple_pipeline/odp_simple_pipeline.c | 11 ++ example/switch/odp_switch.c | 8 ++ example/sysinfo/odp_sysinfo.c | 9 ++ example/time/time_global_test.c | 8 ++ example/timer/odp_timer_accuracy.c | 8 ++ example/timer/odp_timer_simple.c | 8 +- example/timer/odp_timer_test.c | 8 ++ example/traffic_mgmt/odp_traffic_mgmt.c | 8 ++ test/performance/odp_l2fwd.c | 8 ++ 49 files changed, 212 insertions(+), 114 deletions(-) delete mode 100644 doc/application-api-guide/examples.dox diff --git a/doc/application-api-guide/Doxyfile b/doc/application-api-guide/Doxyfile index 7f8c70c8e..13a365c72 100644 --- a/doc/application-api-guide/Doxyfile +++ b/doc/application-api-guide/Doxyfile @@ -5,6 +5,7 @@ PROJECT_NUMBER = $(ODP_VERSION_API) PROJECT_LOGO = $(SRCDIR)/doc/images/ODP-Logo-HQ.svg INPUT = $(SRCDIR)/doc/application-api-guide \ include \ - $(SRCDIR)/include + $(SRCDIR)/include \ + $(SRCDIR)/example EXAMPLE_PATH = $(SRCDIR)/example $(SRCDIR)/CONTRIBUTING $(SRCDIR)/include/README WARNINGS = NO diff --git a/doc/application-api-guide/Makefile.am b/doc/application-api-guide/Makefile.am index 7d9003976..29599597d 100644 --- a/doc/application-api-guide/Makefile.am +++ b/doc/application-api-guide/Makefile.am @@ -1,7 +1,6 @@ EXTRA_DIST = \ Doxyfile \ api_guide_lines.dox \ - examples.dox \ odp.dox \ release.dox diff --git a/doc/application-api-guide/examples.dox b/doc/application-api-guide/examples.dox deleted file mode 100644 index 18817cd63..000000000 --- a/doc/application-api-guide/examples.dox +++ /dev/null @@ -1,100 +0,0 @@ -/* Copyright (c) 2015-2019, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -/** - * @example odp_classifier.c - * Classifier example application - */ - -/** - * @example odp_cli.c - * CLI example application - */ - -/** - * @example odp_debug.c - * Debug example application - */ - -/** - * @example odp_generator.c - * Traffic generator and loopback demo application - */ - -/** - * @example odp_hello.c - * Minimal example application - */ - -/** - * @example odp_ipfragreass.c - * IPv4 lock-free fragmentation and reassembly example application - */ - -/** - * @example ipsec_api/odp_ipsec.c - * IPsec example application using IPsec API - */ - - /** - * @example ipsec_crypto/odp_ipsec.c - * IPsec example application using crypto API - */ - -/** - * @example odp_l2fwd.c - * L2 forwarding example application - */ - -/** - * @example odp_l2fwd_simple.c - * Minimal L2 forwarding example application - */ - -/** - * @example odp_l3fwd.c - * L3 forwarding example application - */ - -/** - *@example odp_ping.c - * Example application which replies to ICMPv4 echo requests (ping) - */ - -/** - *@example odp_pktio.c - * Basic packet IO loopback test application - */ - - /** - *@example odp_simple_pipeline.c - * Simple pipeline example application - */ - -/** - * @example odp_switch.c - * Minimal learning Ethernet switch example application - */ - - /** - * @example odp_sysinfo.c - * System information example application - */ - -/** - * @example time_global_test.c - * Time example application - */ - -/** - * @example odp_timer_test.c - * Timer example application - */ - - /** - * @example odp_traffic_mgmt.c - * Traffic manager example application - */ diff --git a/example/classifier/odp_classifier.c b/example/classifier/odp_classifier.c index 4f71c835b..da92085b0 100644 --- a/example/classifier/odp_classifier.c +++ b/example/classifier/odp_classifier.c @@ -6,6 +6,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_classifier.c + * + * Classifier API example application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/example/cli/odp_cli.c b/example/cli/odp_cli.c index 5f87b2a78..55b70c8bb 100644 --- a/example/cli/odp_cli.c +++ b/example/cli/odp_cli.c @@ -4,12 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ -/* - * ODP CLI Helper Example +/** + * @example odp_cli.c * * This example shows how to start and stop ODP CLI using the CLI helper * API functions. This example application can also be used to try out * the CLI by connecting to a running application with a telnet client. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include diff --git a/example/debug/odp_debug.c b/example/debug/odp_debug.c index a2f322e09..c7cf8b90b 100644 --- a/example/debug/odp_debug.c +++ b/example/debug/odp_debug.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_debug.c + * + * This example application demonstrates the usage of various debug print functions of ODP API. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/example/generator/odp_generator.c b/example/generator/odp_generator.c index 95ba04c1e..e62e46701 100644 --- a/example/generator/odp_generator.c +++ b/example/generator/odp_generator.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_generator.c + * + * Traffic generator and loopback demo application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + /** enable strtok */ #ifndef _GNU_SOURCE #define _GNU_SOURCE diff --git a/example/hello/odp_hello.c b/example/hello/odp_hello.c index 391406946..3516864b8 100644 --- a/example/hello/odp_hello.c +++ b/example/hello/odp_hello.c @@ -4,10 +4,15 @@ * SPDX-License-Identifier: BSD-3-Clause */ -/* This is a minimal application which demonstrates the startup and shutdown +/** + * @example odp_hello.c + * + * This is a minimal application which demonstrates the startup and shutdown * steps of an ODP application. It can be also used to debug API related * build problems, etc. It does not use helpers to minimize dependency to * anything else than the ODP API header file. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include diff --git a/example/ipfragreass/odp_ipfragreass.c b/example/ipfragreass/odp_ipfragreass.c index 55c76d7cc..db2da70ce 100644 --- a/example/ipfragreass/odp_ipfragreass.c +++ b/example/ipfragreass/odp_ipfragreass.c @@ -5,9 +5,11 @@ */ /** - * @file + * @example odp_ipfragreass.c * - * @example odp_ipfragreass.c ODP IPv4 lock-free fragmentation and reassembly + * IPv4 lock-free fragmentation and reassembly example application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include diff --git a/example/ipfragreass/odp_ipfragreass_fragment.c b/example/ipfragreass/odp_ipfragreass_fragment.c index 2cce75e91..85e9180d6 100644 --- a/example/ipfragreass/odp_ipfragreass_fragment.c +++ b/example/ipfragreass/odp_ipfragreass_fragment.c @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #include #include diff --git a/example/ipfragreass/odp_ipfragreass_fragment.h b/example/ipfragreass/odp_ipfragreass_fragment.h index e0e8cfea7..f6be87bf3 100644 --- a/example/ipfragreass/odp_ipfragreass_fragment.h +++ b/example/ipfragreass/odp_ipfragreass_fragment.h @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_FRAGREASS_PP_FRAG_H_ #define ODP_FRAGREASS_PP_FRAG_H_ diff --git a/example/ipfragreass/odp_ipfragreass_helpers.c b/example/ipfragreass/odp_ipfragreass_helpers.c index ac737fe44..a99b6aaa7 100644 --- a/example/ipfragreass/odp_ipfragreass_helpers.c +++ b/example/ipfragreass/odp_ipfragreass_helpers.c @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #include #include diff --git a/example/ipfragreass/odp_ipfragreass_helpers.h b/example/ipfragreass/odp_ipfragreass_helpers.h index af47e326c..e115b3d7f 100644 --- a/example/ipfragreass/odp_ipfragreass_helpers.h +++ b/example/ipfragreass/odp_ipfragreass_helpers.h @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_FRAGREASS_PP_HELPERS_H_ #define ODP_FRAGREASS_PP_HELPERS_H_ diff --git a/example/ipfragreass/odp_ipfragreass_ip.h b/example/ipfragreass/odp_ipfragreass_ip.h index 9c8f2a5f7..d9b64e8c9 100644 --- a/example/ipfragreass/odp_ipfragreass_ip.h +++ b/example/ipfragreass/odp_ipfragreass_ip.h @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_FRAGREASS_PP_IP_H_ #define ODP_FRAGREASS_PP_IP_H_ diff --git a/example/ipfragreass/odp_ipfragreass_reassemble.c b/example/ipfragreass/odp_ipfragreass_reassemble.c index 2542a3b97..d6799c82c 100644 --- a/example/ipfragreass/odp_ipfragreass_reassemble.c +++ b/example/ipfragreass/odp_ipfragreass_reassemble.c @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #include #include diff --git a/example/ipfragreass/odp_ipfragreass_reassemble.h b/example/ipfragreass/odp_ipfragreass_reassemble.h index 1fb71284e..d0ccc2ddb 100644 --- a/example/ipfragreass/odp_ipfragreass_reassemble.h +++ b/example/ipfragreass/odp_ipfragreass_reassemble.h @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_FRAGREASS_PP_REASSEMBLE_H_ #define ODP_FRAGREASS_PP_REASSEMBLE_H_ diff --git a/example/ipsec_api/odp_ipsec.c b/example/ipsec_api/odp_ipsec.c index 524b1ae76..3ac561d90 100644 --- a/example/ipsec_api/odp_ipsec.c +++ b/example/ipsec_api/odp_ipsec.c @@ -5,9 +5,11 @@ */ /** - * @file + * @example ipsec_api/odp_ipsec.c * - * @example odp_example_ipsec.c ODP basic packet IO cross connect with IPsec test application + * IPsec example application using ODP IPsec API + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ /* enable strtok */ diff --git a/example/ipsec_api/odp_ipsec_cache.c b/example/ipsec_api/odp_ipsec_cache.c index c16c71115..d81257569 100644 --- a/example/ipsec_api/odp_ipsec_cache.c +++ b/example/ipsec_api/odp_ipsec_cache.c @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #include #include diff --git a/example/ipsec_api/odp_ipsec_cache.h b/example/ipsec_api/odp_ipsec_cache.h index b0e1fa3f0..2ae29347d 100644 --- a/example/ipsec_api/odp_ipsec_cache.h +++ b/example/ipsec_api/odp_ipsec_cache.h @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_CACHE_H_ #define ODP_IPSEC_CACHE_H_ diff --git a/example/ipsec_crypto/odp_ipsec.c b/example/ipsec_crypto/odp_ipsec.c index 06d52c311..765a96bbe 100644 --- a/example/ipsec_crypto/odp_ipsec.c +++ b/example/ipsec_crypto/odp_ipsec.c @@ -6,9 +6,11 @@ */ /** - * @file + * @example ipsec_crypto/odp_ipsec.c * - * @example odp_example_ipsec.c ODP basic packet IO cross connect with IPsec test application + * IPsec example application using ODP crypto API + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ /* enable strtok */ diff --git a/example/ipsec_crypto/odp_ipsec_cache.c b/example/ipsec_crypto/odp_ipsec_cache.c index 65a51bd1d..b916ce952 100644 --- a/example/ipsec_crypto/odp_ipsec_cache.c +++ b/example/ipsec_crypto/odp_ipsec_cache.c @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #include #include diff --git a/example/ipsec_crypto/odp_ipsec_cache.h b/example/ipsec_crypto/odp_ipsec_cache.h index b1e3e7ac1..e506071e2 100644 --- a/example/ipsec_crypto/odp_ipsec_cache.h +++ b/example/ipsec_crypto/odp_ipsec_cache.h @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_CACHE_H_ #define ODP_IPSEC_CACHE_H_ diff --git a/example/ipsec_crypto/odp_ipsec_fwd_db.c b/example/ipsec_crypto/odp_ipsec_fwd_db.c index 9bd399ca9..71074a46c 100644 --- a/example/ipsec_crypto/odp_ipsec_fwd_db.c +++ b/example/ipsec_crypto/odp_ipsec_fwd_db.c @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + /* enable strtok */ #ifndef _GNU_SOURCE #define _GNU_SOURCE diff --git a/example/ipsec_crypto/odp_ipsec_fwd_db.h b/example/ipsec_crypto/odp_ipsec_fwd_db.h index c6d9c6c99..dc9c66f84 100644 --- a/example/ipsec_crypto/odp_ipsec_fwd_db.h +++ b/example/ipsec_crypto/odp_ipsec_fwd_db.h @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_FWD_DB_H_ #define ODP_IPSEC_FWD_DB_H_ diff --git a/example/ipsec_crypto/odp_ipsec_misc.h b/example/ipsec_crypto/odp_ipsec_misc.h index 69788d115..de9f1ee32 100644 --- a/example/ipsec_crypto/odp_ipsec_misc.h +++ b/example/ipsec_crypto/odp_ipsec_misc.h @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_MISC_H_ #define ODP_IPSEC_MISC_H_ diff --git a/example/ipsec_crypto/odp_ipsec_sa_db.c b/example/ipsec_crypto/odp_ipsec_sa_db.c index 9d3418681..92921cc98 100644 --- a/example/ipsec_crypto/odp_ipsec_sa_db.c +++ b/example/ipsec_crypto/odp_ipsec_sa_db.c @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + /* enable strtok */ #ifndef _GNU_SOURCE #define _GNU_SOURCE diff --git a/example/ipsec_crypto/odp_ipsec_sa_db.h b/example/ipsec_crypto/odp_ipsec_sa_db.h index 72231369f..9e5489b26 100644 --- a/example/ipsec_crypto/odp_ipsec_sa_db.h +++ b/example/ipsec_crypto/odp_ipsec_sa_db.h @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_SA_DB_H_ #define ODP_IPSEC_SA_DB_H_ diff --git a/example/ipsec_crypto/odp_ipsec_sp_db.c b/example/ipsec_crypto/odp_ipsec_sp_db.c index 3ead3b0a3..af0404478 100644 --- a/example/ipsec_crypto/odp_ipsec_sp_db.c +++ b/example/ipsec_crypto/odp_ipsec_sp_db.c @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + /* enable strtok */ #ifndef _GNU_SOURCE #define _GNU_SOURCE diff --git a/example/ipsec_crypto/odp_ipsec_sp_db.h b/example/ipsec_crypto/odp_ipsec_sp_db.h index b71ea9377..e0d3ebfdb 100644 --- a/example/ipsec_crypto/odp_ipsec_sp_db.h +++ b/example/ipsec_crypto/odp_ipsec_sp_db.h @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_SP_DB_H_ #define ODP_IPSEC_SP_DB_H_ diff --git a/example/ipsec_crypto/odp_ipsec_stream.c b/example/ipsec_crypto/odp_ipsec_stream.c index f8bc64eae..1d3419431 100644 --- a/example/ipsec_crypto/odp_ipsec_stream.c +++ b/example/ipsec_crypto/odp_ipsec_stream.c @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + /* enable strtok */ #ifndef _GNU_SOURCE #define _GNU_SOURCE diff --git a/example/ipsec_crypto/odp_ipsec_stream.h b/example/ipsec_crypto/odp_ipsec_stream.h index 2055d3f00..f0cb4f1dd 100644 --- a/example/ipsec_crypto/odp_ipsec_stream.h +++ b/example/ipsec_crypto/odp_ipsec_stream.h @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_STREAM_H_ #define ODP_IPSEC_STREAM_H_ diff --git a/example/l2fwd_simple/odp_l2fwd_simple.c b/example/l2fwd_simple/odp_l2fwd_simple.c index c9ae94c12..d7dad1787 100644 --- a/example/l2fwd_simple/odp_l2fwd_simple.c +++ b/example/l2fwd_simple/odp_l2fwd_simple.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_l2fwd_simple.c + * + * Minimal L2 forwarding example application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/example/l3fwd/odp_l3fwd.c b/example/l3fwd/odp_l3fwd.c index 7d6b15af0..6952ed650 100644 --- a/example/l3fwd/odp_l3fwd.c +++ b/example/l3fwd/odp_l3fwd.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_l3fwd.c + * + * L3 forwarding example application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/example/l3fwd/odp_l3fwd_db.c b/example/l3fwd/odp_l3fwd_db.c index a4e4681f6..2c68d4700 100644 --- a/example/l3fwd/odp_l3fwd_db.c +++ b/example/l3fwd/odp_l3fwd_db.c @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif diff --git a/example/l3fwd/odp_l3fwd_db.h b/example/l3fwd/odp_l3fwd_db.h index b26c67e44..cf8691a2b 100644 --- a/example/l3fwd/odp_l3fwd_db.h +++ b/example/l3fwd/odp_l3fwd_db.h @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef _ODP_L3FWD_DB_H_ #define _ODP_L3FWD_DB_H_ diff --git a/example/l3fwd/odp_l3fwd_lpm.c b/example/l3fwd/odp_l3fwd_lpm.c index 7eb32b5d0..de0b79c6a 100644 --- a/example/l3fwd/odp_l3fwd_lpm.c +++ b/example/l3fwd/odp_l3fwd_lpm.c @@ -3,6 +3,9 @@ * * SPDX-License-Identifier: BSD-3-Clause */ + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif diff --git a/example/l3fwd/odp_l3fwd_lpm.h b/example/l3fwd/odp_l3fwd_lpm.h index 4f2b9ad7f..a3bbf2811 100644 --- a/example/l3fwd/odp_l3fwd_lpm.h +++ b/example/l3fwd/odp_l3fwd_lpm.h @@ -4,6 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef _ODP_L3FWD_LPM_H_ #define _ODP_L3FWD_LPM_H_ diff --git a/example/packet/odp_packet_dump.c b/example/packet/odp_packet_dump.c index 6d00c237e..d08e73cd7 100644 --- a/example/packet/odp_packet_dump.c +++ b/example/packet/odp_packet_dump.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_packet_dump.c + * + * Packet dump example application which prints received packets to terminal + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/example/packet/odp_pktio.c b/example/packet/odp_pktio.c index 191d9fdb6..43037327b 100644 --- a/example/packet/odp_pktio.c +++ b/example/packet/odp_pktio.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_pktio.c + * + * Basic packet IO loopback example application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/example/ping/odp_ping.c b/example/ping/odp_ping.c index dbe453319..97b856895 100644 --- a/example/ping/odp_ping.c +++ b/example/ping/odp_ping.c @@ -2,6 +2,16 @@ * Copyright (c) 2019-2023 Nokia */ +/** + * @example odp_ping.c + * + * This application replies to IPv4 ping requests. It can be used to test + * connectivity with standard ping utility. ARP table needs to be setup manually + * on the sender side as the application does not reply to ARP requests. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/example/simple_pipeline/odp_simple_pipeline.c b/example/simple_pipeline/odp_simple_pipeline.c index b8fb2d0be..177b8bd22 100644 --- a/example/simple_pipeline/odp_simple_pipeline.c +++ b/example/simple_pipeline/odp_simple_pipeline.c @@ -4,6 +4,17 @@ * SPDX-License-Identifier: BSD-3-Clause */ + /** + * @example odp_simple_pipeline.c + * + * Simple pipeline example application which receives packets from one + * interface and passes them through 0-N worker stages before outputting them + * from a second network interface. The RX, worker, and TX stages are connected + * using plain queues and each stage is run on a separate CPU thread. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/example/switch/odp_switch.c b/example/switch/odp_switch.c index b40f5c07c..476cbbae1 100644 --- a/example/switch/odp_switch.c +++ b/example/switch/odp_switch.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_switch.c + * + * Minimal learning Ethernet switch example application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/example/sysinfo/odp_sysinfo.c b/example/sysinfo/odp_sysinfo.c index 151388263..d8a0141c8 100644 --- a/example/sysinfo/odp_sysinfo.c +++ b/example/sysinfo/odp_sysinfo.c @@ -5,6 +5,15 @@ * SPDX-License-Identifier: BSD-3-Clause */ + /** + * @example odp_sysinfo.c + * + * Example application which queries and prints out various system information + * and capabilities which are available through ODP APIs. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif diff --git a/example/time/time_global_test.c b/example/time/time_global_test.c index 2357f25d6..c1d11e5a4 100644 --- a/example/time/time_global_test.c +++ b/example/time/time_global_test.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example time_global_test.c + * + * Time API test application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include diff --git a/example/timer/odp_timer_accuracy.c b/example/timer/odp_timer_accuracy.c index 23c40e66f..fba01be28 100644 --- a/example/timer/odp_timer_accuracy.c +++ b/example/timer/odp_timer_accuracy.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_timer_accuracy.c + * + * ODP timer accuracy test application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/example/timer/odp_timer_simple.c b/example/timer/odp_timer_simple.c index 681f95714..6521313a2 100644 --- a/example/timer/odp_timer_simple.c +++ b/example/timer/odp_timer_simple.c @@ -3,11 +3,13 @@ * * SPDX-License-Identifier: BSD-3-Clause */ + /** - * @file + * @example odp_timer_simple.c + * + * Minimal example application demonstrating ODP timer API usage * - * @example odp_timer_simple.c ODP simple example to schedule timer - * action for 1 second. + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include diff --git a/example/timer/odp_timer_test.c b/example/timer/odp_timer_test.c index 0be53c4cd..e6ebfdcb4 100644 --- a/example/timer/odp_timer_test.c +++ b/example/timer/odp_timer_test.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_timer_test.c + * + * Timer test application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/example/traffic_mgmt/odp_traffic_mgmt.c b/example/traffic_mgmt/odp_traffic_mgmt.c index 5c97c5433..2178b5e2f 100644 --- a/example/traffic_mgmt/odp_traffic_mgmt.c +++ b/example/traffic_mgmt/odp_traffic_mgmt.c @@ -7,6 +7,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ + /** + * @example odp_traffic_mgmt.c + * + * Traffic manager API example application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif diff --git a/test/performance/odp_l2fwd.c b/test/performance/odp_l2fwd.c index 35becb15a..4f4dc4af9 100644 --- a/test/performance/odp_l2fwd.c +++ b/test/performance/odp_l2fwd.c @@ -6,6 +6,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_l2fwd.c + * + * L2 forwarding example application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + /* enable strtok */ #ifndef _GNU_SOURCE #define _GNU_SOURCE From c1b47be9c452f8867fde545f29df55531eb002fe Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 18 Dec 2023 12:10:21 +0200 Subject: [PATCH 006/147] test: performance: improve test application documentations Include short description in Doxygen output for each performance test application. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- doc/application-api-guide/Doxyfile | 6 ++++-- test/performance/bench_common.c | 2 ++ test/performance/bench_common.h | 2 ++ test/performance/dummy_crc.h | 2 ++ test/performance/odp_atomic_perf.c | 8 ++++++++ test/performance/odp_bench_buffer.c | 8 ++++++++ test/performance/odp_bench_misc.c | 8 ++++++++ test/performance/odp_bench_packet.c | 6 ++++-- test/performance/odp_bench_pktio_sp.c | 8 ++++++++ test/performance/odp_bench_timer.c | 8 ++++++++ test/performance/odp_cpu_bench.c | 8 ++++++++ test/performance/odp_crc.c | 8 ++++++++ test/performance/odp_crypto.c | 8 ++++++++ test/performance/odp_dma_perf.c | 4 +++- test/performance/odp_dmafwd.c | 4 +++- test/performance/odp_ipsec.c | 8 ++++++++ test/performance/odp_ipsecfwd.c | 9 +++++++++ test/performance/odp_lock_perf.c | 8 ++++++++ test/performance/odp_mem_perf.c | 8 ++++++++ test/performance/odp_packet_gen.c | 8 ++++++++ test/performance/odp_pktio_ordered.c | 6 ++++-- test/performance/odp_pktio_perf.c | 17 ++++++++++------- test/performance/odp_pool_perf.c | 8 ++++++++ test/performance/odp_queue_perf.c | 8 ++++++++ test/performance/odp_random.c | 8 ++++++++ test/performance/odp_sched_latency.c | 6 ++++-- test/performance/odp_sched_perf.c | 8 ++++++++ test/performance/odp_sched_pktio.c | 8 ++++++++ test/performance/odp_scheduling.c | 6 ++++-- test/performance/odp_stash_perf.c | 8 ++++++++ test/performance/odp_stress.c | 8 ++++++++ test/performance/odp_timer_perf.c | 8 ++++++++ 32 files changed, 211 insertions(+), 19 deletions(-) diff --git a/doc/application-api-guide/Doxyfile b/doc/application-api-guide/Doxyfile index 13a365c72..9c718f203 100644 --- a/doc/application-api-guide/Doxyfile +++ b/doc/application-api-guide/Doxyfile @@ -6,6 +6,8 @@ PROJECT_LOGO = $(SRCDIR)/doc/images/ODP-Logo-HQ.svg INPUT = $(SRCDIR)/doc/application-api-guide \ include \ $(SRCDIR)/include \ - $(SRCDIR)/example -EXAMPLE_PATH = $(SRCDIR)/example $(SRCDIR)/CONTRIBUTING $(SRCDIR)/include/README + $(SRCDIR)/example \ + $(SRCDIR)/test/performance +EXAMPLE_PATH = $(SRCDIR)/example $(SRCDIR)/CONTRIBUTING $(SRCDIR)/include/README \ + $(SRCDIR)/test/performance WARNINGS = NO diff --git a/test/performance/bench_common.c b/test/performance/bench_common.c index f838954ab..640889503 100644 --- a/test/performance/bench_common.c +++ b/test/performance/bench_common.c @@ -2,6 +2,8 @@ * Copyright (c) 2023 Nokia */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #include #include diff --git a/test/performance/bench_common.h b/test/performance/bench_common.h index bd611878d..4b59c941f 100644 --- a/test/performance/bench_common.h +++ b/test/performance/bench_common.h @@ -2,6 +2,8 @@ * Copyright (c) 2023 Nokia */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef BENCH_COMMON_H #define BENCH_COMMON_H diff --git a/test/performance/dummy_crc.h b/test/performance/dummy_crc.h index 68928abee..01e6c2433 100644 --- a/test/performance/dummy_crc.h +++ b/test/performance/dummy_crc.h @@ -37,6 +37,8 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + /** * @file * diff --git a/test/performance/odp_atomic_perf.c b/test/performance/odp_atomic_perf.c index 28217f5d7..b87a5666c 100644 --- a/test/performance/odp_atomic_perf.c +++ b/test/performance/odp_atomic_perf.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_atomic_perf.c + * + * Performance test application for atomic operation APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/test/performance/odp_bench_buffer.c b/test/performance/odp_bench_buffer.c index 8b041eee0..c85340d92 100644 --- a/test/performance/odp_bench_buffer.c +++ b/test/performance/odp_bench_buffer.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_bench_buffer.c + * + * Microbenchmark application for buffer API functions + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include diff --git a/test/performance/odp_bench_misc.c b/test/performance/odp_bench_misc.c index c9c9f0d75..61afdc398 100644 --- a/test/performance/odp_bench_misc.c +++ b/test/performance/odp_bench_misc.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_bench_misc.c + * + * Microbenchmark application for miscellaneous API functions + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE /* Needed for sigaction */ #endif diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c index 67b6b9cfc..57f9ef4b9 100644 --- a/test/performance/odp_bench_packet.c +++ b/test/performance/odp_bench_packet.c @@ -6,9 +6,11 @@ */ /** - * @file + * @example odp_bench_packet.c * - * @example odp_bench_packet.c Microbenchmarks for packet functions + * Microbenchmark application for packet API functions + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include diff --git a/test/performance/odp_bench_pktio_sp.c b/test/performance/odp_bench_pktio_sp.c index 65d85a062..017e7565f 100644 --- a/test/performance/odp_bench_pktio_sp.c +++ b/test/performance/odp_bench_pktio_sp.c @@ -2,6 +2,14 @@ * Copyright (c) 2023 Nokia */ +/** + * @example odp_bench_pktio_sp.c + * + * Microbenchmark application for packet IO slow path functions + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE /* Needed for sigaction */ #endif diff --git a/test/performance/odp_bench_timer.c b/test/performance/odp_bench_timer.c index 78dd9e2d2..65c7a9168 100644 --- a/test/performance/odp_bench_timer.c +++ b/test/performance/odp_bench_timer.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_bench_timer.c + * + * Microbenchmark application for timer API functions + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE /* Needed for sigaction */ #endif diff --git a/test/performance/odp_cpu_bench.c b/test/performance/odp_cpu_bench.c index 7ef12dc30..0ce8aa3fb 100644 --- a/test/performance/odp_cpu_bench.c +++ b/test/performance/odp_cpu_bench.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_cpu_bench.c + * + * Application for CPU stress testing + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include diff --git a/test/performance/odp_crc.c b/test/performance/odp_crc.c index 89e8af837..89e2e971f 100644 --- a/test/performance/odp_crc.c +++ b/test/performance/odp_crc.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_crc.c + * + * Performance test application for CRC hash APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c index 93315ce05..1ae09abab 100644 --- a/test/performance/odp_crypto.c +++ b/test/performance/odp_crypto.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_crypto.c + * + * Performance test application for crypto APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif /* _GNU_SOURCE */ diff --git a/test/performance/odp_dma_perf.c b/test/performance/odp_dma_perf.c index 21c9c0558..f843843c8 100644 --- a/test/performance/odp_dma_perf.c +++ b/test/performance/odp_dma_perf.c @@ -3,12 +3,14 @@ */ /** - * DMA performance tester + * @example odp_dma_perf.c * * This tester application can be used to profile the performance of an ODP DMA implementation. * Tester workflow is simple and consists of issuing as many back-to-back DMA transfers as the * implementation allows and then recording key performance statistics (such as function overhead, * latencies etc.). + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #ifndef _GNU_SOURCE diff --git a/test/performance/odp_dmafwd.c b/test/performance/odp_dmafwd.c index 188a8e358..088fc69e2 100644 --- a/test/performance/odp_dmafwd.c +++ b/test/performance/odp_dmafwd.c @@ -3,12 +3,14 @@ */ /** - * DMA forwarder + * @example odp_dmafwd.c * * This tester application can be used to profile the performance of an ODP DMA implementation. * Tester workflow consists of packet reception, copy and forwarding steps. Packets are first * received from configured interfaces after which packets are copied, either with plain SW memory * copy or with DMA offload copy. Finally, copied packets are echoed back to the sender(s). + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #ifndef _GNU_SOURCE diff --git a/test/performance/odp_ipsec.c b/test/performance/odp_ipsec.c index 8a0bc2989..77437a918 100644 --- a/test/performance/odp_ipsec.c +++ b/test/performance/odp_ipsec.c @@ -6,6 +6,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_ipsec.c + * + * Performance test application for IPsec APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif /* _GNU_SOURCE */ diff --git a/test/performance/odp_ipsecfwd.c b/test/performance/odp_ipsecfwd.c index 5c35d67f7..0220cf6ae 100644 --- a/test/performance/odp_ipsecfwd.c +++ b/test/performance/odp_ipsecfwd.c @@ -2,6 +2,15 @@ * Copyright (c) 2022-2023 Nokia */ +/** + * @example odp_ipsecfwd.c + * + * Simple IPsec performance tester application which forwards and processes + * plain and IPsec packets. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif diff --git a/test/performance/odp_lock_perf.c b/test/performance/odp_lock_perf.c index c12f8c950..0f78db3b8 100644 --- a/test/performance/odp_lock_perf.c +++ b/test/performance/odp_lock_perf.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_lock_perf.c + * + * Performance test application for lock APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/test/performance/odp_mem_perf.c b/test/performance/odp_mem_perf.c index 56a3cdf9a..241128b1f 100644 --- a/test/performance/odp_mem_perf.c +++ b/test/performance/odp_mem_perf.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_mem_perf.c + * + * Test application for measuring memory system bandwidth + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/test/performance/odp_packet_gen.c b/test/performance/odp_packet_gen.c index 247ea2eb6..c88535791 100644 --- a/test/performance/odp_packet_gen.c +++ b/test/performance/odp_packet_gen.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_packet_gen.c + * + * Performance optimized packet generator application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + /* enable usleep */ #ifndef _GNU_SOURCE #define _GNU_SOURCE diff --git a/test/performance/odp_pktio_ordered.c b/test/performance/odp_pktio_ordered.c index bd43ad53d..6177a8160 100644 --- a/test/performance/odp_pktio_ordered.c +++ b/test/performance/odp_pktio_ordered.c @@ -5,9 +5,11 @@ */ /** - * @file + * @example odp_pktio_ordered.c * - * @example odp_pktio_ordered.c ODP ordered pktio test application + * Test application for ordered packet IO + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ /** enable strtok */ diff --git a/test/performance/odp_pktio_perf.c b/test/performance/odp_pktio_perf.c index 06620fd27..4cfeb50cf 100644 --- a/test/performance/odp_pktio_perf.c +++ b/test/performance/odp_pktio_perf.c @@ -2,14 +2,16 @@ * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @example odp_pktio_perf.c * - * ODP Packet IO basic performance test application. - * - * Runs a number of transmit and receive workers on separate cores, the - * transmitters generate packets at a defined rate and the receivers consume - * them. Generated packets are UDP and each packet is marked with a magic - * number in the UDP payload allowing receiver to distinguish them from other - * traffic. + * Packet IO basic performance test application. Runs a number of transmit and + * receive workers on separate cores, the transmitters generate packets at a + * defined rate and the receivers consume them. Generated packets are UDP and + * each packet is marked with a magic number in the UDP payload allowing + * receiver to distinguish them from other traffic. * * Each test iteration runs for a fixed period, at the end of the iteration * it is verified that the number of packets transmitted was as expected and @@ -19,6 +21,7 @@ * determine the maximum rate at which no packet loss occurs. Alternatively * a single packet rate can be specified on the command line. * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include diff --git a/test/performance/odp_pool_perf.c b/test/performance/odp_pool_perf.c index 4ae2cf7d3..43a39a21e 100644 --- a/test/performance/odp_pool_perf.c +++ b/test/performance/odp_pool_perf.c @@ -6,6 +6,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_pool_perf.c + * + * Performance test application for pool APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/test/performance/odp_queue_perf.c b/test/performance/odp_queue_perf.c index 4e4446610..7d4612cb8 100644 --- a/test/performance/odp_queue_perf.c +++ b/test/performance/odp_queue_perf.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_queue_perf.c + * + * Performance test application for queue APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/test/performance/odp_random.c b/test/performance/odp_random.c index 46134ac0c..99714d7b3 100644 --- a/test/performance/odp_random.c +++ b/test/performance/odp_random.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_random.c + * + * Performance test application for random data APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/test/performance/odp_sched_latency.c b/test/performance/odp_sched_latency.c index c8dc74656..0fec49fb9 100644 --- a/test/performance/odp_sched_latency.c +++ b/test/performance/odp_sched_latency.c @@ -6,9 +6,11 @@ */ /** - * @file + * @example odp_sched_latency.c * - * @example odp_sched_latency.c ODP scheduling latency benchmark application + * Scheduling latency benchmark application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c index f89705576..4cd4c3ac4 100644 --- a/test/performance/odp_sched_perf.c +++ b/test/performance/odp_sched_perf.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_sched_perf.c + * + * Performance test application for scheduling + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index 3a85a91a5..d8ab1b279 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_sched_pktio.c + * + * Test application for scheduled packet IO + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/test/performance/odp_scheduling.c b/test/performance/odp_scheduling.c index f9c083c92..86a6be530 100644 --- a/test/performance/odp_scheduling.c +++ b/test/performance/odp_scheduling.c @@ -5,9 +5,11 @@ */ /** - * @file + * @example odp_scheduling.c * - * @example odp_example.c ODP example application + * Performance test application for miscellaneous scheduling operations + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include diff --git a/test/performance/odp_stash_perf.c b/test/performance/odp_stash_perf.c index ffbc92b4b..cb223999e 100644 --- a/test/performance/odp_stash_perf.c +++ b/test/performance/odp_stash_perf.c @@ -4,6 +4,14 @@ * Copyright (c) 2023 Arm */ +/** + * @example odp_stash_perf.c + * + * Performance test application for stash APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/test/performance/odp_stress.c b/test/performance/odp_stress.c index 84bc4fe6c..4fbc303ae 100644 --- a/test/performance/odp_stress.c +++ b/test/performance/odp_stress.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_stress.c + * + * Test application that can be used to stress CPU, memory, and HW accelerators. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include diff --git a/test/performance/odp_timer_perf.c b/test/performance/odp_timer_perf.c index 8632fcb73..16e2cb151 100644 --- a/test/performance/odp_timer_perf.c +++ b/test/performance/odp_timer_perf.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_timer_perf.c + * + * Performance test application for timer APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include #include #include From e176a938e5f6a2bfe0e748204c781c565a9e6f19 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 20 Dec 2023 14:58:35 +0200 Subject: [PATCH 007/147] example: time: add odp_ prefix to time_global_test Rename time_global_test.c to odp_time_global_test.c, so it will be included in Doxygen output. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- example/time/Makefile.am | 2 +- example/time/{time_global_test.c => odp_time_global_test.c} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename example/time/{time_global_test.c => odp_time_global_test.c} (99%) diff --git a/example/time/Makefile.am b/example/time/Makefile.am index 5621ef99e..b5a1ad35d 100644 --- a/example/time/Makefile.am +++ b/example/time/Makefile.am @@ -6,4 +6,4 @@ if test_example TESTS = odp_time_global endif -odp_time_global_SOURCES = time_global_test.c +odp_time_global_SOURCES = odp_time_global_test.c diff --git a/example/time/time_global_test.c b/example/time/odp_time_global_test.c similarity index 99% rename from example/time/time_global_test.c rename to example/time/odp_time_global_test.c index c1d11e5a4..1d788744f 100644 --- a/example/time/time_global_test.c +++ b/example/time/odp_time_global_test.c @@ -5,7 +5,7 @@ */ /** - * @example time_global_test.c + * @example odp_time_global_test.c * * Time API test application * From 17c164b5e2b4b34d942edd3d5f382fcfde8cd741 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 2 Jan 2024 11:08:23 +0200 Subject: [PATCH 008/147] example: update copyright format Update all example files to use the new shorter copyright format: /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) */ Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- example/classifier/odp_classifier.c | 10 ++++------ example/classifier/odp_classifier_run.sh | 6 ++---- example/cli/odp_cli.c | 6 ++---- example/cli/odp_cli_run.sh | 6 ++---- example/debug/odp_debug.c | 6 ++---- example/generator/generator_run.sh | 6 ++---- example/generator/odp_generator.c | 6 ++---- example/hello/odp_hello.c | 6 ++---- example/ipfragreass/odp_ipfragreass.c | 6 ++---- example/ipfragreass/odp_ipfragreass_fragment.c | 6 ++---- example/ipfragreass/odp_ipfragreass_fragment.h | 6 ++---- example/ipfragreass/odp_ipfragreass_helpers.c | 6 ++---- example/ipfragreass/odp_ipfragreass_helpers.h | 6 ++---- example/ipfragreass/odp_ipfragreass_ip.h | 6 ++---- example/ipfragreass/odp_ipfragreass_reassemble.c | 6 ++---- example/ipfragreass/odp_ipfragreass_reassemble.h | 6 ++---- example/ipsec_api/README | 8 +++----- example/ipsec_api/odp_ipsec.c | 6 ++---- example/ipsec_api/odp_ipsec_cache.c | 6 ++---- example/ipsec_api/odp_ipsec_cache.h | 6 ++---- example/ipsec_crypto/README | 6 ++---- example/ipsec_crypto/odp_ipsec.c | 8 +++----- example/ipsec_crypto/odp_ipsec_cache.c | 6 ++---- example/ipsec_crypto/odp_ipsec_cache.h | 6 ++---- example/ipsec_crypto/odp_ipsec_fwd_db.c | 6 ++---- example/ipsec_crypto/odp_ipsec_fwd_db.h | 6 ++---- example/ipsec_crypto/odp_ipsec_misc.h | 6 ++---- example/ipsec_crypto/odp_ipsec_sa_db.c | 6 ++---- example/ipsec_crypto/odp_ipsec_sa_db.h | 6 ++---- example/ipsec_crypto/odp_ipsec_sp_db.c | 6 ++---- example/ipsec_crypto/odp_ipsec_sp_db.h | 6 ++---- example/ipsec_crypto/odp_ipsec_stream.c | 6 ++---- example/ipsec_crypto/odp_ipsec_stream.h | 6 ++---- example/l2fwd_simple/l2fwd_simple_run.sh | 6 ++---- example/l2fwd_simple/odp_l2fwd_simple.c | 6 ++---- example/l3fwd/odp_l3fwd.c | 6 ++---- example/l3fwd/odp_l3fwd_db.c | 6 ++---- example/l3fwd/odp_l3fwd_db.h | 6 ++---- example/l3fwd/odp_l3fwd_lpm.c | 6 ++---- example/l3fwd/odp_l3fwd_lpm.h | 6 ++---- example/l3fwd/odp_l3fwd_run.sh | 6 ++---- example/packet/odp_packet_dump.c | 6 ++---- example/packet/odp_pktio.c | 6 ++---- example/packet/packet_dump_run.sh | 6 ++---- example/packet/pktio_run.sh | 6 ++---- example/ping/ping_run.sh | 6 ++---- example/simple_pipeline/odp_simple_pipeline.c | 6 ++---- example/simple_pipeline/simple_pipeline_run.sh | 6 ++---- example/switch/odp_switch.c | 8 +++----- example/switch/switch_run.sh | 6 ++---- example/sysinfo/odp_sysinfo.c | 8 +++----- example/time/odp_time_global_test.c | 6 ++---- example/timer/odp_timer_accuracy.c | 8 +++----- example/timer/odp_timer_accuracy_run.sh | 6 ++---- example/timer/odp_timer_simple.c | 6 ++---- example/timer/odp_timer_test.c | 6 ++---- example/traffic_mgmt/odp_traffic_mgmt.c | 11 ++++------- 57 files changed, 123 insertions(+), 238 deletions(-) diff --git a/example/classifier/odp_classifier.c b/example/classifier/odp_classifier.c index da92085b0..f4218fd9c 100644 --- a/example/classifier/odp_classifier.c +++ b/example/classifier/odp_classifier.c @@ -1,9 +1,7 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia - * Copyright (c) 2020, Marvell - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2019-2022 Nokia + * Copyright (c) 2020 Marvell */ /** diff --git a/example/classifier/odp_classifier_run.sh b/example/classifier/odp_classifier_run.sh index 2b5fe7ebb..ad0c3a76a 100755 --- a/example/classifier/odp_classifier_run.sh +++ b/example/classifier/odp_classifier_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2020, Marvell -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2020 Marvell # if [ -f ./pktio_env ]; then diff --git a/example/cli/odp_cli.c b/example/cli/odp_cli.c index 55b70c8bb..381fc8a59 100644 --- a/example/cli/odp_cli.c +++ b/example/cli/odp_cli.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2021, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia */ /** diff --git a/example/cli/odp_cli_run.sh b/example/cli/odp_cli_run.sh index 0dc00b793..bb212fffb 100755 --- a/example/cli/odp_cli_run.sh +++ b/example/cli/odp_cli_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2021, Nokia -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2021 Nokia # ./odp_cli${EXEEXT} -t 2 diff --git a/example/debug/odp_debug.c b/example/debug/odp_debug.c index c7cf8b90b..fee1c02ad 100644 --- a/example/debug/odp_debug.c +++ b/example/debug/odp_debug.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2020-2022, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2020-2022 Nokia */ /** diff --git a/example/generator/generator_run.sh b/example/generator/generator_run.sh index 528c1b595..ca69b53bd 100755 --- a/example/generator/generator_run.sh +++ b/example/generator/generator_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2020, Marvell -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2020 Marvell # if [ -f ./pktio_env ]; then diff --git a/example/generator/odp_generator.c b/example/generator/odp_generator.c index e62e46701..6ec19d0ea 100644 --- a/example/generator/odp_generator.c +++ b/example/generator/odp_generator.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** diff --git a/example/hello/odp_hello.c b/example/hello/odp_hello.c index 3516864b8..58cc35502 100644 --- a/example/hello/odp_hello.c +++ b/example/hello/odp_hello.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ /** diff --git a/example/ipfragreass/odp_ipfragreass.c b/example/ipfragreass/odp_ipfragreass.c index db2da70ce..2b7df861e 100644 --- a/example/ipfragreass/odp_ipfragreass.c +++ b/example/ipfragreass/odp_ipfragreass.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ /** diff --git a/example/ipfragreass/odp_ipfragreass_fragment.c b/example/ipfragreass/odp_ipfragreass_fragment.c index 85e9180d6..2e9d9ad64 100644 --- a/example/ipfragreass/odp_ipfragreass_fragment.c +++ b/example/ipfragreass/odp_ipfragreass_fragment.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipfragreass/odp_ipfragreass_fragment.h b/example/ipfragreass/odp_ipfragreass_fragment.h index f6be87bf3..748f84186 100644 --- a/example/ipfragreass/odp_ipfragreass_fragment.h +++ b/example/ipfragreass/odp_ipfragreass_fragment.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipfragreass/odp_ipfragreass_helpers.c b/example/ipfragreass/odp_ipfragreass_helpers.c index a99b6aaa7..a48157fcf 100644 --- a/example/ipfragreass/odp_ipfragreass_helpers.c +++ b/example/ipfragreass/odp_ipfragreass_helpers.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipfragreass/odp_ipfragreass_helpers.h b/example/ipfragreass/odp_ipfragreass_helpers.h index e115b3d7f..9f89ca4c4 100644 --- a/example/ipfragreass/odp_ipfragreass_helpers.h +++ b/example/ipfragreass/odp_ipfragreass_helpers.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipfragreass/odp_ipfragreass_ip.h b/example/ipfragreass/odp_ipfragreass_ip.h index d9b64e8c9..97e68324c 100644 --- a/example/ipfragreass/odp_ipfragreass_ip.h +++ b/example/ipfragreass/odp_ipfragreass_ip.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipfragreass/odp_ipfragreass_reassemble.c b/example/ipfragreass/odp_ipfragreass_reassemble.c index d6799c82c..55ee42504 100644 --- a/example/ipfragreass/odp_ipfragreass_reassemble.c +++ b/example/ipfragreass/odp_ipfragreass_reassemble.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipfragreass/odp_ipfragreass_reassemble.h b/example/ipfragreass/odp_ipfragreass_reassemble.h index d0ccc2ddb..b00f3825e 100644 --- a/example/ipfragreass/odp_ipfragreass_reassemble.h +++ b/example/ipfragreass/odp_ipfragreass_reassemble.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipsec_api/README b/example/ipsec_api/README index f7ea93e59..7a392a728 100644 --- a/example/ipsec_api/README +++ b/example/ipsec_api/README @@ -1,8 +1,6 @@ -Copyright (c) 2014-2018, Linaro Limited -Copyright (c) 2020, Nokia -All rights reserved. - -SPDX-License-Identifier: BSD-3-Clause +SPDX-License-Identifier: BSD-3-Clause +Copyright (c) 2014-2018 Linaro Limited +Copyright (c) 2020 Nokia 1. Intro diff --git a/example/ipsec_api/odp_ipsec.c b/example/ipsec_api/odp_ipsec.c index 3ac561d90..a67a3a1e8 100644 --- a/example/ipsec_api/odp_ipsec.c +++ b/example/ipsec_api/odp_ipsec.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2013-2018 Linaro Limited */ /** diff --git a/example/ipsec_api/odp_ipsec_cache.c b/example/ipsec_api/odp_ipsec_cache.c index d81257569..827c9dce2 100644 --- a/example/ipsec_api/odp_ipsec_cache.c +++ b/example/ipsec_api/odp_ipsec_cache.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipsec_api/odp_ipsec_cache.h b/example/ipsec_api/odp_ipsec_cache.h index 2ae29347d..5dd0c80b3 100644 --- a/example/ipsec_api/odp_ipsec_cache.h +++ b/example/ipsec_api/odp_ipsec_cache.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipsec_crypto/README b/example/ipsec_crypto/README index 8e66372b5..0411396a1 100644 --- a/example/ipsec_crypto/README +++ b/example/ipsec_crypto/README @@ -1,7 +1,5 @@ -Copyright (c) 2014-2018, Linaro Limited -All rights reserved. - -SPDX-License-Identifier: BSD-3-Clause +SPDX-License-Identifier: BSD-3-Clause +Copyright (c) 2014-2018 Linaro Limited 1. Intro diff --git a/example/ipsec_crypto/odp_ipsec.c b/example/ipsec_crypto/odp_ipsec.c index 765a96bbe..31c866b56 100644 --- a/example/ipsec_crypto/odp_ipsec.c +++ b/example/ipsec_crypto/odp_ipsec.c @@ -1,8 +1,6 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2013-2018 Linaro Limited + * Copyright (c) 2021-2022 Nokia */ /** diff --git a/example/ipsec_crypto/odp_ipsec_cache.c b/example/ipsec_crypto/odp_ipsec_cache.c index b916ce952..0461d4fbf 100644 --- a/example/ipsec_crypto/odp_ipsec_cache.c +++ b/example/ipsec_crypto/odp_ipsec_cache.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipsec_crypto/odp_ipsec_cache.h b/example/ipsec_crypto/odp_ipsec_cache.h index e506071e2..29c1b983a 100644 --- a/example/ipsec_crypto/odp_ipsec_cache.h +++ b/example/ipsec_crypto/odp_ipsec_cache.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipsec_crypto/odp_ipsec_fwd_db.c b/example/ipsec_crypto/odp_ipsec_fwd_db.c index 71074a46c..292d9c7c6 100644 --- a/example/ipsec_crypto/odp_ipsec_fwd_db.c +++ b/example/ipsec_crypto/odp_ipsec_fwd_db.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipsec_crypto/odp_ipsec_fwd_db.h b/example/ipsec_crypto/odp_ipsec_fwd_db.h index dc9c66f84..d9c84b29b 100644 --- a/example/ipsec_crypto/odp_ipsec_fwd_db.h +++ b/example/ipsec_crypto/odp_ipsec_fwd_db.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipsec_crypto/odp_ipsec_misc.h b/example/ipsec_crypto/odp_ipsec_misc.h index de9f1ee32..921c4e3c0 100644 --- a/example/ipsec_crypto/odp_ipsec_misc.h +++ b/example/ipsec_crypto/odp_ipsec_misc.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipsec_crypto/odp_ipsec_sa_db.c b/example/ipsec_crypto/odp_ipsec_sa_db.c index 92921cc98..9ddbc1152 100644 --- a/example/ipsec_crypto/odp_ipsec_sa_db.c +++ b/example/ipsec_crypto/odp_ipsec_sa_db.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipsec_crypto/odp_ipsec_sa_db.h b/example/ipsec_crypto/odp_ipsec_sa_db.h index 9e5489b26..83f103af3 100644 --- a/example/ipsec_crypto/odp_ipsec_sa_db.h +++ b/example/ipsec_crypto/odp_ipsec_sa_db.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipsec_crypto/odp_ipsec_sp_db.c b/example/ipsec_crypto/odp_ipsec_sp_db.c index af0404478..956fb212d 100644 --- a/example/ipsec_crypto/odp_ipsec_sp_db.c +++ b/example/ipsec_crypto/odp_ipsec_sp_db.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipsec_crypto/odp_ipsec_sp_db.h b/example/ipsec_crypto/odp_ipsec_sp_db.h index e0d3ebfdb..ac6c0c896 100644 --- a/example/ipsec_crypto/odp_ipsec_sp_db.h +++ b/example/ipsec_crypto/odp_ipsec_sp_db.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipsec_crypto/odp_ipsec_stream.c b/example/ipsec_crypto/odp_ipsec_stream.c index 1d3419431..330b4cc54 100644 --- a/example/ipsec_crypto/odp_ipsec_stream.c +++ b/example/ipsec_crypto/odp_ipsec_stream.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/ipsec_crypto/odp_ipsec_stream.h b/example/ipsec_crypto/odp_ipsec_stream.h index f0cb4f1dd..8e1e936dd 100644 --- a/example/ipsec_crypto/odp_ipsec_stream.h +++ b/example/ipsec_crypto/odp_ipsec_stream.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/l2fwd_simple/l2fwd_simple_run.sh b/example/l2fwd_simple/l2fwd_simple_run.sh index b44e10ae1..741240e27 100755 --- a/example/l2fwd_simple/l2fwd_simple_run.sh +++ b/example/l2fwd_simple/l2fwd_simple_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2016-2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2016-2018 Linaro Limited # if [ -f ./pktio_env ]; then diff --git a/example/l2fwd_simple/odp_l2fwd_simple.c b/example/l2fwd_simple/odp_l2fwd_simple.c index d7dad1787..5d9aaec87 100644 --- a/example/l2fwd_simple/odp_l2fwd_simple.c +++ b/example/l2fwd_simple/odp_l2fwd_simple.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ /** diff --git a/example/l3fwd/odp_l3fwd.c b/example/l3fwd/odp_l3fwd.c index 6952ed650..c74c7d4d6 100644 --- a/example/l3fwd/odp_l3fwd.c +++ b/example/l3fwd/odp_l3fwd.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ /** diff --git a/example/l3fwd/odp_l3fwd_db.c b/example/l3fwd/odp_l3fwd_db.c index 2c68d4700..7d82695be 100644 --- a/example/l3fwd/odp_l3fwd_db.c +++ b/example/l3fwd/odp_l3fwd_db.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/l3fwd/odp_l3fwd_db.h b/example/l3fwd/odp_l3fwd_db.h index cf8691a2b..5c9d63e6e 100644 --- a/example/l3fwd/odp_l3fwd_db.h +++ b/example/l3fwd/odp_l3fwd_db.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/l3fwd/odp_l3fwd_lpm.c b/example/l3fwd/odp_l3fwd_lpm.c index de0b79c6a..41572d179 100644 --- a/example/l3fwd/odp_l3fwd_lpm.c +++ b/example/l3fwd/odp_l3fwd_lpm.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/l3fwd/odp_l3fwd_lpm.h b/example/l3fwd/odp_l3fwd_lpm.h index a3bbf2811..bc3382532 100644 --- a/example/l3fwd/odp_l3fwd_lpm.h +++ b/example/l3fwd/odp_l3fwd_lpm.h @@ -1,7 +1,5 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ diff --git a/example/l3fwd/odp_l3fwd_run.sh b/example/l3fwd/odp_l3fwd_run.sh index 89be1510f..0b39d5a52 100755 --- a/example/l3fwd/odp_l3fwd_run.sh +++ b/example/l3fwd/odp_l3fwd_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2016-2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2016-2018 Linaro Limited # if [ -f ./pktio_env ]; then diff --git a/example/packet/odp_packet_dump.c b/example/packet/odp_packet_dump.c index d08e73cd7..dc9f49dd6 100644 --- a/example/packet/odp_packet_dump.c +++ b/example/packet/odp_packet_dump.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018 Linaro Limited */ /** diff --git a/example/packet/odp_pktio.c b/example/packet/odp_pktio.c index 43037327b..6d66185e0 100644 --- a/example/packet/odp_pktio.c +++ b/example/packet/odp_pktio.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2013-2018 Linaro Limited */ /** diff --git a/example/packet/packet_dump_run.sh b/example/packet/packet_dump_run.sh index 4e7861b1c..fea5d8b3c 100755 --- a/example/packet/packet_dump_run.sh +++ b/example/packet/packet_dump_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2018 Linaro Limited # if [ -f ./pktio_env ]; then diff --git a/example/packet/pktio_run.sh b/example/packet/pktio_run.sh index 33115b41f..bb1854b43 100755 --- a/example/packet/pktio_run.sh +++ b/example/packet/pktio_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2016-2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2016-2018 Linaro Limited # if [ -f ./pktio_env ]; then diff --git a/example/ping/ping_run.sh b/example/ping/ping_run.sh index 0017c4ce0..9598db70a 100755 --- a/example/ping/ping_run.sh +++ b/example/ping/ping_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2019, Nokia -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2019 Nokia # if [ -f ./pktio_env ]; then diff --git a/example/simple_pipeline/odp_simple_pipeline.c b/example/simple_pipeline/odp_simple_pipeline.c index 177b8bd22..ad80bc394 100644 --- a/example/simple_pipeline/odp_simple_pipeline.c +++ b/example/simple_pipeline/odp_simple_pipeline.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2019, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2019 Nokia */ /** diff --git a/example/simple_pipeline/simple_pipeline_run.sh b/example/simple_pipeline/simple_pipeline_run.sh index f9248c42d..ba66e506e 100755 --- a/example/simple_pipeline/simple_pipeline_run.sh +++ b/example/simple_pipeline/simple_pipeline_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2019, Nokia -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2019 Nokia # # Exit code expected by automake for skipped tests diff --git a/example/switch/odp_switch.c b/example/switch/odp_switch.c index 476cbbae1..bd987a61f 100644 --- a/example/switch/odp_switch.c +++ b/example/switch/odp_switch.c @@ -1,8 +1,6 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2020, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited + * Copyright (c) 2020 Nokia */ /** diff --git a/example/switch/switch_run.sh b/example/switch/switch_run.sh index 489b17de5..c01c505ba 100755 --- a/example/switch/switch_run.sh +++ b/example/switch/switch_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2016-2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2016-2018 Linaro Limited # RETVAL=0 diff --git a/example/sysinfo/odp_sysinfo.c b/example/sysinfo/odp_sysinfo.c index d8a0141c8..11b789254 100644 --- a/example/sysinfo/odp_sysinfo.c +++ b/example/sysinfo/odp_sysinfo.c @@ -1,8 +1,6 @@ -/* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2022-2023, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018 Linaro Limited + * Copyright (c) 2022-2023 Nokia */ /** diff --git a/example/time/odp_time_global_test.c b/example/time/odp_time_global_test.c index 1d788744f..7c1409abf 100644 --- a/example/time/odp_time_global_test.c +++ b/example/time/odp_time_global_test.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited */ /** diff --git a/example/timer/odp_timer_accuracy.c b/example/timer/odp_timer_accuracy.c index fba01be28..347ad7697 100644 --- a/example/timer/odp_timer_accuracy.c +++ b/example/timer/odp_timer_accuracy.c @@ -1,8 +1,6 @@ -/* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2019-2023, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018 Linaro Limited + * Copyright (c) 2019-2023 Nokia */ /** diff --git a/example/timer/odp_timer_accuracy_run.sh b/example/timer/odp_timer_accuracy_run.sh index 3b29191c4..1c879f4e6 100755 --- a/example/timer/odp_timer_accuracy_run.sh +++ b/example/timer/odp_timer_accuracy_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2022, Nokia -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2022 Nokia # ./odp_timer_accuracy${EXEEXT} -p 100000000 -n 10 diff --git a/example/timer/odp_timer_simple.c b/example/timer/odp_timer_simple.c index 6521313a2..e3afed5be 100644 --- a/example/timer/odp_timer_simple.c +++ b/example/timer/odp_timer_simple.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ /** diff --git a/example/timer/odp_timer_test.c b/example/timer/odp_timer_test.c index e6ebfdcb4..e96b4a87b 100644 --- a/example/timer/odp_timer_test.c +++ b/example/timer/odp_timer_test.c @@ -1,7 +1,5 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2013-2018 Linaro Limited */ /** diff --git a/example/traffic_mgmt/odp_traffic_mgmt.c b/example/traffic_mgmt/odp_traffic_mgmt.c index 2178b5e2f..3ca70b950 100644 --- a/example/traffic_mgmt/odp_traffic_mgmt.c +++ b/example/traffic_mgmt/odp_traffic_mgmt.c @@ -1,10 +1,7 @@ -/* Copyright 2015 EZchip Semiconductor Ltd. All Rights Reserved. - * - * Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2022, Marvell - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015 EZchip Semiconductor Ltd. + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2022 Marvell */ /** From f0cb71a927615ed53ef07ae36c23450407234547 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 5 Jan 2024 15:43:49 +0200 Subject: [PATCH 009/147] validation: pool: fix pool statistics test Values of per thread counters (odp_pool_stats_t.thread.cache_available[]) are undefined if these counters are not supported/enabled. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- test/validation/api/pool/pool.c | 65 ++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/test/validation/api/pool/pool.c b/test/validation/api/pool/pool.c index 6f9b054ac..ee8aa4b67 100644 --- a/test/validation/api/pool/pool.c +++ b/test/validation/api/pool/pool.c @@ -1446,8 +1446,10 @@ static void pool_test_pool_statistics(odp_pool_type_t pool_type) CU_ASSERT(stats.thread.first == first); CU_ASSERT(stats.thread.last == last); - for (j = 0; j < num_thr; j++) - CU_ASSERT(stats.thread.cache_available[j] <= stats.cache_available); + if (supported.bit.thread_cache_available) { + for (j = 0; j < num_thr; j++) + CU_ASSERT(stats.thread.cache_available[j] <= stats.cache_available); + } /* Allocate the events */ for (j = 0; j < num_allocs; j++) { @@ -1493,29 +1495,31 @@ static void pool_test_pool_statistics(odp_pool_type_t pool_type) if (supported.bit.cache_available) CU_ASSERT(selected.cache_available <= num_obj - num_events); - while (first_id < odp_thread_count_max()) { - memset(&stats, 0xff, sizeof(odp_pool_stats_t)); - - stats.thread.first = first_id; - stats.thread.last = last_id; - num_thr = last_id - first_id + 1; - CU_ASSERT_FATAL(odp_pool_stats(pool[i], &stats) == 0); - - for (uint32_t k = 0; k < num_thr; k++) { - uint64_t cached = stats.thread.cache_available[k]; - - CU_ASSERT(cached <= num_obj - num_events); - total_cached += cached; - } - first_id = last_id + 1; - last_id += ODP_POOL_MAX_THREAD_STATS; - if (last_id >= odp_thread_count_max()) - last_id = odp_thread_count_max() - 1; - }; - - if (supported.bit.cache_available && supported.bit.thread_cache_available && - ODP_POOL_MAX_THREAD_STATS >= odp_thread_count_max()) - CU_ASSERT(stats.cache_available == total_cached); + if (supported.bit.thread_cache_available) { + while (first_id < odp_thread_count_max()) { + memset(&stats, 0xff, sizeof(odp_pool_stats_t)); + + stats.thread.first = first_id; + stats.thread.last = last_id; + num_thr = last_id - first_id + 1; + CU_ASSERT_FATAL(odp_pool_stats(pool[i], &stats) == 0); + + for (uint32_t k = 0; k < num_thr; k++) { + uint64_t cached = stats.thread.cache_available[k]; + + CU_ASSERT(cached <= num_obj - num_events); + total_cached += cached; + } + first_id = last_id + 1; + last_id += ODP_POOL_MAX_THREAD_STATS; + if (last_id >= odp_thread_count_max()) + last_id = odp_thread_count_max() - 1; + }; + + if (supported.bit.cache_available && + ODP_POOL_MAX_THREAD_STATS >= odp_thread_count_max()) + CU_ASSERT(stats.cache_available == total_cached); + } } CU_ASSERT(num_events == num_obj); @@ -1541,8 +1545,10 @@ static void pool_test_pool_statistics(odp_pool_type_t pool_type) CU_ASSERT(stats.cache_available == 0); if (supported.bit.cache_available) CU_ASSERT(selected.cache_available == 0); - for (j = 0; j < num_thr; j++) - CU_ASSERT(stats.thread.cache_available[j] == 0); + if (supported.bit.thread_cache_available) { + for (j = 0; j < num_thr; j++) + CU_ASSERT(stats.thread.cache_available[j] == 0); + } if (supported.bit.alloc_ops) { CU_ASSERT(stats.alloc_ops > 0 && stats.alloc_ops <= num_allocs); CU_ASSERT(selected.alloc_ops > 0 && selected.alloc_ops <= num_allocs); @@ -1600,8 +1606,9 @@ static void pool_test_pool_statistics(odp_pool_type_t pool_type) printf(" cache_available: %" PRIu64 "\n", stats.cache_available); printf(" cache_alloc_ops: %" PRIu64 "\n", stats.cache_alloc_ops); printf(" cache_free_ops: %" PRIu64 "\n", stats.cache_free_ops); - printf(" thread.cache_available[0]: %" PRIu64 "\n", - stats.thread.cache_available[0]); + if (supported.bit.thread_cache_available) + printf(" thread.cache_available[0]: %" PRIu64 "\n", + stats.thread.cache_available[0]); } CU_ASSERT_FATAL(odp_pool_stats_reset(pool[i]) == 0); From d3a12ff91e3e7b67851edd628fa0fff4a07ee113 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 4 Jan 2024 08:44:48 +0200 Subject: [PATCH 010/147] api: fix doxygen documentation grouping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, all types and defines from ABI headers were listed as functions in Doxygen HTML output. No actual API changes. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- doc/implementers-guide/implementers-guide.adoc | 2 +- include/odp/api/abi-default/align.h | 2 +- include/odp/api/abi-default/buffer_types.h | 2 +- include/odp/api/abi-default/classification.h | 2 +- include/odp/api/abi-default/comp.h | 2 +- include/odp/api/abi-default/crypto_types.h | 2 +- include/odp/api/abi-default/dma_types.h | 2 +- include/odp/api/abi-default/event_types.h | 2 +- include/odp/api/abi-default/ipsec_types.h | 2 +- include/odp/api/abi-default/packet_io_types.h | 1 - include/odp/api/abi-default/packet_types.h | 2 +- include/odp/api/abi-default/pool_types.h | 2 +- include/odp/api/abi-default/proto_stats_types.h | 3 +-- include/odp/api/abi-default/queue_types.h | 2 +- include/odp/api/abi-default/shared_memory.h | 2 +- include/odp/api/abi-default/stash_types.h | 2 +- include/odp/api/abi-default/thread_types.h | 2 +- include/odp/api/abi-default/time_types.h | 2 +- include/odp/api/spec/buffer.h | 2 +- include/odp/api/spec/buffer_types.h | 2 +- include/odp/api/spec/crypto.h | 2 +- include/odp/api/spec/crypto_types.h | 2 +- include/odp/api/spec/debug.h | 8 ++++++++ include/odp/api/spec/event.h | 2 +- include/odp/api/spec/event_types.h | 2 +- include/odp/api/spec/ipsec.h | 2 +- include/odp/api/spec/ipsec_types.h | 2 +- include/odp/api/spec/packet.h | 2 +- include/odp/api/spec/packet_io.h | 2 +- include/odp/api/spec/packet_io_types.h | 2 +- include/odp/api/spec/packet_types.h | 2 +- include/odp/api/spec/pool.h | 2 +- include/odp/api/spec/pool_types.h | 2 +- include/odp/api/spec/proto_stats.h | 2 +- include/odp/api/spec/proto_stats_types.h | 7 ++++++- include/odp/api/spec/queue.h | 2 +- include/odp/api/spec/queue_types.h | 2 +- include/odp/api/spec/random.h | 2 +- include/odp/api/spec/random_types.h | 2 +- include/odp/api/spec/schedule.h | 2 +- include/odp/api/spec/schedule_types.h | 2 +- include/odp/api/spec/stash.h | 2 +- include/odp/api/spec/stash_types.h | 2 +- include/odp/api/spec/std.h | 10 ++++------ include/odp/api/spec/std_types.h | 2 +- include/odp/api/spec/thread.h | 2 +- include/odp/api/spec/thread_types.h | 2 +- include/odp/api/spec/time.h | 2 +- include/odp/api/spec/time_types.h | 2 +- 49 files changed, 63 insertions(+), 54 deletions(-) diff --git a/doc/implementers-guide/implementers-guide.adoc b/doc/implementers-guide/implementers-guide.adoc index 7b234ecea..c20f04906 100644 --- a/doc/implementers-guide/implementers-guide.adoc +++ b/doc/implementers-guide/implementers-guide.adoc @@ -208,7 +208,7 @@ Within the platform agnostic area, the validation tests for a given interface are also grouped by modules, matching the ODP interface modules: for instance, `test/common_plat/validation/api` mainly contains a list of directories matching each module name (as defined by the doxygen `@defgroup` -or `@ingroup` statement present in each API `.h` file). +or `@addtogroup` statement present in each API `.h` file). Within each of these directories, a library (called `libtest.la`) and its associated `.h` file (called `.h`) defines all the test functions diff --git a/include/odp/api/abi-default/align.h b/include/odp/api/abi-default/align.h index 0fa058549..fa95d728b 100644 --- a/include/odp/api/abi-default/align.h +++ b/include/odp/api/abi-default/align.h @@ -17,7 +17,7 @@ extern "C" { #include -/** @ingroup odp_compiler_optim +/** @addtogroup odp_compiler_optim * @{ */ diff --git a/include/odp/api/abi-default/buffer_types.h b/include/odp/api/abi-default/buffer_types.h index 59588224f..9179ae321 100644 --- a/include/odp/api/abi-default/buffer_types.h +++ b/include/odp/api/abi-default/buffer_types.h @@ -13,7 +13,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_buffer_t; -/** @ingroup odp_buffer +/** @addtogroup odp_buffer * @{ */ diff --git a/include/odp/api/abi-default/classification.h b/include/odp/api/abi-default/classification.h index e7519329a..fdc98f252 100644 --- a/include/odp/api/abi-default/classification.h +++ b/include/odp/api/abi-default/classification.h @@ -15,7 +15,7 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_cos_t; /** Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_pmr_t; -/** @ingroup odp_classification +/** @addtogroup odp_classification * @{ */ diff --git a/include/odp/api/abi-default/comp.h b/include/odp/api/abi-default/comp.h index b5638eba8..3f936aa20 100644 --- a/include/odp/api/abi-default/comp.h +++ b/include/odp/api/abi-default/comp.h @@ -14,7 +14,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_comp_session_t; -/** @ingroup odp_compression +/** @addtogroup odp_compression * @{ */ diff --git a/include/odp/api/abi-default/crypto_types.h b/include/odp/api/abi-default/crypto_types.h index 58898dfea..8d860b6ef 100644 --- a/include/odp/api/abi-default/crypto_types.h +++ b/include/odp/api/abi-default/crypto_types.h @@ -12,7 +12,7 @@ extern "C" { #include -/** @ingroup odp_crypto +/** @addtogroup odp_crypto * @{ */ diff --git a/include/odp/api/abi-default/dma_types.h b/include/odp/api/abi-default/dma_types.h index 1d27a11aa..005ba3d16 100644 --- a/include/odp/api/abi-default/dma_types.h +++ b/include/odp/api/abi-default/dma_types.h @@ -17,7 +17,7 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_dma_t; /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_dma_compl_t; -/** @ingroup odp_dma +/** @addtogroup odp_dma * @{ */ diff --git a/include/odp/api/abi-default/event_types.h b/include/odp/api/abi-default/event_types.h index d6231a98f..7169a3bd4 100644 --- a/include/odp/api/abi-default/event_types.h +++ b/include/odp/api/abi-default/event_types.h @@ -15,7 +15,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_event_t; -/** @ingroup odp_event +/** @addtogroup odp_event * @{ */ diff --git a/include/odp/api/abi-default/ipsec_types.h b/include/odp/api/abi-default/ipsec_types.h index 737f67153..9d099b80d 100644 --- a/include/odp/api/abi-default/ipsec_types.h +++ b/include/odp/api/abi-default/ipsec_types.h @@ -15,7 +15,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_ipsec_sa_t; -/** @ingroup odp_ipsec +/** @addtogroup odp_ipsec * @{ */ diff --git a/include/odp/api/abi-default/packet_io_types.h b/include/odp/api/abi-default/packet_io_types.h index ddf8c3a3f..1aa1cf816 100644 --- a/include/odp/api/abi-default/packet_io_types.h +++ b/include/odp/api/abi-default/packet_io_types.h @@ -25,7 +25,6 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_pktio_t; typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_lso_profile_t; /** @addtogroup odp_packet_io - * Operations on a packet. * @{ */ diff --git a/include/odp/api/abi-default/packet_types.h b/include/odp/api/abi-default/packet_types.h index 300eaf177..e8b2c8484 100644 --- a/include/odp/api/abi-default/packet_types.h +++ b/include/odp/api/abi-default/packet_types.h @@ -27,7 +27,7 @@ typedef struct { char dummy; /**< *internal Dummy */ } _odp_abi_packet_vector_t; /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< *internal Dummy */ } _odp_abi_packet_tx_compl_t; -/** @ingroup odp_packet +/** @addtogroup odp_packet * @{ */ diff --git a/include/odp/api/abi-default/pool_types.h b/include/odp/api/abi-default/pool_types.h index e4ca40422..ce1042c12 100644 --- a/include/odp/api/abi-default/pool_types.h +++ b/include/odp/api/abi-default/pool_types.h @@ -13,7 +13,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_pool_t; -/** @ingroup odp_pool +/** @addtogroup odp_pool * @{ */ diff --git a/include/odp/api/abi-default/proto_stats_types.h b/include/odp/api/abi-default/proto_stats_types.h index 0d87012f3..e17adf886 100644 --- a/include/odp/api/abi-default/proto_stats_types.h +++ b/include/odp/api/abi-default/proto_stats_types.h @@ -15,8 +15,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_proto_stats_t; -/** @ingroup odp_proto_stats - * Operations on a proto stats object. +/** @addtogroup odp_proto_stats * @{ */ diff --git a/include/odp/api/abi-default/queue_types.h b/include/odp/api/abi-default/queue_types.h index 5a1dc40d9..677348c18 100644 --- a/include/odp/api/abi-default/queue_types.h +++ b/include/odp/api/abi-default/queue_types.h @@ -13,7 +13,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_queue_t; -/** @ingroup odp_queue +/** @addtogroup odp_queue * @{ */ diff --git a/include/odp/api/abi-default/shared_memory.h b/include/odp/api/abi-default/shared_memory.h index f4930da27..70d6e906f 100644 --- a/include/odp/api/abi-default/shared_memory.h +++ b/include/odp/api/abi-default/shared_memory.h @@ -12,7 +12,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_shm_t; -/** @ingroup odp_shared_memory +/** @addtogroup odp_shared_memory * @{ */ diff --git a/include/odp/api/abi-default/stash_types.h b/include/odp/api/abi-default/stash_types.h index 10db242d3..6779f3af6 100644 --- a/include/odp/api/abi-default/stash_types.h +++ b/include/odp/api/abi-default/stash_types.h @@ -12,7 +12,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_stash_t; -/** @ingroup odp_stash +/** @addtogroup odp_stash * @{ */ diff --git a/include/odp/api/abi-default/thread_types.h b/include/odp/api/abi-default/thread_types.h index 1511f488d..d8c27fb98 100644 --- a/include/odp/api/abi-default/thread_types.h +++ b/include/odp/api/abi-default/thread_types.h @@ -9,7 +9,7 @@ extern "C" { #endif -/** @ingroup odp_thread +/** @addtogroup odp_thread * @{ */ diff --git a/include/odp/api/abi-default/time_types.h b/include/odp/api/abi-default/time_types.h index 32d9384dd..afbe6d188 100644 --- a/include/odp/api/abi-default/time_types.h +++ b/include/odp/api/abi-default/time_types.h @@ -11,7 +11,7 @@ extern "C" { #include -/** @ingroup odp_time +/** @addtogroup odp_time * @{ **/ diff --git a/include/odp/api/spec/buffer.h b/include/odp/api/spec/buffer.h index 11750136d..5ce2355b8 100644 --- a/include/odp/api/spec/buffer.h +++ b/include/odp/api/spec/buffer.h @@ -22,7 +22,7 @@ extern "C" { #include #include -/** @defgroup odp_buffer ODP BUFFER +/** @addtogroup odp_buffer * Buffer event metadata and operations. * @{ */ diff --git a/include/odp/api/spec/buffer_types.h b/include/odp/api/spec/buffer_types.h index 7b0e80584..7307e72f7 100644 --- a/include/odp/api/spec/buffer_types.h +++ b/include/odp/api/spec/buffer_types.h @@ -17,7 +17,7 @@ extern "C" { #endif -/** @addtogroup odp_buffer +/** @defgroup odp_buffer ODP BUFFER * @{ */ diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index 49bc29ee4..8b7d1df54 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -21,7 +21,7 @@ extern "C" { #endif -/** @defgroup odp_crypto ODP CRYPTO +/** @addtogroup odp_crypto * Data ciphering and authentication. * @{ */ diff --git a/include/odp/api/spec/crypto_types.h b/include/odp/api/spec/crypto_types.h index 31214c0e9..579022762 100644 --- a/include/odp/api/spec/crypto_types.h +++ b/include/odp/api/spec/crypto_types.h @@ -20,7 +20,7 @@ extern "C" { #endif -/** @addtogroup odp_crypto +/** @defgroup odp_crypto ODP CRYPTO * @{ */ diff --git a/include/odp/api/spec/debug.h b/include/odp/api/spec/debug.h index 0cf5179a8..4a3365a3e 100644 --- a/include/odp/api/spec/debug.h +++ b/include/odp/api/spec/debug.h @@ -15,6 +15,10 @@ extern "C" { #endif +/** @addtogroup odp_initialization + * @{ + */ + /** * @def ODP_STATIC_ASSERT * Compile time assertion macro. Fails compilation and outputs message 'msg' @@ -26,6 +30,10 @@ extern "C" { * @param msg Compile time error message to be displayed if cond is false */ +/** + * @} + */ + #ifdef __cplusplus } #endif diff --git a/include/odp/api/spec/event.h b/include/odp/api/spec/event.h index 220c955c3..69464125b 100644 --- a/include/odp/api/spec/event.h +++ b/include/odp/api/spec/event.h @@ -21,7 +21,7 @@ extern "C" { #include #include -/** @defgroup odp_event ODP EVENT +/** @addtogroup odp_event * Generic event metadata and operations. * @{ */ diff --git a/include/odp/api/spec/event_types.h b/include/odp/api/spec/event_types.h index 9df5e03ef..05c4d1f3c 100644 --- a/include/odp/api/spec/event_types.h +++ b/include/odp/api/spec/event_types.h @@ -17,7 +17,7 @@ extern "C" { #endif -/** @addtogroup odp_event +/** @defgroup odp_event ODP EVENT * @{ */ diff --git a/include/odp/api/spec/ipsec.h b/include/odp/api/spec/ipsec.h index b66e9a1ca..cdb18116b 100644 --- a/include/odp/api/spec/ipsec.h +++ b/include/odp/api/spec/ipsec.h @@ -23,7 +23,7 @@ extern "C" { #include #include -/** @defgroup odp_ipsec ODP IPSEC +/** @addtogroup odp_ipsec * IPSEC protocol offload. * @{ */ diff --git a/include/odp/api/spec/ipsec_types.h b/include/odp/api/spec/ipsec_types.h index 00fd944f5..71b53a770 100644 --- a/include/odp/api/spec/ipsec_types.h +++ b/include/odp/api/spec/ipsec_types.h @@ -24,7 +24,7 @@ extern "C" { #include #include -/** @addtogroup odp_ipsec +/** @defgroup odp_ipsec ODP IPSEC * @{ */ diff --git a/include/odp/api/spec/packet.h b/include/odp/api/spec/packet.h index ca22280ec..7f6c732ee 100644 --- a/include/odp/api/spec/packet.h +++ b/include/odp/api/spec/packet.h @@ -25,7 +25,7 @@ extern "C" { #include #include -/** @defgroup odp_packet ODP PACKET +/** @addtogroup odp_packet * Packet event metadata and operations. * @{ */ diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h index cfb463c39..a83617f7c 100644 --- a/include/odp/api/spec/packet_io.h +++ b/include/odp/api/spec/packet_io.h @@ -25,7 +25,7 @@ extern "C" { #include #include -/** @defgroup odp_packet_io ODP PACKET IO +/** @addtogroup odp_packet_io * Packet IO interfaces. * * Packet IO is the Ingress and Egress interface to ODP processing. It diff --git a/include/odp/api/spec/packet_io_types.h b/include/odp/api/spec/packet_io_types.h index 6b80611ec..9e56e087a 100644 --- a/include/odp/api/spec/packet_io_types.h +++ b/include/odp/api/spec/packet_io_types.h @@ -24,7 +24,7 @@ extern "C" { #include #include -/** @addtogroup odp_packet_io +/** @defgroup odp_packet_io ODP PACKET IO * @{ */ diff --git a/include/odp/api/spec/packet_types.h b/include/odp/api/spec/packet_types.h index ee62de4ff..b9d55abde 100644 --- a/include/odp/api/spec/packet_types.h +++ b/include/odp/api/spec/packet_types.h @@ -20,7 +20,7 @@ extern "C" { #include #include -/** @addtogroup odp_packet +/** @defgroup odp_packet ODP PACKET * @{ */ diff --git a/include/odp/api/spec/pool.h b/include/odp/api/spec/pool.h index b02c0d294..1b71a5a09 100644 --- a/include/odp/api/spec/pool.h +++ b/include/odp/api/spec/pool.h @@ -20,7 +20,7 @@ extern "C" { #include #include -/** @defgroup odp_pool ODP POOL +/** @addtogroup odp_pool * Packet and buffer (event) pools. * @{ */ diff --git a/include/odp/api/spec/pool_types.h b/include/odp/api/spec/pool_types.h index 7820349ef..da9f0d9e2 100644 --- a/include/odp/api/spec/pool_types.h +++ b/include/odp/api/spec/pool_types.h @@ -19,7 +19,7 @@ extern "C" { #include #include -/** @addtogroup odp_pool +/** @defgroup odp_pool ODP POOL * @{ */ diff --git a/include/odp/api/spec/proto_stats.h b/include/odp/api/spec/proto_stats.h index 1a1f67886..7dd57ac0f 100644 --- a/include/odp/api/spec/proto_stats.h +++ b/include/odp/api/spec/proto_stats.h @@ -18,7 +18,7 @@ extern "C" { #include -/** @defgroup odp_proto_stats ODP PROTO STATS +/** @addtogroup odp_proto_stats * Flow specific packet statistics. * @{ */ diff --git a/include/odp/api/spec/proto_stats_types.h b/include/odp/api/spec/proto_stats_types.h index f3ca80426..4c08e60ab 100644 --- a/include/odp/api/spec/proto_stats_types.h +++ b/include/odp/api/spec/proto_stats_types.h @@ -19,10 +19,15 @@ extern "C" { #include -/** @addtogroup odp_proto_stats +/** @defgroup odp_proto_stats ODP PROTO STATS * @{ */ +/** + * @typedef odp_proto_stats_t + * ODP proto stats handle + */ + /** * @def ODP_PROTO_STATS_INVALID * Invalid proto stats handle diff --git a/include/odp/api/spec/queue.h b/include/odp/api/spec/queue.h index 87f6e0d19..9ce2ac73f 100644 --- a/include/odp/api/spec/queue.h +++ b/include/odp/api/spec/queue.h @@ -21,7 +21,7 @@ extern "C" { #include #include -/** @defgroup odp_queue ODP QUEUE +/** @addtogroup odp_queue * Queues for event passing and scheduling. * @{ */ diff --git a/include/odp/api/spec/queue_types.h b/include/odp/api/spec/queue_types.h index 5f84a5f49..9edf7271d 100644 --- a/include/odp/api/spec/queue_types.h +++ b/include/odp/api/spec/queue_types.h @@ -18,7 +18,7 @@ extern "C" { #include -/** @addtogroup odp_queue +/** @defgroup odp_queue ODP QUEUE * @{ */ diff --git a/include/odp/api/spec/random.h b/include/odp/api/spec/random.h index dd30f9d48..10f1026b3 100644 --- a/include/odp/api/spec/random.h +++ b/include/odp/api/spec/random.h @@ -19,7 +19,7 @@ extern "C" { #include #include -/** @defgroup odp_random ODP RANDOM +/** @addtogroup odp_random * Random number generation. * @{ */ diff --git a/include/odp/api/spec/random_types.h b/include/odp/api/spec/random_types.h index cb7dccc7c..5d5ee9450 100644 --- a/include/odp/api/spec/random_types.h +++ b/include/odp/api/spec/random_types.h @@ -19,7 +19,7 @@ extern "C" { #include -/** @addtogroup odp_random +/** @defgroup odp_random ODP RANDOM * @{ */ diff --git a/include/odp/api/spec/schedule.h b/include/odp/api/spec/schedule.h index 31da38e4d..7226c198b 100644 --- a/include/odp/api/spec/schedule.h +++ b/include/odp/api/spec/schedule.h @@ -22,7 +22,7 @@ extern "C" { #include #include -/** @defgroup odp_scheduler ODP SCHEDULER +/** @addtogroup odp_scheduler * Event scheduler for work load balancing and prioritization. * @{ */ diff --git a/include/odp/api/spec/schedule_types.h b/include/odp/api/spec/schedule_types.h index b15397b96..30cb939dc 100644 --- a/include/odp/api/spec/schedule_types.h +++ b/include/odp/api/spec/schedule_types.h @@ -20,7 +20,7 @@ extern "C" { #endif -/** @addtogroup odp_scheduler +/** @defgroup odp_scheduler ODP SCHEDULER * @{ */ diff --git a/include/odp/api/spec/stash.h b/include/odp/api/spec/stash.h index 756214abe..61ae58eba 100644 --- a/include/odp/api/spec/stash.h +++ b/include/odp/api/spec/stash.h @@ -18,7 +18,7 @@ extern "C" { #include -/** @defgroup odp_stash ODP STASH +/** @addtogroup odp_stash * Stash for storing object handles * @{ */ diff --git a/include/odp/api/spec/stash_types.h b/include/odp/api/spec/stash_types.h index 96e136d78..48c4b9be8 100644 --- a/include/odp/api/spec/stash_types.h +++ b/include/odp/api/spec/stash_types.h @@ -18,7 +18,7 @@ extern "C" { #include -/** @addtogroup odp_stash +/** @defgroup odp_stash ODP STASH * @{ */ diff --git a/include/odp/api/spec/std.h b/include/odp/api/spec/std.h index fba1ee31d..7be543338 100644 --- a/include/odp/api/spec/std.h +++ b/include/odp/api/spec/std.h @@ -19,12 +19,10 @@ extern "C" { #include -/** - * @defgroup odp_std ODP STD - * Standard types and performance optimized versions of selected C library - * functions. - * - * @{ +/** @addtogroup odp_std + * Standard types and performance optimized versions of selected C library + * functions. + * @{ */ /** diff --git a/include/odp/api/spec/std_types.h b/include/odp/api/spec/std_types.h index e2630e044..e23dfd224 100644 --- a/include/odp/api/spec/std_types.h +++ b/include/odp/api/spec/std_types.h @@ -22,7 +22,7 @@ extern "C" { #endif -/** @addtogroup odp_std ODP STD +/** @defgroup odp_std ODP STD * @{ */ diff --git a/include/odp/api/spec/thread.h b/include/odp/api/spec/thread.h index dbb033da1..34d3c2b82 100644 --- a/include/odp/api/spec/thread.h +++ b/include/odp/api/spec/thread.h @@ -19,7 +19,7 @@ extern "C" { #include -/** @defgroup odp_thread ODP THREAD +/** @addtogroup odp_thread * Thread types, masks and IDs. * @{ */ diff --git a/include/odp/api/spec/thread_types.h b/include/odp/api/spec/thread_types.h index 204d28cad..13846cd8f 100644 --- a/include/odp/api/spec/thread_types.h +++ b/include/odp/api/spec/thread_types.h @@ -15,7 +15,7 @@ extern "C" { #endif -/** @ingroup odp_thread ODP THREAD +/** @defgroup odp_thread ODP THREAD * @{ */ diff --git a/include/odp/api/spec/time.h b/include/odp/api/spec/time.h index c3571f7fa..f7e20a6f4 100644 --- a/include/odp/api/spec/time.h +++ b/include/odp/api/spec/time.h @@ -20,7 +20,7 @@ extern "C" { #include #include -/** @defgroup odp_time ODP TIME +/** @addtogroup odp_time * SoC global and CPU local wall clock time * * @{ diff --git a/include/odp/api/spec/time_types.h b/include/odp/api/spec/time_types.h index cf0393ef0..bd8f324a3 100644 --- a/include/odp/api/spec/time_types.h +++ b/include/odp/api/spec/time_types.h @@ -17,7 +17,7 @@ extern "C" { #endif -/** @addtogroup odp_time +/** @defgroup odp_time ODP TIME * @{ */ From ac147a12f09649edd2ca3fae855d6253ebf880ae Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 4 Jan 2024 08:45:16 +0200 Subject: [PATCH 011/147] linux-gen: fix doxygen documentation grouping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, all types and defines from ABI headers were listed as functions in Doxygen HTML output. Signed-off-by: Matias Elo Reviewed-by: Jere Leppänen Reviewed-by: Petri Savolainen --- platform/linux-generic/include-abi/odp/api/abi/atomic.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/buffer_types.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/classification.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/comp.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/crypto_types.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/dma_types.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/event_types.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/ipsec_types.h | 2 +- .../linux-generic/include-abi/odp/api/abi/packet_io_types.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/packet_types.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/pool_types.h | 2 +- .../linux-generic/include-abi/odp/api/abi/proto_stats_types.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/queue_types.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/shared_memory.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/stash_types.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/sync.h | 2 +- platform/linux-generic/include-abi/odp/api/abi/ticketlock.h | 2 +- platform/linux-generic/include/odp_ipsec_internal.h | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/platform/linux-generic/include-abi/odp/api/abi/atomic.h b/platform/linux-generic/include-abi/odp/api/abi/atomic.h index 9c87f9cb8..4f481f913 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/atomic.h +++ b/platform/linux-generic/include-abi/odp/api/abi/atomic.h @@ -80,7 +80,7 @@ typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s { #endif -/** @ingroup odp_atomic +/** @addtogroup odp_atomic * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h b/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h index 1d54bab07..63067268c 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h @@ -21,7 +21,7 @@ extern "C" { #include #include -/** @ingroup odp_buffer +/** @addtogroup odp_buffer * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/classification.h b/platform/linux-generic/include-abi/odp/api/abi/classification.h index 342f4124c..d63763dbd 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/classification.h +++ b/platform/linux-generic/include-abi/odp/api/abi/classification.h @@ -19,7 +19,7 @@ extern "C" { #include -/** @ingroup odp_classification +/** @addtogroup odp_classification * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/comp.h b/platform/linux-generic/include-abi/odp/api/abi/comp.h index ac3d3a4a9..45681e961 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/comp.h +++ b/platform/linux-generic/include-abi/odp/api/abi/comp.h @@ -15,7 +15,7 @@ extern "C" { #include -/** @ingroup odp_compression +/** @addtogroup odp_compression * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h b/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h index d49caf89a..b1e4aa5ae 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h @@ -22,7 +22,7 @@ extern "C" { #include -/** @ingroup odp_crypto +/** @addtogroup odp_crypto * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/dma_types.h b/platform/linux-generic/include-abi/odp/api/abi/dma_types.h index 768591b10..d5bee0374 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/dma_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/dma_types.h @@ -13,7 +13,7 @@ extern "C" { #include -/** @ingroup odp_dma +/** @addtogroup odp_dma * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/event_types.h b/platform/linux-generic/include-abi/odp/api/abi/event_types.h index 8ff5acd6b..8208ad5a8 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/event_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/event_types.h @@ -20,7 +20,7 @@ extern "C" { #include -/** @ingroup odp_event +/** @addtogroup odp_event * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/ipsec_types.h b/platform/linux-generic/include-abi/odp/api/abi/ipsec_types.h index 376666ded..1c5501997 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/ipsec_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/ipsec_types.h @@ -22,7 +22,7 @@ extern "C" { #include -/** @ingroup odp_ipsec +/** @addtogroup odp_ipsec * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/packet_io_types.h b/platform/linux-generic/include-abi/odp/api/abi/packet_io_types.h index 87e297f1d..76b162020 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/packet_io_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/packet_io_types.h @@ -21,7 +21,7 @@ extern "C" { #include #include -/** @ingroup odp_packet_io +/** @addtogroup odp_packet_io * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/packet_types.h b/platform/linux-generic/include-abi/odp/api/abi/packet_types.h index 4da9332ba..90b2af107 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/packet_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/packet_types.h @@ -21,7 +21,7 @@ extern "C" { #include #include -/** @ingroup odp_packet +/** @addtogroup odp_packet * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/pool_types.h b/platform/linux-generic/include-abi/odp/api/abi/pool_types.h index 0c0dbc97f..77b0ff638 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/pool_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/pool_types.h @@ -19,7 +19,7 @@ extern "C" { #include -/** @ingroup odp_pool +/** @addtogroup odp_pool * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/proto_stats_types.h b/platform/linux-generic/include-abi/odp/api/abi/proto_stats_types.h index 2ebddce62..d9db29188 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/proto_stats_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/proto_stats_types.h @@ -21,7 +21,7 @@ extern "C" { #include #include -/** @ingroup odp_proto_stats +/** @addtogroup odp_proto_stats * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/queue_types.h b/platform/linux-generic/include-abi/odp/api/abi/queue_types.h index 1a56c7682..4eff762bd 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/queue_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/queue_types.h @@ -21,7 +21,7 @@ extern "C" { #include #include -/** @ingroup odp_queue +/** @addtogroup odp_queue * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/shared_memory.h b/platform/linux-generic/include-abi/odp/api/abi/shared_memory.h index 551d49e30..bfcb9ebe5 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/shared_memory.h +++ b/platform/linux-generic/include-abi/odp/api/abi/shared_memory.h @@ -21,7 +21,7 @@ extern "C" { #include #include -/** @ingroup odp_shared_memory +/** @addtogroup odp_shared_memory * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/stash_types.h b/platform/linux-generic/include-abi/odp/api/abi/stash_types.h index 960f3ef17..2a4115886 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/stash_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/stash_types.h @@ -17,7 +17,7 @@ extern "C" { #include -/** @ingroup odp_stash +/** @addtogroup odp_stash * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/sync.h b/platform/linux-generic/include-abi/odp/api/abi/sync.h index 236e92c8c..276514b58 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/sync.h +++ b/platform/linux-generic/include-abi/odp/api/abi/sync.h @@ -17,7 +17,7 @@ extern "C" { #endif -/** @ingroup odp_barrier +/** @addtogroup odp_barrier * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/ticketlock.h b/platform/linux-generic/include-abi/odp/api/abi/ticketlock.h index 6543a1cf7..b621bea7e 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/ticketlock.h +++ b/platform/linux-generic/include-abi/odp/api/abi/ticketlock.h @@ -19,7 +19,7 @@ extern "C" { #include -/** @ingroup odp_locks +/** @addtogroup odp_locks * @{ */ diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index 571796691..b97aa7031 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -30,7 +30,7 @@ extern "C" { #include #include -/** @ingroup odp_ipsec +/** @addtogroup odp_ipsec * @{ */ From 5c818c64bd9c361fb62ce20cc2bb8798616ecb45 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Wed, 3 Jan 2024 14:20:50 +0200 Subject: [PATCH 012/147] example: debug: print link info MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Start interface and wait until link is up to ensure correct link speed, etc properties. Print link info, MTU and other packet IO information. Signed-off-by: Petri Savolainen Reviewed-by: Jere Leppänen --- example/debug/odp_debug.c | 62 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/example/debug/odp_debug.c b/example/debug/odp_debug.c index fee1c02ad..8da9bf265 100644 --- a/example/debug/odp_debug.c +++ b/example/debug/odp_debug.c @@ -325,6 +325,8 @@ static int pktio_debug(void) odp_pool_t pool; odp_pool_param_t pool_param; odp_pktio_t pktio; + odp_pktio_link_info_t info; + uint8_t mac[ODPH_ETHADDR_LEN]; int pkt_len = 100; odp_pool_param_init(&pool_param); @@ -346,12 +348,72 @@ static int pktio_debug(void) return -1; } + /* Start interface with default config */ + if (odp_pktin_queue_config(pktio, NULL)) { + ODPH_ERR("Packet input queue config failed\n"); + return -1; + } + + if (odp_pktout_queue_config(pktio, NULL)) { + ODPH_ERR("Packet output queue config failed\n"); + return -1; + } + + if (odp_pktio_start(pktio)) { + ODPH_ERR("Pktio start failed\n"); + return -1; + } + + printf("\nWaiting link up"); + + /* Wait max 5 seconds for link up */ + for (int i = 0; i < 25; i++) { + if (odp_pktio_link_status(pktio) == ODP_PKTIO_LINK_STATUS_UP) + break; + + odp_time_wait_ns(200 * ODP_TIME_MSEC_IN_NS); + printf("."); + fflush(NULL); + } + + printf("\n\n"); + + printf("Packet IO\n---------\n"); + printf(" index: %i\n", odp_pktio_index(pktio)); + printf(" handle: 0x%" PRIx64 "\n", odp_pktio_to_u64(pktio)); + + if (odp_pktio_mac_addr(pktio, mac, ODPH_ETHADDR_LEN) == ODPH_ETHADDR_LEN) { + printf(" mac address: %02x:%02x:%02x:%02x:%02x:%02x\n", mac[0], mac[1], mac[2], + mac[3], mac[4], mac[5]); + } + + printf(" input maxlen: %u\n", odp_pktin_maxlen(pktio)); + printf(" output maxlen: %u\n", odp_pktout_maxlen(pktio)); + printf(" promisc mode: %i\n", odp_pktio_promisc_mode(pktio)); + printf(" timestamp res: %" PRIu64 " Hz\n\n", odp_pktio_ts_res(pktio)); + + if (odp_pktio_link_info(pktio, &info) == 0) { + printf("Link info\n---------\n"); + printf(" auto neg: %i\n", info.autoneg); + printf(" duplex: %i\n", info.duplex); + printf(" media: %s\n", info.media); + printf(" pause_rx: %i\n", info.pause_rx); + printf(" pause_tx: %i\n", info.pause_tx); + printf(" speed: %u Mbit/s\n", info.speed); + printf(" status: %i\n", info.status); + } + printf("\n"); odp_pktio_print(pktio); printf("\n"); odp_pktio_extra_stats_print(pktio); + if (odp_pktio_stop(pktio)) { + ODPH_ERR("Pktio stop failed\n"); + return -1; + } + if (odp_pktio_close(pktio)) { ODPH_ERR("Pktio close failed\n"); return -1; From 4f94f642791271eca620c6e447233558b7d51d5e Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 4 Jan 2024 14:39:03 +0200 Subject: [PATCH 013/147] example: debug: pktio interface name option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add interface name as parameter to the packet IO option (-i). The loop interface is still used by default. Signed-off-by: Petri Savolainen Reviewed-by: Jere Leppänen --- example/debug/odp_debug.c | 46 ++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/example/debug/odp_debug.c b/example/debug/odp_debug.c index 8da9bf265..79107ca8f 100644 --- a/example/debug/odp_debug.c +++ b/example/debug/odp_debug.c @@ -20,6 +20,8 @@ #include #include +#define MAX_NAME_LEN 128 + typedef struct test_global_t { int system; int shm; @@ -29,6 +31,7 @@ typedef struct test_global_t { int ipsec; int timer; int stash; + char pktio_name[MAX_NAME_LEN]; } test_global_t; @@ -41,35 +44,37 @@ static void print_usage(void) "are called when no options are given.\n" "\n" "OPTIONS:\n" - " -S, --system Call odp_sys_info_print() and odp_sys_config_print()\n" - " -s, --shm Create a SHM and call odp_shm_print()\n" - " -p, --pool Create various types of pools and call odp_pool_print()\n" - " -q, --queue Create various types of queues and call odp_queue_print()\n" - " -i, --interface Create packet IO interface (loop), and call both odp_pktio_print()\n" - " and odp_pktio_extra_stats_print()\n" - " -I, --ipsec Call odp_ipsec_print()\n" - " -t, --timer Call timer pool, timer and timeout print functions\n" - " -a, --stash Create stash and call odp_stash_print()\n" - " -h, --help Display help and exit.\n\n"); + " -S, --system Call odp_sys_info_print() and odp_sys_config_print()\n" + " -s, --shm Create a SHM and call odp_shm_print()\n" + " -p, --pool Create various types of pools and call odp_pool_print()\n" + " -q, --queue Create various types of queues and call odp_queue_print()\n" + " -i, --interface Start a packet IO interface, and call odp_pktio_print(),\n" + " odp_pktio_extra_stats_print(), etc. Uses loop interface by default.\n" + " -I, --ipsec Call odp_ipsec_print()\n" + " -t, --timer Call timer pool, timer and timeout print functions\n" + " -a, --stash Create stash and call odp_stash_print()\n" + " -h, --help Display help and exit.\n\n"); } static int parse_options(int argc, char *argv[], test_global_t *global) { int opt, long_index; + char *str; + uint32_t str_len = 0; const struct option longopts[] = { {"system", no_argument, NULL, 'S'}, {"shm", no_argument, NULL, 's'}, {"pool", no_argument, NULL, 'p'}, {"queue", no_argument, NULL, 'q'}, - {"interface", no_argument, NULL, 'i'}, + {"interface", required_argument, NULL, 'i'}, {"ipsec", no_argument, NULL, 'I'}, {"timer", no_argument, NULL, 't'}, {"stash", no_argument, NULL, 'a'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - const char *shortopts = "+SspqiItah"; + const char *shortopts = "+Sspqi:Itah"; int ret = 0; while (1) { @@ -93,6 +98,11 @@ static int parse_options(int argc, char *argv[], test_global_t *global) break; case 'i': global->pktio = 1; + str = optarg; + str_len = strlen(str); + + if (str_len && str_len < MAX_NAME_LEN) + strcpy(global->pktio_name, str); break; case 'I': global->ipsec = 1; @@ -110,6 +120,11 @@ static int parse_options(int argc, char *argv[], test_global_t *global) } } + if (global->pktio && (str_len == 0 || str_len >= MAX_NAME_LEN)) { + ODPH_ERR("Bad interface name length: %u\n", str_len); + ret = -1; + } + return ret; } @@ -320,7 +335,7 @@ static int queue_debug(void) return 0; } -static int pktio_debug(void) +static int pktio_debug(test_global_t *global) { odp_pool_t pool; odp_pool_param_t pool_param; @@ -341,7 +356,7 @@ static int pktio_debug(void) return -1; } - pktio = odp_pktio_open("loop", pool, NULL); + pktio = odp_pktio_open(global->pktio_name, pool, NULL); if (pktio == ODP_PKTIO_INVALID) { ODPH_ERR("Pktio open failed\n"); @@ -630,6 +645,7 @@ int main(int argc, char *argv[]) global->ipsec = 1; global->timer = 1; global->stash = 1; + strcpy(global->pktio_name, "loop"); } else { if (parse_options(argc, argv, global)) exit(EXIT_FAILURE); @@ -674,7 +690,7 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } - if (global->pktio && pktio_debug()) { + if (global->pktio && pktio_debug(global)) { ODPH_ERR("Packet debug failed.\n"); exit(EXIT_FAILURE); } From 23c7f61183a89b66cffd50b12afebd1708f09949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jere=20Lepp=C3=A4nen?= Date: Fri, 5 Jan 2024 14:22:52 +0200 Subject: [PATCH 014/147] test: timer_perf: avoid modulo operation in a couple of loops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simplify a couple of loops and avoid a modulo operation. Signed-off-by: Jere Leppänen Reviewed-by: Petri Savolainen --- test/performance/odp_timer_perf.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/test/performance/odp_timer_perf.c b/test/performance/odp_timer_perf.c index 16e2cb151..43eaa0625 100644 --- a/test/performance/odp_timer_perf.c +++ b/test/performance/odp_timer_perf.c @@ -667,10 +667,7 @@ static int cancel_timers(test_global_t *global, uint32_t worker_idx) int ret = 0; for (i = 0; i < num_tp; i++) { - for (j = 0; j < num_timer; j++) { - if ((j % num_worker) != worker_idx) - continue; - + for (j = worker_idx; j < num_timer; j += num_worker) { timer = global->timer[i][j]; if (timer == ODP_TIMER_INVALID) continue; @@ -783,10 +780,7 @@ static int set_cancel_mode_worker(void *arg) tick = odp_timer_current_tick(tp) + start_tick; - for (j = 0; j < num_timer; j++) { - if ((j % num_worker) != worker_idx) - continue; - + for (j = worker_idx; j < num_timer; j += num_worker) { timer = global->timer[i][j]; if (timer == ODP_TIMER_INVALID) continue; From 09b0c17bbdeb9f4abf5adbed081604f8a203d1fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jere=20Lepp=C3=A4nen?= Date: Fri, 5 Jan 2024 15:04:27 +0200 Subject: [PATCH 015/147] test: timer_perf: measure cycles separately for start and cancel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In mode 1 ("timer set + cancel performance"), measure CPU cycles for odp_timer_start() and odp_timer_cancel() separately. Signed-off-by: Jere Leppänen Reviewed-by: Petri Savolainen --- test/performance/odp_timer_perf.c | 64 +++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/test/performance/odp_timer_perf.c b/test/performance/odp_timer_perf.c index 43eaa0625..918267a1b 100644 --- a/test/performance/odp_timer_perf.c +++ b/test/performance/odp_timer_perf.c @@ -691,7 +691,7 @@ static int cancel_timers(test_global_t *global, uint32_t worker_idx) static int set_cancel_mode_worker(void *arg) { uint64_t tick, start_tick, period_tick, nsec; - uint64_t c1, c2, diff; + uint64_t c1, c2; int thr, status; uint32_t i, j, worker_idx; odp_event_t ev; @@ -712,11 +712,12 @@ static int set_cancel_mode_worker(void *arg) uint64_t num_tmo = 0; uint64_t num_cancel = 0; uint64_t num_set = 0; + uint64_t cancel_cycles = 0, start_cycles = 0; + odp_event_t ev_tbl[MAX_TIMERS]; thr = odp_thread_id(); worker_idx = thread_arg->worker_idx; t1 = ODP_TIME_NULL; - c1 = 0; /* Start all workers at the same time */ odp_barrier_wait(&global->barrier); @@ -766,7 +767,6 @@ static int set_cancel_mode_worker(void *arg) /* Start measurements */ started = 1; t1 = odp_time_local(); - c1 = odp_cpu_cycles(); } /* Cancel and set timers again */ @@ -779,13 +779,16 @@ static int set_cancel_mode_worker(void *arg) period_tick = global->timer_pool[i].period_tick; tick = odp_timer_current_tick(tp) + start_tick; + c1 = odp_cpu_cycles(); for (j = worker_idx; j < num_timer; j += num_worker) { + ev_tbl[j] = ODP_EVENT_INVALID; + timer = global->timer[i][j]; if (timer == ODP_TIMER_INVALID) continue; - status = odp_timer_cancel(timer, &ev); + status = odp_timer_cancel(timer, &ev_tbl[j]); num_cancel++; if (odp_unlikely(status == ODP_TIMER_TOO_NEAR)) { @@ -796,10 +799,23 @@ static int set_cancel_mode_worker(void *arg) ret = -1; break; } + } + + c2 = odp_cpu_cycles(); + cancel_cycles += odp_cpu_cycles_diff(c2, c1); + c1 = c2; + + for (j = worker_idx; j < num_timer; j += num_worker) { + if (ev_tbl[j] == ODP_EVENT_INVALID) + continue; + + timer = global->timer[i][j]; + if (timer == ODP_TIMER_INVALID) + continue; start_param.tick_type = ODP_TIMER_TICK_ABS; start_param.tick = tick + j * period_tick; - start_param.tmo_ev = ev; + start_param.tmo_ev = ev_tbl[j]; status = odp_timer_start(timer, &start_param); num_set++; @@ -811,6 +827,9 @@ static int set_cancel_mode_worker(void *arg) break; } } + + c2 = odp_cpu_cycles(); + start_cycles += odp_cpu_cycles_diff(c2, c1); } if (test_rounds) { @@ -821,9 +840,7 @@ static int set_cancel_mode_worker(void *arg) } t2 = odp_time_local(); - c2 = odp_cpu_cycles(); nsec = odp_time_diff_ns(t2, t1); - diff = odp_cpu_cycles_diff(c2, c1); /* Cancel all timers that belong to this thread */ if (cancel_timers(global, worker_idx)) @@ -833,7 +850,8 @@ static int set_cancel_mode_worker(void *arg) global->stat[thr].events = num_tmo; global->stat[thr].rounds = test_options->test_rounds - test_rounds; global->stat[thr].nsec = nsec; - global->stat[thr].cycles_0 = diff; + global->stat[thr].cycles_0 = cancel_cycles; + global->stat[thr].cycles_1 = start_cycles; global->stat[thr].cancels = num_cancel; global->stat[thr].sets = num_set; @@ -1106,16 +1124,38 @@ static void print_stat_set_cancel_mode(test_global_t *global) int num = 0; printf("\n"); - printf("RESULTS - timer cancel + set cycles per thread:\n"); - printf("-----------------------------------------------\n"); + printf("RESULTS\n"); + printf("odp_timer_cancel() cycles per thread:\n"); + printf("-------------------------------------------------\n"); printf(" 1 2 3 4 5 6 7 8 9 10"); for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { - if (global->stat[i].sets) { + const test_stat_t *si = &global->stat[i]; + + if (si->cancels) { + if ((num % 10) == 0) + printf("\n "); + + printf("%6.1f ", (double)si->cycles_0 / si->cancels); + num++; + } + } + + printf("\n\n"); + + num = 0; + printf("odp_timer_start() cycles per thread:\n"); + printf("-------------------------------------------------\n"); + printf(" 1 2 3 4 5 6 7 8 9 10"); + + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + const test_stat_t *si = &global->stat[i]; + + if (si->sets) { if ((num % 10) == 0) printf("\n "); - printf("%6.1f ", (double)global->stat[i].cycles_0 / global->stat[i].sets); + printf("%6.1f ", (double)si->cycles_1 / si->sets); num++; } } From 81573060a48a1436ff1a7f51216a4d23cbf3378f Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Wed, 10 Jan 2024 16:48:08 +0200 Subject: [PATCH 016/147] test: bench_buffer: add new test cases Add test cases for recently added event and buffer API calls. Corrected also bugs in event vs buffer allocation in test case initialization. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/performance/odp_bench_buffer.c | 123 ++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 9 deletions(-) diff --git a/test/performance/odp_bench_buffer.c b/test/performance/odp_bench_buffer.c index c85340d92..91ec73c46 100644 --- a/test/performance/odp_bench_buffer.c +++ b/test/performance/odp_bench_buffer.c @@ -89,9 +89,9 @@ typedef struct { /** Array for storing test pool handles */ odp_pool_t pool_tbl[TEST_REPEAT_COUNT]; /** Array for storing test event types */ - odp_event_type_t event_type_tbl[TEST_REPEAT_COUNT]; + odp_event_type_t event_type_tbl[TEST_REPEAT_COUNT * TEST_MAX_BURST]; /** Array for storing test event subtypes */ - odp_event_subtype_t event_subtype_tbl[TEST_REPEAT_COUNT]; + odp_event_subtype_t event_subtype_tbl[TEST_REPEAT_COUNT * TEST_MAX_BURST]; /** CPU mask as string */ char cpumask_str[ODP_CPUMASK_STR_SIZE]; } args_t; @@ -121,14 +121,14 @@ static void allocate_test_buffers(odp_buffer_t buf[], int num) } } -static void alloc_buffers_multi(void) +static void create_buffers(void) { - allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT * gbl_args->appl.burst_size); + allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT); } -static void create_buffers(void) +static void create_buffers_multi(void) { - allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT); + allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT * gbl_args->appl.burst_size); } static void create_events(void) @@ -184,6 +184,20 @@ static int buffer_from_event(void) return i; } +static int buffer_from_event_multi(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + odp_event_t *event_tbl = gbl_args->event_tbl; + int burst_size = gbl_args->appl.burst_size; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_buffer_from_event_multi(&buf_tbl[i * burst_size], + &event_tbl[i * burst_size], burst_size); + + return i; +} + static int buffer_to_event(void) { odp_buffer_t *buf_tbl = gbl_args->buf_tbl; @@ -196,6 +210,20 @@ static int buffer_to_event(void) return i; } +static int buffer_to_event_multi(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + odp_event_t *event_tbl = gbl_args->event_tbl; + int burst_size = gbl_args->appl.burst_size; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_buffer_to_event_multi(&buf_tbl[i * burst_size], + &event_tbl[i * burst_size], burst_size); + + return i; +} + static int buffer_addr(void) { odp_buffer_t *buf_tbl = gbl_args->buf_tbl; @@ -373,6 +401,36 @@ static int event_types(void) return i; } +static int event_types_multi(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl; + odp_event_subtype_t *event_subtype_tbl = gbl_args->event_subtype_tbl; + int burst_size = gbl_args->appl.burst_size; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_event_types_multi(&event_tbl[i * burst_size], + &event_type_tbl[i * burst_size], + &event_subtype_tbl[i * burst_size], burst_size); + + return i; +} + +static int event_types_multi_no_sub(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl; + int burst_size = gbl_args->appl.burst_size; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_event_types_multi(&event_tbl[i * burst_size], + &event_type_tbl[i * burst_size], NULL, burst_size); + + return i; +} + static int event_type_multi(void) { odp_event_t *event_tbl = gbl_args->event_tbl; @@ -387,6 +445,45 @@ static int event_type_multi(void) return ret; } +static int event_pool(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + odp_pool_t *pool_tbl = gbl_args->pool_tbl; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + pool_tbl[i] = odp_event_pool(event_tbl[i]); + + return i; +} + +static int event_user_area(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + void **ptr_tbl = gbl_args->ptr_tbl; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + ptr_tbl[i] = odp_event_user_area(event_tbl[i]); + + return i; +} + +static int event_user_area_and_flag(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + void **ptr_tbl = gbl_args->ptr_tbl; + int ret = 0; + int flag; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) { + ptr_tbl[i] = odp_event_user_area_and_flag(event_tbl[i], &flag); + ret += flag; + } + + return ret; +} + static int event_is_valid(void) { odp_event_t *event_tbl = gbl_args->event_tbl; @@ -580,7 +677,9 @@ static void print_info(void) */ bench_info_t test_suite[] = { BENCH_INFO(buffer_from_event, create_events, free_buffers, NULL), + BENCH_INFO(buffer_from_event_multi, create_events_multi, free_buffers_multi, NULL), BENCH_INFO(buffer_to_event, create_buffers, free_buffers, NULL), + BENCH_INFO(buffer_to_event_multi, create_buffers_multi, free_buffers_multi, NULL), BENCH_INFO(buffer_addr, create_buffers, free_buffers, NULL), BENCH_INFO(buffer_size, create_buffers, free_buffers, NULL), BENCH_INFO_COND(buffer_user_area, create_buffers, free_buffers, NULL, check_uarea), @@ -588,14 +687,20 @@ bench_info_t test_suite[] = { BENCH_INFO(buffer_alloc, NULL, free_buffers, NULL), BENCH_INFO(buffer_alloc_multi, NULL, free_buffers_multi, NULL), BENCH_INFO(buffer_free, create_buffers, NULL, NULL), - BENCH_INFO(buffer_free_multi, alloc_buffers_multi, NULL, NULL), + BENCH_INFO(buffer_free_multi, create_buffers_multi, NULL, NULL), BENCH_INFO(buffer_alloc_free, NULL, NULL, NULL), BENCH_INFO(buffer_alloc_free_multi, NULL, NULL, NULL), BENCH_INFO(buffer_is_valid, create_buffers, free_buffers, NULL), BENCH_INFO(event_type, create_events, free_buffers, NULL), - BENCH_INFO(event_subtype, create_buffers, free_buffers, NULL), - BENCH_INFO(event_types, create_buffers, free_buffers, NULL), + BENCH_INFO(event_subtype, create_events, free_buffers, NULL), + BENCH_INFO(event_types, create_events, free_buffers, NULL), + BENCH_INFO(event_types_multi, create_events_multi, free_buffers_multi, NULL), + BENCH_INFO(event_types_multi_no_sub, create_events_multi, free_buffers_multi, + "event_types_multi (no sub)"), BENCH_INFO(event_type_multi, create_events_multi, free_buffers_multi, NULL), + BENCH_INFO(event_pool, create_events, free_buffers, NULL), + BENCH_INFO_COND(event_user_area, create_events, free_buffers, NULL, check_uarea), + BENCH_INFO_COND(event_user_area_and_flag, create_events, free_buffers, NULL, check_uarea), BENCH_INFO(event_is_valid, create_events, free_buffers, NULL), BENCH_INFO(event_free, create_events, NULL, NULL), BENCH_INFO(event_free_multi, create_events_multi, NULL, NULL), From 187680b446c5ae2a399f89329b7fca05caeda56a Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 23 Nov 2023 16:15:58 +0200 Subject: [PATCH 017/147] linux-gen: event: move subtype into event header Other events in addition to packets may have event subtype in the future. Move subtype field from packet to event header. Event header size did not change due to the move. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- .../include/odp/api/plat/event_inline_types.h | 1 + .../include/odp/api/plat/event_inlines.h | 17 ++++++++++++----- .../include/odp/api/plat/packet_inline_types.h | 1 - .../include/odp/api/plat/packet_inlines.h | 4 +++- .../linux-generic/include/odp_event_internal.h | 3 +++ .../linux-generic/include/odp_packet_internal.h | 17 ++++++++--------- platform/linux-generic/odp_event.c | 1 + platform/linux-generic/odp_packet.c | 5 ++--- 8 files changed, 30 insertions(+), 19 deletions(-) diff --git a/platform/linux-generic/include/odp/api/plat/event_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_inline_types.h index caa075871..cbf01588f 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/event_inline_types.h @@ -28,6 +28,7 @@ extern "C" { typedef struct _odp_event_inline_offset_t { uint16_t event_type; uint16_t base_data; + uint16_t subtype; uint16_t flow_id; uint16_t pool; diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index b68ced244..924b6dc65 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -49,6 +49,15 @@ static inline odp_event_type_t __odp_event_type_get(odp_event_t event) return (odp_event_type_t)type; } +static inline odp_event_subtype_t __odp_event_subtype_get(odp_event_t event) +{ + int8_t type; + + type = _odp_event_hdr_field(event, int8_t, subtype); + + return (odp_event_subtype_t)type; +} + _ODP_INLINE odp_event_type_t odp_event_type(odp_event_t event) { return __odp_event_type_get(event); @@ -148,7 +157,7 @@ _ODP_INLINE odp_event_subtype_t odp_event_subtype(odp_event_t event) if (__odp_event_type_get(event) != ODP_EVENT_PACKET) return ODP_EVENT_NO_SUBTYPE; - return (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event, int8_t, subtype); + return __odp_event_subtype_get(event); } _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, @@ -157,8 +166,7 @@ _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, odp_event_type_t event_type = __odp_event_type_get(event); *subtype = event_type == ODP_EVENT_PACKET ? - (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event, int8_t, subtype) : - ODP_EVENT_NO_SUBTYPE; + __odp_event_subtype_get(event) : ODP_EVENT_NO_SUBTYPE; return event_type; } @@ -174,8 +182,7 @@ _ODP_INLINE void odp_event_types_multi(const odp_event_t event[], odp_event_type for (int i = 0; i < num; i++) { subtype[i] = (type[i] == ODP_EVENT_PACKET) ? - (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event[i], int8_t, - subtype) : ODP_EVENT_NO_SUBTYPE; + __odp_event_subtype_get(event[i]) : ODP_EVENT_NO_SUBTYPE; } } diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index eb20ca7d7..691965624 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -50,7 +50,6 @@ typedef struct _odp_packet_inline_offset_t { uint16_t timestamp; uint16_t input_flags; uint16_t flags; - uint16_t subtype; uint16_t cls_mark; uint16_t ipsec_ctx; uint16_t crypto_op; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 960dbc5fc..2dd74fa29 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -571,7 +572,8 @@ _ODP_INLINE void odp_packet_to_event_multi(const odp_packet_t pkt[], _ODP_INLINE odp_event_subtype_t odp_packet_subtype(odp_packet_t pkt) { - return (odp_event_subtype_t)_odp_pkt_get(pkt, int8_t, subtype); + return (odp_event_subtype_t)_odp_event_hdr_field((odp_event_t)(uintptr_t)pkt, + int8_t, subtype); } _ODP_INLINE odp_packet_tx_compl_t odp_packet_tx_compl_from_event(odp_event_t ev) diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h index d9957e530..1b85d64fc 100644 --- a/platform/linux-generic/include/odp_event_internal.h +++ b/platform/linux-generic/include/odp_event_internal.h @@ -65,6 +65,9 @@ typedef struct _odp_event_hdr_t { /* Event type. Maybe different than pool type (crypto compl event) */ int8_t event_type; + /* Event subtype */ + int8_t subtype; + /* Event flow id */ uint8_t flow_id; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 41a44b83c..f7904c3bc 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -107,9 +107,6 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { uint16_t tailroom; - /* Event subtype */ - int8_t subtype; - /* Used as classifier destination queue, in IPsec inline input processing and as Tx * completion event queue. */ odp_queue_t dst_queue; @@ -214,9 +211,11 @@ static inline odp_packet_hdr_t *packet_last_seg(odp_packet_hdr_t *hdr) return hdr; } -static inline void packet_subtype_set(odp_packet_t pkt, int ev) +static inline void packet_subtype_set(odp_packet_t pkt, int subtype) { - packet_hdr(pkt)->subtype = ev; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + pkt_hdr->event_hdr.subtype = subtype; } /** @@ -258,8 +257,8 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->headroom = pool->headroom; pkt_hdr->tailroom = pool->seg_len - seg_len + pool->tailroom; - if (odp_unlikely(pkt_hdr->subtype != ODP_EVENT_PACKET_BASIC)) - pkt_hdr->subtype = ODP_EVENT_PACKET_BASIC; + if (odp_unlikely(pkt_hdr->event_hdr.subtype != ODP_EVENT_PACKET_BASIC)) + pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC; pkt_hdr->input = ODP_PKTIO_INVALID; } @@ -304,7 +303,7 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, odp_packet_hdr_t *src_hdr, odp_bool_t uarea_copy) { - int8_t subtype = src_hdr->subtype; + int8_t subtype = src_hdr->event_hdr.subtype; /* Lengths and segmentation data are not copied: * .frame_len @@ -316,7 +315,7 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, * .seg_count */ dst_hdr->input = src_hdr->input; - dst_hdr->subtype = subtype; + dst_hdr->event_hdr.subtype = subtype; dst_hdr->dst_queue = src_hdr->dst_queue; dst_hdr->cos = src_hdr->cos; dst_hdr->cls_mark = src_hdr->cls_mark; diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c index e15cb1c50..9ec4b4bfb 100644 --- a/platform/linux-generic/odp_event.c +++ b/platform/linux-generic/odp_event.c @@ -36,6 +36,7 @@ const _odp_event_inline_offset_t _odp_event_inline_offset ODP_ALIGNED_CACHE = { .event_type = offsetof(_odp_event_hdr_t, event_type), .base_data = offsetof(_odp_event_hdr_t, base_data), + .subtype = offsetof(_odp_event_hdr_t, subtype), .flow_id = offsetof(_odp_event_hdr_t, flow_id), .pool = offsetof(_odp_event_hdr_t, pool), }; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 96fcd928a..17a4a9298 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -66,7 +66,6 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .timestamp = offsetof(odp_packet_hdr_t, timestamp), .input_flags = offsetof(odp_packet_hdr_t, p.input_flags), .flags = offsetof(odp_packet_hdr_t, p.flags), - .subtype = offsetof(odp_packet_hdr_t, subtype), .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), .ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx), .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result), @@ -1454,7 +1453,7 @@ void odp_packet_print(odp_packet_t pkt) len += _odp_snprint(&str[len], n - len, " pool index %u\n", hdr->event_hdr.index.pool); len += _odp_snprint(&str[len], n - len, " buf index %u\n", hdr->event_hdr.index.event); - len += _odp_snprint(&str[len], n - len, " ev subtype %i\n", hdr->subtype); + len += _odp_snprint(&str[len], n - len, " ev subtype %i\n", hdr->event_hdr.subtype); len += _odp_snprint(&str[len], n - len, " input_flags 0x%" PRIx64 "\n", hdr->p.input_flags.all); if (hdr->p.input_flags.all) { @@ -2401,7 +2400,7 @@ odp_packet_t odp_packet_reassemble(odp_pool_t pool_hdl, odp_packet_buf_t pkt_buf pkt_hdr->tailroom = tailroom; /* Reset metadata */ - pkt_hdr->subtype = ODP_EVENT_PACKET_BASIC; + pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC; pkt_hdr->input = ODP_PKTIO_INVALID; packet_parse_reset(pkt_hdr, 1); From dfec7b83270466d01999ee9a6ba58317ae0c3c1a Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 23 Nov 2023 16:35:50 +0200 Subject: [PATCH 018/147] linux-gen: event: initialize subtype always Subtype get functions are simpler when subtype is always initialized. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- .../include/odp/api/plat/event_inlines.h | 12 +++--------- platform/linux-generic/odp_pool.c | 1 + 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 924b6dc65..d30c6acbb 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -154,9 +154,6 @@ _ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) _ODP_INLINE odp_event_subtype_t odp_event_subtype(odp_event_t event) { - if (__odp_event_type_get(event) != ODP_EVENT_PACKET) - return ODP_EVENT_NO_SUBTYPE; - return __odp_event_subtype_get(event); } @@ -165,8 +162,7 @@ _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, { odp_event_type_t event_type = __odp_event_type_get(event); - *subtype = event_type == ODP_EVENT_PACKET ? - __odp_event_subtype_get(event) : ODP_EVENT_NO_SUBTYPE; + *subtype = __odp_event_subtype_get(event); return event_type; } @@ -180,10 +176,8 @@ _ODP_INLINE void odp_event_types_multi(const odp_event_t event[], odp_event_type if (subtype == NULL) return; - for (int i = 0; i < num; i++) { - subtype[i] = (type[i] == ODP_EVENT_PACKET) ? - __odp_event_subtype_get(event[i]) : ODP_EVENT_NO_SUBTYPE; - } + for (int i = 0; i < num; i++) + subtype[i] = __odp_event_subtype_get(event[i]); } _ODP_INLINE uint32_t odp_event_flow_id(odp_event_t event) diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 94461e6b1..4bc10bb38 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -495,6 +495,7 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e event_hdr->index.event = event_index; event_hdr->type = type; event_hdr->event_type = type; + event_hdr->subtype = ODP_EVENT_NO_SUBTYPE; event_hdr->pool = _odp_pool_handle(pool); /* Store base values for fast init */ From 8c2eede6bc954616477beab45dbede84d88b7b9d Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:03 +0200 Subject: [PATCH 019/147] validation: system: fix false cpu cycle wrap printout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Do not print "wrap was detected" with 64 bit cycle counters when a wrap has not actually been detected. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- test/validation/api/system/system.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/validation/api/system/system.c b/test/validation/api/system/system.c index 89a46a7f2..c074fd92b 100644 --- a/test/validation/api/system/system.c +++ b/test/validation/api/system/system.c @@ -241,7 +241,7 @@ static void system_test_cpu_cycles_long_period(void) } /* wrap was detected, no need to continue */ - if (i < PERIODS_100_MSEC) { + if (i < periods) { printf("wrap was detected.\n"); return; } From 41c1bb193dd09c2be1b29edad0aec5d05da1012c Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:05 +0200 Subject: [PATCH 020/147] validation: system: shorten cycle counter wrap-around test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check how far the cycle counter is from its maximum value before trying to test cycle counter wrap-around. If a wrap-around is not expected to occur during the test, then shorten the test to make it complete faster. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- test/validation/api/system/system.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/validation/api/system/system.c b/test/validation/api/system/system.c index c074fd92b..0ff103091 100644 --- a/test/validation/api/system/system.c +++ b/test/validation/api/system/system.c @@ -202,23 +202,27 @@ static void system_test_cpu_cycles_long_period(void) { int i; int periods = PERIODS_100_MSEC; + uint64_t max_period_duration = 100 * ODP_TIME_MSEC_IN_NS + periods - 1; uint64_t c2, c1, c3, max; uint64_t tmp, diff, res; res = odp_cpu_cycles_resolution(); max = odp_cpu_cycles_max(); + c3 = odp_cpu_cycles(); + + CU_ASSERT(c3 <= max); /* - * We can virtually never see a 64 bit cycle counter wrap around, - * so let's not even try. Use small a number of periods to speed - * up testing in this common case. + * If the cycle counter is not close to wrapping around during + * the test, then speed up the test by not trying to see the wrap + * around too hard. Assume cycle counter frequency of less than 10 GHz. */ - if (max == UINT64_MAX) + CU_ASSERT(odp_cpu_hz_max() < 10ULL * ODP_TIME_SEC_IN_NS); + if (max - c3 > 10 * periods * max_period_duration) periods = 10; printf("\n Testing CPU cycles for %i seconds... ", periods / 10); - c3 = odp_cpu_cycles(); for (i = 0; i < periods; i++) { c1 = odp_cpu_cycles(); odp_time_wait_ns(100 * ODP_TIME_MSEC_IN_NS + i); From a6c5147977f31cfc26675755153af7900e10e1c3 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:06 +0200 Subject: [PATCH 021/147] validation: scheduler: shorten wait time in wait time test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shorten wait time and wait tolerance in scheduler_test_wait_time() to speed up testing. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- test/validation/api/scheduler/scheduler.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index 8f450a057..5741a782f 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -54,9 +54,9 @@ #define CHAOS_PTR_TO_NDX(p) ((uint64_t)(uint32_t)(uintptr_t)p) #define CHAOS_NDX_TO_PTR(n) ((void *)(uintptr_t)n) -#define WAIT_TIMEOUT (1000 * ODP_TIME_MSEC_IN_NS) +#define WAIT_TIMEOUT (100 * ODP_TIME_MSEC_IN_NS) #define WAIT_ROUNDS 5 -#define WAIT_TOLERANCE (150 * ODP_TIME_MSEC_IN_NS) +#define WAIT_TOLERANCE (15 * ODP_TIME_MSEC_IN_NS) #define WAIT_1MS_RETRIES 1000 #define SCHED_AND_PLAIN_ROUNDS 10000 From f306204545009abe6f55a4dd7de1ebcb66133533 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:08 +0200 Subject: [PATCH 022/147] validation: pktio: lower wait times to speed up testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lower various wait times to receive more events from 1 second to 0.1 seconds to speed up testing. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- test/validation/api/pktio/lso.c | 8 ++++---- test/validation/api/pktio/pktio.c | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/validation/api/pktio/lso.c b/test/validation/api/pktio/lso.c index 3e033ee8b..832c08859 100644 --- a/test/validation/api/pktio/lso.c +++ b/test/validation/api/pktio/lso.c @@ -689,9 +689,9 @@ static void lso_send_custom_eth(const uint8_t *test_packet, uint32_t pkt_len, ui ODPH_DBG("\n Sent payload length: %u bytes\n", sent_payload); - /* Wait 1 sec to receive all created segments. Timeout and MAX_NUM_SEG values should be + /* Wait a bit to receive all created segments. Timeout and MAX_NUM_SEG values should be * large enough to ensure that we receive all created segments. */ - num = recv_packets(pktio_b, ODP_TIME_SEC_IN_NS, pkt_out, MAX_NUM_SEG); + num = recv_packets(pktio_b, 100 * ODP_TIME_MSEC_IN_NS, pkt_out, MAX_NUM_SEG); CU_ASSERT(num > 0); CU_ASSERT(num < MAX_NUM_SEG); @@ -814,9 +814,9 @@ static void lso_send_ipv4(const uint8_t *test_packet, uint32_t pkt_len, uint32_t ODPH_DBG("\n Sent payload length: %u bytes\n", sent_payload); - /* Wait 1 sec to receive all created segments. Timeout and MAX_NUM_SEG values should be + /* Wait a bit to receive all created segments. Timeout and MAX_NUM_SEG values should be * large enough to ensure that we receive all created segments. */ - num = recv_packets(pktio_b, ODP_TIME_SEC_IN_NS, packet, MAX_NUM_SEG); + num = recv_packets(pktio_b, 100 * ODP_TIME_MSEC_IN_NS, packet, MAX_NUM_SEG); CU_ASSERT(num > 0); CU_ASSERT(num < MAX_NUM_SEG); diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c index 780d83066..9b9150ae8 100644 --- a/test/validation/api/pktio/pktio.c +++ b/test/validation/api/pktio/pktio.c @@ -5097,7 +5097,7 @@ static void pktio_test_pktin_event_queue(odp_pktin_mode_t pktin_mode) int num_buf = 0; int num_bad = 0; odp_pktio_t pktio[MAX_NUM_IFACES] = {0}; - uint64_t wait_sec = odp_schedule_wait_time(ODP_TIME_SEC_IN_NS); + uint64_t wait_time = odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS); CU_ASSERT_FATAL(num_ifaces >= 1); @@ -5170,9 +5170,9 @@ static void pktio_test_pktin_event_queue(odp_pktin_mode_t pktin_mode) /* Receive events */ while (1) { - /* Break after 1 sec of inactivity */ + /* Break after a period of inactivity */ if (pktin_mode == ODP_PKTIN_MODE_SCHED) { - ev = odp_schedule(&from, wait_sec); + ev = odp_schedule(&from, wait_time); if (ev == ODP_EVENT_INVALID) break; From 117066572709900273f0e0e5375fd77f1e45cdbf Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:10 +0200 Subject: [PATCH 023/147] test: performance: stress: shorten the default odp_stress run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce the default run time of odp_stress to speed up make check. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- test/performance/odp_stress.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/performance/odp_stress.c b/test/performance/odp_stress.c index 4fbc303ae..3ec01df33 100644 --- a/test/performance/odp_stress.c +++ b/test/performance/odp_stress.c @@ -81,8 +81,8 @@ static void print_usage(void) "Stress test options:\n" "\n" " -c, --num_cpu Number of CPUs (worker threads). 0: all available CPUs. Default: 1\n" - " -p, --period_ns Timeout period in nsec. Default: 1 sec\n" - " -r, --rounds Number of timeout rounds. Default: 10\n" + " -p, --period_ns Timeout period in nsec. Default: 100 ms\n" + " -r, --rounds Number of timeout rounds. Default: 2\n" " -m, --mode Select test mode. Default: 1\n" " 0: No stress, just wait for timeouts\n" " 1: Memcpy\n" @@ -114,8 +114,8 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) static const char *shortopts = "+c:p:r:m:s:g:h"; test_options->num_cpu = 1; - test_options->period_ns = 1000 * ODP_TIME_MSEC_IN_NS; - test_options->rounds = 10; + test_options->period_ns = 100 * ODP_TIME_MSEC_IN_NS; + test_options->rounds = 2; test_options->mode = 1; test_options->mem_size = 2048; test_options->group_mode = 1; From db5e26e56927170dc673ff727bdb4ef45bbdd650 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:11 +0200 Subject: [PATCH 024/147] test: performance: atomic perf: lower the default number of rounds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lower the default number of test rounds to make the program complete faster during make check. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- test/performance/odp_atomic_perf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/performance/odp_atomic_perf.c b/test/performance/odp_atomic_perf.c index b87a5666c..e665081a2 100644 --- a/test/performance/odp_atomic_perf.c +++ b/test/performance/odp_atomic_perf.c @@ -24,7 +24,7 @@ #include /* Default number of test rounds */ -#define NUM_ROUNDS 1000000u +#define NUM_ROUNDS 100000u /* Initial value for atomic variables. Supports up to 2 billion * rounds of 32-bit min and max tests. */ From f4a72446f52b872eff9b2f56d22f2584607bfb6d Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:13 +0200 Subject: [PATCH 025/147] test: performance: bench_buffer: lower the default number of rounds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lower the default number of test rounds to make the test run faster in make check. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- test/performance/odp_bench_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/performance/odp_bench_buffer.c b/test/performance/odp_bench_buffer.c index 91ec73c46..ce14ec8b3 100644 --- a/test/performance/odp_bench_buffer.c +++ b/test/performance/odp_bench_buffer.c @@ -34,7 +34,7 @@ #define TEST_REPEAT_COUNT 1000 /** Default number of rounds per test case */ -#define TEST_ROUNDS 1000u +#define TEST_ROUNDS 100u /** Maximum burst size for *_multi operations */ #define TEST_MAX_BURST 64 From 49b8e735f62adc0ad7b7a31044e66d525f4ff47d Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:15 +0200 Subject: [PATCH 026/147] test: performance: bench_packet: lower the default number of rounds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lower the default number of test rounds to make the test run faster in make check. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- test/performance/odp_bench_packet.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c index 57f9ef4b9..9d6072275 100644 --- a/test/performance/odp_bench_packet.c +++ b/test/performance/odp_bench_packet.c @@ -41,7 +41,7 @@ #define TEST_REPEAT_COUNT 1000 /** Number of rounds per test case */ -#define TEST_ROUNDS 10u +#define TEST_ROUNDS 2u /** Maximum burst size for *_multi operations */ #define TEST_MAX_BURST 64 From 2c860adeae3603cbaf605454f4a03ca23660b16a Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:16 +0200 Subject: [PATCH 027/147] test: performance: scheduling: add duration option for cpu freq test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a command line option for the duration of the CPU frequency measurement. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- test/performance/odp_scheduling.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/test/performance/odp_scheduling.c b/test/performance/odp_scheduling.c index 86a6be530..15a1506a5 100644 --- a/test/performance/odp_scheduling.c +++ b/test/performance/odp_scheduling.c @@ -1,7 +1,6 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2013-2018 Linaro Limited + * Copyright (c) 2019-2023 Nokia */ /** @@ -49,6 +48,7 @@ typedef struct { /** Test arguments */ typedef struct { + double test_sec; /**< CPU frequency test duration in seconds */ unsigned int cpu_count; /**< CPU count */ int fairness; /**< Check fairness */ } test_args_t; @@ -687,17 +687,17 @@ static int run_thread(void *arg ODP_UNUSED) /** * @internal Test cycle counter frequency */ -static void test_cpu_freq(void) +static void test_cpu_freq(double test_sec) { odp_time_t cur_time, test_time, start_time, end_time; uint64_t c1, c2, cycles; uint64_t nsec; double diff_max_hz, max_cycles; - printf("\nCPU cycle count frequency test (runs about %i sec)\n", - TEST_SEC); + printf("\nCPU cycle count frequency test (runs about %f sec)\n", + test_sec); - test_time = odp_time_local_from_ns(TEST_SEC * ODP_TIME_SEC_IN_NS); + test_time = odp_time_local_from_ns(test_sec * ODP_TIME_SEC_IN_NS); start_time = odp_time_local(); end_time = odp_time_sum(start_time, test_time); @@ -734,6 +734,7 @@ static void print_usage(void) { printf("\n\nUsage: ./odp_example [options]\n"); printf("Options:\n"); + printf(" -t, --time test duration, default=%.1f\n", (double)TEST_SEC); printf(" -c, --count CPU count, 0=all available, default=1\n"); printf(" -h, --help this help\n"); printf(" -f, --fair collect fairness statistics\n"); @@ -753,15 +754,17 @@ static void parse_args(int argc, char *argv[], test_args_t *args) int long_index; static const struct option longopts[] = { + {"time", required_argument, NULL, 't'}, {"count", required_argument, NULL, 'c'}, {"fair", no_argument, NULL, 'f'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:fh"; + static const char *shortopts = "+t:c:fh"; args->cpu_count = 1; /* use one worker by default */ + args->test_sec = TEST_SEC; while (1) { opt = getopt_long(argc, argv, shortopts, longopts, &long_index); @@ -774,6 +777,10 @@ static void parse_args(int argc, char *argv[], test_args_t *args) args->fairness = 1; break; + case 't': + args->test_sec = atof(optarg); + break; + case 'c': args->cpu_count = atoi(optarg); break; @@ -864,7 +871,7 @@ int main(int argc, char *argv[]) } /* Test cycle count frequency */ - test_cpu_freq(); + test_cpu_freq(args.test_sec); shm = odp_shm_reserve("test_globals", sizeof(test_globals_t), ODP_CACHE_LINE_SIZE, 0); From d59f44ec12385614c2e3d4f55681c3e30be6e0b3 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:18 +0200 Subject: [PATCH 028/147] test: performance: scheduling: speed up cpu freq test in make check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce the duration of the CPU frequency measurement to 0.1 seconds in make check. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- test/performance/odp_scheduling_run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/performance/odp_scheduling_run.sh b/test/performance/odp_scheduling_run.sh index 2b4281ee9..4e004264e 100755 --- a/test/performance/odp_scheduling_run.sh +++ b/test/performance/odp_scheduling_run.sh @@ -19,7 +19,7 @@ run() if [ $(nproc) -lt $1 ]; then echo "Not enough CPU cores. Skipping test." else - $TEST_DIR/odp_scheduling${EXEEXT} -c $1 + $TEST_DIR/odp_scheduling${EXEEXT} -c $1 -t 0.1 RET_VAL=$? if [ $RET_VAL -ne 0 ]; then echo odp_scheduling FAILED From 0dec9527ac2cf4fd74d0e71b4be1c7213a0188c4 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 9 Jan 2024 10:39:07 +0200 Subject: [PATCH 029/147] linux-gen: test: pktio_ipc: fix help text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add missing -p/--pid option to help text and remove the stale -n/--ns option from the help. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- platform/linux-generic/test/pktio_ipc/ipc_common.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/platform/linux-generic/test/pktio_ipc/ipc_common.c b/platform/linux-generic/test/pktio_ipc/ipc_common.c index f693feeb2..2e987feec 100644 --- a/platform/linux-generic/test/pktio_ipc/ipc_common.c +++ b/platform/linux-generic/test/pktio_ipc/ipc_common.c @@ -151,15 +151,13 @@ void usage(char *progname) { printf("\n" "Usage: %s OPTIONS\n" - " E.g. -n ipc_name_space %s -t seconds\n" "\n" "OpenDataPlane odp-linux ipc test application.\n" "\n" - "Mandatory OPTIONS:\n" - " -n, --ns IPC name space ID /dev/shm/odp--objname.\n" "Optional OPTIONS\n" " -h, --help Display help and exit.\n" + " -p, --pid PID of the master process.\n" " -t, --time Time to run in seconds.\n" - "\n", NO_PATH(progname), NO_PATH(progname) + "\n", NO_PATH(progname) ); } From 48629a04a2da90b39d4577cc69caed46cae0d419 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:19 +0200 Subject: [PATCH 030/147] linux-gen: test: pktio_ipc: separate start timeout from run time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a separate parameter for the timeout used in starting and setting up the test instead of using the run time parameter for that purpose too. This enables shortening the run time without affecting the start timeout. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- .../linux-generic/test/pktio_ipc/ipc_common.c | 21 +++++++++++------ .../linux-generic/test/pktio_ipc/ipc_common.h | 10 ++++---- .../linux-generic/test/pktio_ipc/pktio_ipc1.c | 23 ++++++++++--------- .../linux-generic/test/pktio_ipc/pktio_ipc2.c | 23 ++++++++++--------- .../test/pktio_ipc/pktio_ipc_run.sh | 16 ++++++------- 5 files changed, 52 insertions(+), 41 deletions(-) diff --git a/platform/linux-generic/test/pktio_ipc/ipc_common.c b/platform/linux-generic/test/pktio_ipc/ipc_common.c index 2e987feec..128a7c6e1 100644 --- a/platform/linux-generic/test/pktio_ipc/ipc_common.c +++ b/platform/linux-generic/test/pktio_ipc/ipc_common.c @@ -1,11 +1,12 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ #include "ipc_common.h" +/** Start time in seconds */ +int start_time_sec; /** Run time in seconds */ int run_time_sec; /** Pid of the master process */ @@ -97,23 +98,28 @@ void parse_args(int argc, char *argv[]) int opt; int long_index; static struct option longopts[] = { - {"time", required_argument, NULL, 't'}, + {"start-timeout", required_argument, NULL, 's'}, + {"run-time", required_argument, NULL, 't'}, {"pid", required_argument, NULL, 'p'}, /* master process pid */ {"help", no_argument, NULL, 'h'}, /* return 'h' */ {NULL, 0, NULL, 0} }; + start_time_sec = 0; /* wait forever if time is 0 */ run_time_sec = 0; /* loop forever if time to run is 0 */ master_pid = 0; while (1) { - opt = getopt_long(argc, argv, "+t:p:h", + opt = getopt_long(argc, argv, "+s:t:p:h", longopts, &long_index); if (opt == -1) break; /* No more options */ switch (opt) { + case 's': + start_time_sec = atoi(optarg); + break; case 't': run_time_sec = atoi(optarg); break; @@ -157,7 +163,8 @@ void usage(char *progname) "Optional OPTIONS\n" " -h, --help Display help and exit.\n" " -p, --pid PID of the master process.\n" - " -t, --time Time to run in seconds.\n" + " -t, --run-time Time to run in seconds.\n" + " -s, --start-timeout Maximum time for pktio startup.\n" "\n", NO_PATH(progname) ); } diff --git a/platform/linux-generic/test/pktio_ipc/ipc_common.h b/platform/linux-generic/test/pktio_ipc/ipc_common.h index b2b469553..94ec21460 100644 --- a/platform/linux-generic/test/pktio_ipc/ipc_common.h +++ b/platform/linux-generic/test/pktio_ipc/ipc_common.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ #define _POSIX_C_SOURCE 200809L @@ -64,6 +63,9 @@ typedef struct ODP_PACKED { odp_u32be_t magic; } pkt_tail_t; +/** Start time in seconds */ +extern int start_time_sec; + /** Run time in seconds */ extern int run_time_sec; diff --git a/platform/linux-generic/test/pktio_ipc/pktio_ipc1.c b/platform/linux-generic/test/pktio_ipc/pktio_ipc1.c index 6c71e18da..df7a5ca3f 100644 --- a/platform/linux-generic/test/pktio_ipc/pktio_ipc1.c +++ b/platform/linux-generic/test/pktio_ipc/pktio_ipc1.c @@ -1,7 +1,6 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ #include "ipc_common.h" @@ -49,17 +48,17 @@ static int pktio_run_loop(odp_pool_t pool) else sprintf(name, TEST_IPC_PKTIO_NAME); - wait = odp_time_local_from_ns(run_time_sec * ODP_TIME_SEC_IN_NS); + wait = odp_time_local_from_ns(start_time_sec * ODP_TIME_SEC_IN_NS); start_cycle = odp_time_local(); current_cycle = start_cycle; for (;;) { - if (run_time_sec) { + if (start_time_sec) { cycle = odp_time_local(); diff = odp_time_diff(cycle, start_cycle); if (odp_time_cmp(wait, diff) < 0) { - printf("timeout exit, run_time_sec %d\n", - run_time_sec); + printf("timeout exit 1, start_time_sec %d\n", + start_time_sec); return -1; } } @@ -83,12 +82,12 @@ static int pktio_run_loop(odp_pool_t pool) /* start ipc pktio, i.e. wait until other process connects */ for (;;) { - if (run_time_sec) { + if (start_time_sec) { cycle = odp_time_local(); diff = odp_time_diff(cycle, start_cycle); if (odp_time_cmp(wait, diff) < 0) { - printf("timeout exit, run_time_sec %d\n", - run_time_sec); + printf("timeout exit 2, start_time_sec %d\n", + start_time_sec); goto exit; } } @@ -102,6 +101,8 @@ static int pktio_run_loop(odp_pool_t pool) } /* packets loop */ + wait = odp_time_local_from_ns(run_time_sec * ODP_TIME_SEC_IN_NS); + start_cycle = odp_time_local(); for (;;) { int i; diff --git a/platform/linux-generic/test/pktio_ipc/pktio_ipc2.c b/platform/linux-generic/test/pktio_ipc/pktio_ipc2.c index e6ca5f5e5..fc3b6833a 100644 --- a/platform/linux-generic/test/pktio_ipc/pktio_ipc2.c +++ b/platform/linux-generic/test/pktio_ipc/pktio_ipc2.c @@ -1,7 +1,6 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ /** @@ -46,17 +45,17 @@ static int ipc_second_process(int master_pid) exit(EXIT_FAILURE); } - wait = odp_time_local_from_ns(run_time_sec * ODP_TIME_SEC_IN_NS); + wait = odp_time_local_from_ns(start_time_sec * ODP_TIME_SEC_IN_NS); start_cycle = odp_time_local(); for (;;) { /* exit loop if time specified */ - if (run_time_sec) { + if (start_time_sec) { cycle = odp_time_local(); diff = odp_time_diff(cycle, start_cycle); if (odp_time_cmp(wait, diff) < 0) { - printf("timeout exit, run_time_sec %d\n", - run_time_sec); + printf("timeout exit 1, start_time_sec %d\n", + start_time_sec); goto not_started; } } @@ -85,12 +84,12 @@ static int ipc_second_process(int master_pid) /* start ipc pktio, i.e. wait until other process connects */ for (;;) { /* 1. exit loop if time specified */ - if (run_time_sec) { + if (start_time_sec) { cycle = odp_time_local(); diff = odp_time_diff(cycle, start_cycle); if (odp_time_cmp(wait, diff) < 0) { - printf("timeout exit, run_time_sec %d\n", - run_time_sec); + printf("timeout exit 2, start_time_sec %d\n", + start_time_sec); goto not_started; } } @@ -103,6 +102,8 @@ static int ipc_second_process(int master_pid) odp_time_wait_ns(50 * ODP_TIME_MSEC_IN_NS); } + wait = odp_time_local_from_ns(run_time_sec * ODP_TIME_SEC_IN_NS); + start_cycle = odp_time_local(); for (;;) { /* exit loop if time specified */ if (run_time_sec) { diff --git a/platform/linux-generic/test/pktio_ipc/pktio_ipc_run.sh b/platform/linux-generic/test/pktio_ipc/pktio_ipc_run.sh index bad2626bd..475b92db3 100755 --- a/platform/linux-generic/test/pktio_ipc/pktio_ipc_run.sh +++ b/platform/linux-generic/test/pktio_ipc/pktio_ipc_run.sh @@ -1,9 +1,8 @@ #!/bin/sh # -# Copyright (c) 2015-2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2015-2018 Linaro Limited +# Copyright (c) 2023 Nokia # # directories where test binary can be found: @@ -17,6 +16,7 @@ PATH=$(dirname $0):$PATH PATH=$(dirname $0)/../../../../platform/linux-generic/test/pktio_ipc:$PATH PATH=.:$PATH +STARTTIME=30 RUNTIME1=3 RUNTIME2=1 TIMEOUT=3 @@ -31,10 +31,10 @@ run() local ret=0 echo "==== run pktio_ipc1 then pktio_ipc2 ====" - pktio_ipc1${EXEEXT} -t ${RUNTIME1} & + pktio_ipc1${EXEEXT} -s ${STARTTIME} -t ${RUNTIME1} & IPC_PID=$! - pktio_ipc2${EXEEXT} -p ${IPC_PID} -t ${RUNTIME2} + pktio_ipc2${EXEEXT} -p ${IPC_PID} -s ${STARTTIME} -t ${RUNTIME2} ret=$? # pktio_ipc1 should do clean up and exit just # after pktio_ipc2 exited. If it does not happen @@ -58,10 +58,10 @@ run() fi echo "==== run pktio_ipc2 then pktio_ipc1 ====" - pktio_ipc2${EXEEXT} -t ${RUNTIME1} & + pktio_ipc2${EXEEXT} -s ${STARTTIME} -t ${RUNTIME1} & IPC_PID=$! - pktio_ipc1${EXEEXT} -p ${IPC_PID} -t ${RUNTIME2} + pktio_ipc1${EXEEXT} -p ${IPC_PID} -s ${STARTTIME} -t ${RUNTIME2} ret=$? # pktio_ipc2 do not exit on pktio_ipc1 disconnect # wait until it exits cleanly From 3e20ca8e1b4836cab3bd0916bb8b38dc5798ee52 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:21 +0200 Subject: [PATCH 031/147] linux-gen: test: pktio_ipc: reduce test run time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simplify the script and reduce test run time now that start time can be set separately from run time. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- .../test/pktio_ipc/pktio_ipc_run.sh | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/platform/linux-generic/test/pktio_ipc/pktio_ipc_run.sh b/platform/linux-generic/test/pktio_ipc/pktio_ipc_run.sh index 475b92db3..b181668e8 100755 --- a/platform/linux-generic/test/pktio_ipc/pktio_ipc_run.sh +++ b/platform/linux-generic/test/pktio_ipc/pktio_ipc_run.sh @@ -17,31 +17,22 @@ PATH=$(dirname $0)/../../../../platform/linux-generic/test/pktio_ipc:$PATH PATH=.:$PATH STARTTIME=30 -RUNTIME1=3 -RUNTIME2=1 -TIMEOUT=3 -if [ "${TEST}" = "coverage" ]; then - RUNTIME1=30 - RUNTIME2=15 - TIMEOUT=20 -fi +RUNTIME=1 run() { local ret=0 echo "==== run pktio_ipc1 then pktio_ipc2 ====" - pktio_ipc1${EXEEXT} -s ${STARTTIME} -t ${RUNTIME1} & + pktio_ipc1${EXEEXT} -s ${STARTTIME} -t ${RUNTIME} & IPC_PID=$! - pktio_ipc2${EXEEXT} -p ${IPC_PID} -s ${STARTTIME} -t ${RUNTIME2} + pktio_ipc2${EXEEXT} -p ${IPC_PID} -s ${STARTTIME} -t ${RUNTIME} ret=$? - # pktio_ipc1 should do clean up and exit just - # after pktio_ipc2 exited. If it does not happen - # kill him in test. - sleep ${TIMEOUT} + (kill ${IPC_PID} 2>&1 > /dev/null ) > /dev/null if [ $? -eq 0 ]; then + wait $IPC_PID echo "pktio_ipc1${EXEEXT} was killed" ls -l /dev/shm/${UID}/odp* 2> /dev/null rm -rf /dev/shm/${UID}/odp-${IPC_PID}* 2>&1 > /dev/null @@ -58,16 +49,15 @@ run() fi echo "==== run pktio_ipc2 then pktio_ipc1 ====" - pktio_ipc2${EXEEXT} -s ${STARTTIME} -t ${RUNTIME1} & + pktio_ipc2${EXEEXT} -s ${STARTTIME} -t ${RUNTIME} & IPC_PID=$! - pktio_ipc1${EXEEXT} -p ${IPC_PID} -s ${STARTTIME} -t ${RUNTIME2} + pktio_ipc1${EXEEXT} -p ${IPC_PID} -s ${STARTTIME} -t ${RUNTIME} ret=$? - # pktio_ipc2 do not exit on pktio_ipc1 disconnect - # wait until it exits cleanly - sleep ${TIMEOUT} + (kill ${IPC_PID} 2>&1 > /dev/null ) > /dev/null if [ $? -eq 0 ]; then + wait $IPC_PID echo "pktio_ipc2${EXEEXT} was killed" ls -l /dev/shm/${UID}/odp* 2> /dev/null rm -rf /dev/shm/${UID}/odp-${IPC_PID}* 2>&1 > /dev/null From daf37c32acac2d30e94037fc41d13dd1cbd3613c Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:22 +0200 Subject: [PATCH 032/147] example: timer_simple: speed up the program by using a shorter timer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use a shorter time out in the timer to make the program complete faster during make check. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- example/timer/odp_timer_simple.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/example/timer/odp_timer_simple.c b/example/timer/odp_timer_simple.c index e3afed5be..ceba66c62 100644 --- a/example/timer/odp_timer_simple.c +++ b/example/timer/odp_timer_simple.c @@ -121,9 +121,8 @@ int main(int argc ODP_UNUSED, char *argv[] ODP_UNUSED) ev = odp_timeout_to_event(tmo); - /* Calculate period for timer in uint64_t value, in current case - * we will schedule timer for 1 second */ - period = odp_timer_ns_to_tick(timer_pool, 1 * ODP_TIME_SEC_IN_NS); + /* Calculate timer period in ticks */ + period = odp_timer_ns_to_tick(timer_pool, 100 * ODP_TIME_MSEC_IN_NS); /* Wait time to return from odp_schedule() if there are no * events @@ -159,7 +158,7 @@ int main(int argc ODP_UNUSED, char *argv[] ODP_UNUSED) i, odp_time_to_ns(time)); /* Do not free current event, just go back to loop and program - * timeout to next second. + * the next timeout. */ } From 0be8686b5b7e5329ecf924bb614dd45180bc53ee Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:24 +0200 Subject: [PATCH 033/147] example: tm: send fewer packets to speed up the program MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Send fewer packets to make the program complete faster during make check. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- example/traffic_mgmt/odp_traffic_mgmt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/traffic_mgmt/odp_traffic_mgmt.c b/example/traffic_mgmt/odp_traffic_mgmt.c index 3ca70b950..38c39bd5c 100644 --- a/example/traffic_mgmt/odp_traffic_mgmt.c +++ b/example/traffic_mgmt/odp_traffic_mgmt.c @@ -292,7 +292,7 @@ static uint32_t next_rand_byte; static odp_atomic_u32_t atomic_pkts_into_tm; static odp_atomic_u32_t atomic_pkts_from_tm; -static uint32_t g_num_pkts_to_send = 1000; +static uint32_t g_num_pkts_to_send = 100; static uint8_t g_print_tm_stats = TRUE; static void tester_egress_fcn(odp_packet_t odp_pkt); From e86b978a5c04a515ecc3d1b190fb20896db533a3 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:25 +0200 Subject: [PATCH 034/147] example: pktio: allow decimal run time parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow decimal run time parameter to enable sub-second run times. Also reduce the wait time for queue cleanup at the end. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- example/packet/odp_pktio.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/example/packet/odp_pktio.c b/example/packet/odp_pktio.c index 6d66185e0..a82537fe7 100644 --- a/example/packet/odp_pktio.c +++ b/example/packet/odp_pktio.c @@ -72,7 +72,7 @@ typedef struct { char **if_names; /**< Array of pointers to interface names */ int mode; /**< Packet IO mode */ char *if_str; /**< Storage for interface names */ - int time; /**< Time to run app */ + double time; /**< Time to run app */ } appl_args_t; /** @@ -479,7 +479,7 @@ int main(int argc, char *argv[]) odp_pktio_stop(pktio); } /* use delay to let workers clean up queues */ - odp_time_wait_ns(ODP_TIME_SEC_IN_NS); + odp_time_wait_ns(100 * ODP_TIME_MSEC_IN_NS); odp_atomic_store_u32(&args->exit_threads, 1); } @@ -615,7 +615,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->cpu_count = atoi(optarg); break; case 't': - appl_args->time = atoi(optarg); + appl_args->time = atof(optarg); break; /* parse packet-io interface names */ case 'i': @@ -742,7 +742,7 @@ static void usage(char *progname) "\n" "Optional OPTIONS\n" " -c, --count CPU count, 0=all available, default=1\n" - " -t, --time Number of seconds to run.\n" + " -t, --time Number of seconds to run (e.g. 0.1).\n" " -m, --mode 0: Receive and send directly (no queues)\n" " 1: Receive and send via queues.\n" " 2: Receive via scheduler, send via queues.\n" From 812a53cb45ebaa2459a23a6cede194dfcf0f5954 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 21 Dec 2023 13:34:27 +0200 Subject: [PATCH 035/147] example: pktio: reduce test run time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce test run time during make check. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale Reviewed-by: Jere Leppänen --- example/packet/pktio_run.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/example/packet/pktio_run.sh b/example/packet/pktio_run.sh index bb1854b43..f08c45936 100755 --- a/example/packet/pktio_run.sh +++ b/example/packet/pktio_run.sh @@ -4,6 +4,8 @@ # Copyright (c) 2016-2018 Linaro Limited # +TEST_TIME=0.1 + if [ -f ./pktio_env ]; then . ./pktio_env else @@ -15,7 +17,7 @@ fi setup_interfaces # burst mode -./odp_pktio${EXEEXT} -i $IF1 -t 1 -m 0 +./odp_pktio${EXEEXT} -i $IF1 -t $TEST_TIME -m 0 STATUS=$? if [ ${STATUS} -ne 0 ]; then echo "Error: status ${STATUS}" @@ -26,7 +28,7 @@ validate_result echo "Pass -m 0: status ${STATUS}" # queue mode -./odp_pktio${EXEEXT} -i $IF1 -t 1 -m 1 +./odp_pktio${EXEEXT} -i $IF1 -t $TEST_TIME -m 1 STATUS=$? if [ ${STATUS} -ne 0 ]; then @@ -38,7 +40,7 @@ validate_result echo "Pass -m 1: status ${STATUS}" # sched/queue mode -./odp_pktio${EXEEXT} -i $IF1 -t 1 -m 2 +./odp_pktio${EXEEXT} -i $IF1 -t $TEST_TIME -m 2 STATUS=$? if [ ${STATUS} -ne 0 ]; then @@ -50,7 +52,7 @@ validate_result echo "Pass -m 2: status ${STATUS}" # cpu number option test 1 -./odp_pktio${EXEEXT} -i $IF1 -t 1 -m 0 -c 1 +./odp_pktio${EXEEXT} -i $IF1 -t $TEST_TIME -m 0 -c 1 STATUS=$? if [ ${STATUS} -ne 0 ]; then From 43d74f57b2a9e8270e1c1b3831c8a5334bd4182a Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 12 Jan 2024 17:30:49 +0200 Subject: [PATCH 036/147] github_ci: update github actions Update all x86 GitHub Actions into their latest release versions. Project documentation build runner is updated to Ubuntu 22.04 for newer Doxygen version. Arm64 runners have been held back due to missing host system dependency (`GLIBC_2.28'). Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .github/workflows/ci-pipeline.yml | 62 +++++++++++++++---------------- .github/workflows/coverity.yml | 2 +- .github/workflows/gh-pages.yml | 6 +-- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 60d126d3f..72b84eff3 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -11,7 +11,7 @@ jobs: Checkpatch: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Install dependencies @@ -22,7 +22,7 @@ jobs: if: github.event_name == 'pull_request' env: CHECKPATCH_COMMAND: ./scripts/checkpatch.pl - uses: webispy/checkpatch-action@v8 + uses: webispy/checkpatch-action@v9 - name: Check push if: github.event_name == 'push' && github.ref != 'refs/heads/master' run: | @@ -38,7 +38,7 @@ jobs: Documentation: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install dependencies run: | sudo apt update @@ -72,7 +72,7 @@ jobs: '--enable-lto --enable-abi-compat', '--enable-pcapng-support'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=gcc -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -92,7 +92,7 @@ jobs: '--enable-pcapng-support', '--without-openssl --without-pcap'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=clang -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -108,7 +108,7 @@ jobs: cc_ver: [9] conf: ['--disable-shared', '--enable-lto --disable-shared'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="gcc-${{matrix.cc_ver}}" -e CXX="g++-${{matrix.cc_ver}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_static.sh - if: ${{ failure() }} @@ -125,7 +125,7 @@ jobs: cc_ver: [10, 11, 12] conf: ['', '--enable-lto'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="gcc-${{matrix.cc_ver}}" -e CXX="g++-${{matrix.cc_ver}}" -e CONF="${CONF} ${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_static.sh - if: ${{ failure() }} @@ -143,7 +143,7 @@ jobs: conf: ['', '--enable-abi-compat', 'CFLAGS=-march=armv8.2-a', 'CFLAGS=-march=armv8-a+lse', '--with-crypto=armv8crypto', '--enable-wfe-locks'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${CONF} ${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -159,7 +159,7 @@ jobs: cc: [gcc, clang] conf: ['', '--enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -176,7 +176,7 @@ jobs: cc: [gcc, clang] conf: ['', '--enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${CONF} ${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -193,7 +193,7 @@ jobs: cc: [gcc, clang] conf: ['', '--enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -209,7 +209,7 @@ jobs: cc: [gcc] conf: ['', '--enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -224,7 +224,7 @@ jobs: os: ['centos_7', 'rocky_linux_8'] conf: ['--enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -240,7 +240,7 @@ jobs: cc_ver: [7, 8] conf: ['', '--enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="gcc-${{matrix.cc_ver}}" -e CXX="g++-${{matrix.cc_ver}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -256,7 +256,7 @@ jobs: cc_ver: [10, 11, 12, 13] conf: ['', '--enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="gcc-${{matrix.cc_ver}}" -e CXX="g++-${{matrix.cc_ver}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -265,7 +265,7 @@ jobs: Build_out-of-tree: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/out_of_tree.sh - if: ${{ failure() }} @@ -281,7 +281,7 @@ jobs: matrix: cc: [gcc, clang] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -294,7 +294,7 @@ jobs: matrix: conf: ['--enable-user-guides', '--enable-user-guides --enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/distcheck.sh - if: ${{ failure() }} @@ -313,7 +313,7 @@ jobs: '--disable-host-optimization --enable-abi-compat', '--without-openssl --without-pcap'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=gcc -e ARCH="${ARCH}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -333,7 +333,7 @@ jobs: '--disable-host-optimization --enable-event-validation=warn', '--disable-host-optimization --enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=clang -e ARCH="${ARCH}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -347,7 +347,7 @@ jobs: cc: [gcc, clang] os: ['ubuntu_22.04'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e ARCH="${ARCH}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH} /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -356,7 +356,7 @@ jobs: Run_sched_config: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-generic/test/sched-basic.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -365,7 +365,7 @@ jobs: Run_stash_config: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-generic/test/stash-custom.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -374,7 +374,7 @@ jobs: Run_scheduler_sp: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_SCHEDULER=sp $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -383,7 +383,7 @@ jobs: Run_scheduler_scalable: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_SCHEDULER=scalable -e CI_SKIP=pktio_test_pktin_event_sched $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -392,7 +392,7 @@ jobs: Run_process_mode: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-generic/test/process-mode.conf -e ODPH_PROC_MODE=1 $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh @@ -402,7 +402,7 @@ jobs: Run_inline_timer: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-generic/test/inline-timer.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check_inline_timer.sh @@ -412,7 +412,7 @@ jobs: Run_packet_align: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-generic/test/packet_align.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check_pktio.sh @@ -422,7 +422,7 @@ jobs: Run_dpdk-19_11: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-dpdk_19.11 /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -431,7 +431,7 @@ jobs: Run_dpdk-20_11: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-dpdk_20.11 /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -440,7 +440,7 @@ jobs: Run_dpdk-21_11: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-dpdk_21.11 /odp/scripts/ci/check.sh - if: ${{ failure() }} diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index bb8347a8d..136077c1f 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -15,7 +15,7 @@ jobs: Coverity-analysis: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e GITHUB_SHA="${GITHUB_SHA}" -e COVERITY_TOKEN="${{ secrets.COVERITY_TOKEN }}" diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 7031da5d3..fd7664a85 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -7,9 +7,9 @@ on: jobs: Documentation: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install dependencies run: | sudo apt update @@ -44,7 +44,7 @@ jobs: - name: Deploy env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: crazy-max/ghaction-github-pages@v3 + uses: crazy-max/ghaction-github-pages@v4 with: allow_empty_commit: false build_dir: ./doc/gh-pages From cf0fca8a1963c9ad939e8b0cce986c850a41c6ca Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 4 Jan 2024 15:38:15 +0200 Subject: [PATCH 037/147] test: cunit: make odp_cunit_parse_options() update argc Have odp_cunit_parse_options() update the option count after it has removed the options it handled. This enables the callers to have their own command line option processing after odp_cunit_parse_options() as probably originally intended. Signed-off-by: Janne Peltonen Reviewed-by: Matias Elo --- platform/linux-generic/test/validation/api/shmem/shmem_odp1.c | 2 +- test/common/odp_cunit_common.c | 4 ++-- test/common/odp_cunit_common.h | 2 +- test/validation/api/atomic/atomic.c | 2 +- test/validation/api/barrier/barrier.c | 2 +- test/validation/api/buffer/buffer.c | 2 +- test/validation/api/chksum/chksum.c | 2 +- test/validation/api/classification/classification.c | 2 +- test/validation/api/comp/comp.c | 2 +- test/validation/api/cpumask/cpumask.c | 2 +- test/validation/api/crypto/odp_crypto_test_inp.c | 2 +- test/validation/api/dma/dma.c | 2 +- test/validation/api/errno/errno.c | 2 +- test/validation/api/event/event.c | 2 +- test/validation/api/hash/hash.c | 2 +- test/validation/api/init/init_main.c | 2 +- test/validation/api/ipsec/ipsec_async.c | 2 +- test/validation/api/ipsec/ipsec_inline_in.c | 2 +- test/validation/api/ipsec/ipsec_inline_out.c | 2 +- test/validation/api/ipsec/ipsec_sync.c | 2 +- test/validation/api/lock/lock.c | 2 +- test/validation/api/packet/packet.c | 2 +- test/validation/api/pktio/pktio.c | 2 +- test/validation/api/pool/pool.c | 2 +- test/validation/api/queue/queue.c | 2 +- test/validation/api/random/random.c | 2 +- test/validation/api/scheduler/scheduler.c | 2 +- test/validation/api/scheduler/scheduler_no_predef_groups.c | 2 +- test/validation/api/shmem/shmem.c | 2 +- test/validation/api/stash/stash.c | 2 +- test/validation/api/std/std.c | 2 +- test/validation/api/system/system.c | 2 +- test/validation/api/thread/thread.c | 2 +- test/validation/api/time/time.c | 2 +- test/validation/api/timer/timer.c | 2 +- test/validation/api/traffic_mngr/traffic_mngr.c | 2 +- 36 files changed, 37 insertions(+), 37 deletions(-) diff --git a/platform/linux-generic/test/validation/api/shmem/shmem_odp1.c b/platform/linux-generic/test/validation/api/shmem/shmem_odp1.c index 3b4ba819c..98148d6c7 100644 --- a/platform/linux-generic/test/validation/api/shmem/shmem_odp1.c +++ b/platform/linux-generic/test/validation/api/shmem/shmem_odp1.c @@ -79,7 +79,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(shmem_suites); diff --git a/test/common/odp_cunit_common.c b/test/common/odp_cunit_common.c index a5b9ff1ea..651ae791e 100644 --- a/test/common/odp_cunit_common.c +++ b/test/common/odp_cunit_common.c @@ -708,13 +708,13 @@ int odp_cunit_register(odp_suiteinfo_t testsuites[]) * (hence also helpers options as cunit_common uses the helpers) * Options private to the test calling cunit_common are not parsed here. */ -int odp_cunit_parse_options(int argc, char *argv[]) +int odp_cunit_parse_options(int *argc, char *argv[]) { const char *ctrl_thread_env = getenv("CI_THREAD_TYPE_CONTROL"); const char *env = getenv("CI"); progname = argv[0]; - odph_parse_options(argc, argv); + *argc = odph_parse_options(*argc, argv); /* Check if we need to use control thread */ if (ctrl_thread_env && !strcmp(ctrl_thread_env, "true")) control_thread = true; diff --git a/test/common/odp_cunit_common.h b/test/common/odp_cunit_common.h index 77822ee60..63e95d5fb 100644 --- a/test/common/odp_cunit_common.h +++ b/test/common/odp_cunit_common.h @@ -66,7 +66,7 @@ typedef struct { } test_shared_data_t; /* parse parameters that affect the behaviour of odp_cunit_common */ -int odp_cunit_parse_options(int argc, char *argv[]); +int odp_cunit_parse_options(int *argc, char *argv[]); /* register suites to be run via odp_cunit_run() */ int odp_cunit_register(odp_suiteinfo_t testsuites[]); /* update tests previously registered via odp_cunit_register() */ diff --git a/test/validation/api/atomic/atomic.c b/test/validation/api/atomic/atomic.c index 76e3c0d49..fab982462 100644 --- a/test/validation/api/atomic/atomic.c +++ b/test/validation/api/atomic/atomic.c @@ -1702,7 +1702,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(atomic_init); diff --git a/test/validation/api/barrier/barrier.c b/test/validation/api/barrier/barrier.c index e4fba770f..7dc9a44c6 100644 --- a/test/validation/api/barrier/barrier.c +++ b/test/validation/api/barrier/barrier.c @@ -444,7 +444,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(barrier_init); diff --git a/test/validation/api/buffer/buffer.c b/test/validation/api/buffer/buffer.c index 91cfbfb5f..2a79ed27e 100644 --- a/test/validation/api/buffer/buffer.c +++ b/test/validation/api/buffer/buffer.c @@ -598,7 +598,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(buffer_suites); diff --git a/test/validation/api/chksum/chksum.c b/test/validation/api/chksum/chksum.c index 86306ab0b..0be418f3a 100644 --- a/test/validation/api/chksum/chksum.c +++ b/test/validation/api/chksum/chksum.c @@ -442,7 +442,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(chksum_suites); diff --git a/test/validation/api/classification/classification.c b/test/validation/api/classification/classification.c index ef35377dc..ef9a647cb 100644 --- a/test/validation/api/classification/classification.c +++ b/test/validation/api/classification/classification.c @@ -36,7 +36,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(classification_suites); diff --git a/test/validation/api/comp/comp.c b/test/validation/api/comp/comp.c index cb771c989..b7dfcd359 100644 --- a/test/validation/api/comp/comp.c +++ b/test/validation/api/comp/comp.c @@ -558,7 +558,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(comp_init); diff --git a/test/validation/api/cpumask/cpumask.c b/test/validation/api/cpumask/cpumask.c index b358a5be6..db500df3a 100644 --- a/test/validation/api/cpumask/cpumask.c +++ b/test/validation/api/cpumask/cpumask.c @@ -188,7 +188,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(cpumask_suites); diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index d5f2e3e7a..532aaf525 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -2399,7 +2399,7 @@ int main(int argc, char *argv[]) printf("Test mode: %s\n", full_test ? "full" : "partial"); /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(crypto_init); diff --git a/test/validation/api/dma/dma.c b/test/validation/api/dma/dma.c index 4f454168d..efc7fa039 100644 --- a/test/validation/api/dma/dma.c +++ b/test/validation/api/dma/dma.c @@ -1693,7 +1693,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(dma_suites); diff --git a/test/validation/api/errno/errno.c b/test/validation/api/errno/errno.c index ed2ab391d..70708ce01 100644 --- a/test/validation/api/errno/errno.c +++ b/test/validation/api/errno/errno.c @@ -33,7 +33,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(errno_suites); diff --git a/test/validation/api/event/event.c b/test/validation/api/event/event.c index f6ad86365..fbcc08d6f 100644 --- a/test/validation/api/event/event.c +++ b/test/validation/api/event/event.c @@ -461,7 +461,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(event_suites); diff --git a/test/validation/api/hash/hash.c b/test/validation/api/hash/hash.c index 5e4209800..a935ef7ac 100644 --- a/test/validation/api/hash/hash.c +++ b/test/validation/api/hash/hash.c @@ -753,7 +753,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(hash_suites); diff --git a/test/validation/api/init/init_main.c b/test/validation/api/init/init_main.c index 64cefa30b..f2a141b5d 100644 --- a/test/validation/api/init/init_main.c +++ b/test/validation/api/init/init_main.c @@ -302,7 +302,7 @@ int main(int argc, char *argv[]) return -1; /* Parse common options */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; /* Prevent default ODP init */ diff --git a/test/validation/api/ipsec/ipsec_async.c b/test/validation/api/ipsec/ipsec_async.c index 44c968a4d..053de32ec 100644 --- a/test/validation/api/ipsec/ipsec_async.c +++ b/test/validation/api/ipsec/ipsec_async.c @@ -41,7 +41,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(ipsec_async_init); diff --git a/test/validation/api/ipsec/ipsec_inline_in.c b/test/validation/api/ipsec/ipsec_inline_in.c index cbb3a178a..8deb3c8df 100644 --- a/test/validation/api/ipsec/ipsec_inline_in.c +++ b/test/validation/api/ipsec/ipsec_inline_in.c @@ -40,7 +40,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(ipsec_sync_init); diff --git a/test/validation/api/ipsec/ipsec_inline_out.c b/test/validation/api/ipsec/ipsec_inline_out.c index 3da19892b..89035f37b 100644 --- a/test/validation/api/ipsec/ipsec_inline_out.c +++ b/test/validation/api/ipsec/ipsec_inline_out.c @@ -40,7 +40,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(ipsec_sync_init); diff --git a/test/validation/api/ipsec/ipsec_sync.c b/test/validation/api/ipsec/ipsec_sync.c index 74ae1fe19..68aa76e4c 100644 --- a/test/validation/api/ipsec/ipsec_sync.c +++ b/test/validation/api/ipsec/ipsec_sync.c @@ -35,7 +35,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(ipsec_sync_init); diff --git a/test/validation/api/lock/lock.c b/test/validation/api/lock/lock.c index 78c6ee79b..a4e6932c4 100644 --- a/test/validation/api/lock/lock.c +++ b/test/validation/api/lock/lock.c @@ -1246,7 +1246,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(lock_init); diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index 79b8fbe50..6ae558d09 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -4591,7 +4591,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(packet_suites); diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c index 9b9150ae8..c9f8f98c3 100644 --- a/test/validation/api/pktio/pktio.c +++ b/test/validation/api/pktio/pktio.c @@ -5492,7 +5492,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(pktio_suites); diff --git a/test/validation/api/pool/pool.c b/test/validation/api/pool/pool.c index ee8aa4b67..c78931d5c 100644 --- a/test/validation/api/pool/pool.c +++ b/test/validation/api/pool/pool.c @@ -2373,7 +2373,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(pool_suites); diff --git a/test/validation/api/queue/queue.c b/test/validation/api/queue/queue.c index cb2b147ac..4b5ccde65 100644 --- a/test/validation/api/queue/queue.c +++ b/test/validation/api/queue/queue.c @@ -1164,7 +1164,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(queue_suites); diff --git a/test/validation/api/random/random.c b/test/validation/api/random/random.c index 97e367678..e8b4d962a 100644 --- a/test/validation/api/random/random.c +++ b/test/validation/api/random/random.c @@ -527,7 +527,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(random_suites); diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index 5741a782f..dac4becbd 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -3795,7 +3795,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(global_init); diff --git a/test/validation/api/scheduler/scheduler_no_predef_groups.c b/test/validation/api/scheduler/scheduler_no_predef_groups.c index 1b941ac61..ad6f6d3a2 100644 --- a/test/validation/api/scheduler/scheduler_no_predef_groups.c +++ b/test/validation/api/scheduler/scheduler_no_predef_groups.c @@ -212,7 +212,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(global_init); diff --git a/test/validation/api/shmem/shmem.c b/test/validation/api/shmem/shmem.c index 257e0214e..9e91dab35 100644 --- a/test/validation/api/shmem/shmem.c +++ b/test/validation/api/shmem/shmem.c @@ -1163,7 +1163,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(shmem_suites); diff --git a/test/validation/api/stash/stash.c b/test/validation/api/stash/stash.c index 67c336a2f..162697ba9 100644 --- a/test/validation/api/stash/stash.c +++ b/test/validation/api/stash/stash.c @@ -1385,7 +1385,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(stash_suites); diff --git a/test/validation/api/std/std.c b/test/validation/api/std/std.c index 56d05a4b4..161ee87cf 100644 --- a/test/validation/api/std/std.c +++ b/test/validation/api/std/std.c @@ -97,7 +97,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(std_suites); diff --git a/test/validation/api/system/system.c b/test/validation/api/system/system.c index 0ff103091..3f7e0497d 100644 --- a/test/validation/api/system/system.c +++ b/test/validation/api/system/system.c @@ -687,7 +687,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(system_suites); diff --git a/test/validation/api/thread/thread.c b/test/validation/api/thread/thread.c index 840256fcf..ad9ffa745 100644 --- a/test/validation/api/thread/thread.c +++ b/test/validation/api/thread/thread.c @@ -255,7 +255,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(thread_global_init); diff --git a/test/validation/api/time/time.c b/test/validation/api/time/time.c index 8d3481e26..ef406a025 100644 --- a/test/validation/api/time/time.c +++ b/test/validation/api/time/time.c @@ -1007,7 +1007,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(time_global_init); diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index 28b1399a2..281a8d763 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -3262,7 +3262,7 @@ int main(int argc, char *argv[]) int ret = 0; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; if (global_init()) diff --git a/test/validation/api/traffic_mngr/traffic_mngr.c b/test/validation/api/traffic_mngr/traffic_mngr.c index ed07db751..b7f546dcd 100644 --- a/test/validation/api/traffic_mngr/traffic_mngr.c +++ b/test/validation/api/traffic_mngr/traffic_mngr.c @@ -5025,7 +5025,7 @@ odp_suiteinfo_t traffic_mngr_suites[] = { int main(int argc, char *argv[]) { /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; int ret = odp_cunit_register(traffic_mngr_suites); From 830429333cb58dd6e59a0f0b6712f81c0fd9a7ee Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 5 Jan 2024 16:29:35 +0200 Subject: [PATCH 038/147] validation: fix TEST_DIR value Fix the value of the currently unused TEST_DIR environment variable passed to test programs. Signed-off-by: Janne Peltonen Reviewed-by: Matias Elo --- test/validation/api/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/validation/api/Makefile.am b/test/validation/api/Makefile.am index ade387152..70e322ff0 100644 --- a/test/validation/api/Makefile.am +++ b/test/validation/api/Makefile.am @@ -31,7 +31,7 @@ ODP_MODULES = atomic \ SUBDIRS = $(ODP_MODULES) include $(top_srcdir)/test/Makefile.inc -TESTS_ENVIRONMENT += TEST_DIR=${top_builddir}/test/validation +TESTS_ENVIRONMENT += TEST_DIR=${top_builddir}/test/validation/api TESTS = \ atomic/atomic_main$(EXEEXT) \ From da8f0d9038b37f4165a3ac053c33e5300118109c Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 4 Jan 2024 15:38:16 +0200 Subject: [PATCH 039/147] validation: ipsec: combine tests to single test binary Do not make separate binaries for different ipsec operation modes but use single binary that takes the operation mode as a parameter. Also simplify the related code a bit. Signed-off-by: Janne Peltonen Reviewed-by: Matias Elo --- test/validation/api/Makefile.am | 8 +- test/validation/api/ipsec/.gitignore | 5 +- test/validation/api/ipsec/Makefile.am | 17 ++-- test/validation/api/ipsec/ipsec.c | 32 +++---- test/validation/api/ipsec/ipsec.h | 5 +- test/validation/api/ipsec/ipsec_async.c | 55 ------------ test/validation/api/ipsec/ipsec_async.sh | 3 + test/validation/api/ipsec/ipsec_inline_in.c | 54 ------------ test/validation/api/ipsec/ipsec_inline_in.sh | 3 + test/validation/api/ipsec/ipsec_inline_out.c | 54 ------------ test/validation/api/ipsec/ipsec_inline_out.sh | 3 + test/validation/api/ipsec/ipsec_main.c | 83 +++++++++++++++++++ test/validation/api/ipsec/ipsec_sync.c | 49 ----------- test/validation/api/ipsec/ipsec_sync.sh | 3 + 14 files changed, 121 insertions(+), 253 deletions(-) delete mode 100644 test/validation/api/ipsec/ipsec_async.c create mode 100755 test/validation/api/ipsec/ipsec_async.sh delete mode 100644 test/validation/api/ipsec/ipsec_inline_in.c create mode 100755 test/validation/api/ipsec/ipsec_inline_in.sh delete mode 100644 test/validation/api/ipsec/ipsec_inline_out.c create mode 100755 test/validation/api/ipsec/ipsec_inline_out.sh create mode 100644 test/validation/api/ipsec/ipsec_main.c delete mode 100644 test/validation/api/ipsec/ipsec_sync.c create mode 100755 test/validation/api/ipsec/ipsec_sync.sh diff --git a/test/validation/api/Makefile.am b/test/validation/api/Makefile.am index 70e322ff0..abafa1e10 100644 --- a/test/validation/api/Makefile.am +++ b/test/validation/api/Makefile.am @@ -55,10 +55,10 @@ TESTS = \ init/init_feature_disabled$(EXEEXT) \ init/init_test_param_init$(EXEEXT) \ init/init_test_term_abnormal$(EXEEXT) \ - ipsec/ipsec_sync$(EXEEXT) \ - ipsec/ipsec_async$(EXEEXT) \ - ipsec/ipsec_inline_in$(EXEEXT) \ - ipsec/ipsec_inline_out$(EXEEXT) \ + ipsec/ipsec_sync.sh \ + ipsec/ipsec_async.sh \ + ipsec/ipsec_inline_in.sh \ + ipsec/ipsec_inline_out.sh \ lock/lock_main$(EXEEXT) \ packet/packet_main$(EXEEXT) \ pktio/pktio_main$(EXEEXT) \ diff --git a/test/validation/api/ipsec/.gitignore b/test/validation/api/ipsec/.gitignore index 967356064..2def047f3 100644 --- a/test/validation/api/ipsec/.gitignore +++ b/test/validation/api/ipsec/.gitignore @@ -1,4 +1 @@ -ipsec_sync -ipsec_async -ipsec_inline_in -ipsec_inline_out +ipsec_main diff --git a/test/validation/api/ipsec/Makefile.am b/test/validation/api/ipsec/Makefile.am index 1a29d687d..51b50dd02 100644 --- a/test/validation/api/ipsec/Makefile.am +++ b/test/validation/api/ipsec/Makefile.am @@ -11,14 +11,15 @@ libtestipsec_la_SOURCES = \ reass_test_vectors.c test_PROGRAMS = \ - ipsec_sync \ - ipsec_async \ - ipsec_inline_in \ - ipsec_inline_out + ipsec_main -ipsec_sync_SOURCES = ipsec_sync.c -ipsec_async_SOURCES = ipsec_async.c -ipsec_inline_in_SOURCES = ipsec_inline_in.c -ipsec_inline_out_SOURCES = ipsec_inline_out.c +ipsec_main_SOURCES = \ + ipsec_main.c PRELDADD += libtestipsec.la + +EXTRA_DIST = \ + ipsec_sync.sh \ + ipsec_async.sh \ + ipsec_inline_in.sh \ + ipsec_inline_out.sh diff --git a/test/validation/api/ipsec/ipsec.c b/test/validation/api/ipsec/ipsec.c index 9849a44b5..5ad7bd48d 100644 --- a/test/validation/api/ipsec/ipsec.c +++ b/test/validation/api/ipsec/ipsec.c @@ -29,7 +29,6 @@ static odp_ipsec_capability_t capa; static int sched_ev_buffer_tail; odp_bool_t sa_expiry_notified; - #define PKT_POOL_NUM 64 #define EVENT_WAIT_TIME ODP_TIME_SEC_IN_NS #define STATUS_EVENT_WAIT_TIME ODP_TIME_MSEC_IN_NS @@ -38,6 +37,8 @@ odp_bool_t sa_expiry_notified; #define PACKET_USER_PTR ((void *)0x1212fefe) #define IPSEC_SA_CTX ((void *)0xfefefafa) +static int ipsec_config(void); + static odp_pktio_t pktio_create(odp_pool_t pool) { odp_pktio_t pktio; @@ -1296,16 +1297,6 @@ int ipsec_suite_term(void) return 0; } -int ipsec_in_term(void) -{ - return ipsec_suite_term(); -} - -int ipsec_out_term(void) -{ - return ipsec_suite_term(); -} - static odp_queue_t sched_queue_create(const char *name) { odp_queue_param_t qparam; @@ -1366,12 +1357,10 @@ int ipsec_suite_sched_init(void) return ipsec_suite_init(); } -int ipsec_init(odp_instance_t *inst, odp_ipsec_op_mode_t mode) +int ipsec_init(odp_instance_t *inst) { odp_pool_param_t params; - odp_pool_t pool; odp_pool_capability_t pool_capa; - odp_pktio_t pktio; odp_init_t init_param; odph_helper_options_t helper_options; @@ -1427,25 +1416,26 @@ int ipsec_init(odp_instance_t *inst, odp_ipsec_op_mode_t mode) return -1; } - pool = odp_pool_create("packet_pool", ¶ms); + suite_context.pool = odp_pool_create("packet_pool", ¶ms); - if (ODP_POOL_INVALID == pool) { + if (suite_context.pool == ODP_POOL_INVALID) { ODPH_ERR("Packet pool creation failed\n"); return -1; } - if (mode == ODP_IPSEC_OP_MODE_INLINE) { - pktio = pktio_create(pool); - if (ODP_PKTIO_INVALID == pktio) { + if (suite_context.inbound_op_mode == ODP_IPSEC_OP_MODE_INLINE || + suite_context.outbound_op_mode == ODP_IPSEC_OP_MODE_INLINE) { + suite_context.pktio = pktio_create(suite_context.pool); + if (suite_context.pktio == ODP_PKTIO_INVALID) { ODPH_ERR("IPsec pktio creation failed\n"); return -1; } } - return 0; + return ipsec_config(); } -int ipsec_config(odp_instance_t ODP_UNUSED inst) +static int ipsec_config(void) { odp_ipsec_config_t ipsec_config; diff --git a/test/validation/api/ipsec/ipsec.h b/test/validation/api/ipsec/ipsec.h index 8eed65c50..47612e3b3 100644 --- a/test/validation/api/ipsec/ipsec.h +++ b/test/validation/api/ipsec/ipsec.h @@ -20,9 +20,8 @@ extern odp_testinfo_t ipsec_in_suite[]; extern odp_testinfo_t ipsec_out_suite[]; -int ipsec_init(odp_instance_t *inst, odp_ipsec_op_mode_t mode); +int ipsec_init(odp_instance_t *inst); int ipsec_term(odp_instance_t inst); -int ipsec_config(odp_instance_t inst); int ipsec_in_inline_init(void); int ipsec_out_inline_init(void); @@ -31,8 +30,6 @@ int ipsec_suite_sync_init(void); int ipsec_suite_plain_init(void); int ipsec_suite_sched_init(void); int ipsec_suite_term(void); -int ipsec_in_term(void); -int ipsec_out_term(void); struct suite_context_s { odp_bool_t reass_ipv4; diff --git a/test/validation/api/ipsec/ipsec_async.c b/test/validation/api/ipsec/ipsec_async.c deleted file mode 100644 index 053de32ec..000000000 --- a/test/validation/api/ipsec/ipsec_async.c +++ /dev/null @@ -1,55 +0,0 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include "ipsec.h" - -static int ipsec_async_init(odp_instance_t *inst) -{ - int rc; - - suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; - suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; - - rc = ipsec_init(inst, ODP_IPSEC_OP_MODE_ASYNC); - if (rc != 0) - return rc; - - suite_context.pool = odp_pool_lookup("packet_pool"); - if (suite_context.pool == ODP_POOL_INVALID) - return -1; - - return ipsec_config(*inst); -} - -odp_suiteinfo_t ipsec_suites[] = { - {"IPsec-plain-in", ipsec_suite_plain_init, ipsec_suite_term, - ipsec_in_suite}, - {"IPsec-sched-in", ipsec_suite_sched_init, ipsec_suite_term, - ipsec_in_suite}, - {"IPsec-plain-out", ipsec_suite_plain_init, ipsec_suite_term, - ipsec_out_suite}, - {"IPsec-sched-out", ipsec_suite_sched_init, ipsec_suite_term, - ipsec_out_suite}, - ODP_SUITE_INFO_NULL, -}; - -int main(int argc, char *argv[]) -{ - int ret; - - /* parse common options: */ - if (odp_cunit_parse_options(&argc, argv)) - return -1; - - odp_cunit_register_global_init(ipsec_async_init); - odp_cunit_register_global_term(ipsec_term); - - ret = odp_cunit_register(ipsec_suites); - if (ret == 0) - ret = odp_cunit_run(); - - return ret; -} diff --git a/test/validation/api/ipsec/ipsec_async.sh b/test/validation/api/ipsec/ipsec_async.sh new file mode 100755 index 000000000..c12f3e70a --- /dev/null +++ b/test/validation/api/ipsec/ipsec_async.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/ipsec" +$TEST_DIR/ipsec_main$EXEEXT async diff --git a/test/validation/api/ipsec/ipsec_inline_in.c b/test/validation/api/ipsec/ipsec_inline_in.c deleted file mode 100644 index 8deb3c8df..000000000 --- a/test/validation/api/ipsec/ipsec_inline_in.c +++ /dev/null @@ -1,54 +0,0 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include "ipsec.h" - -static int ipsec_sync_init(odp_instance_t *inst) -{ - int rc; - - suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_INLINE; - suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; - - rc = ipsec_init(inst, ODP_IPSEC_OP_MODE_INLINE); - if (rc != 0) - return rc; - - suite_context.pool = odp_pool_lookup("packet_pool"); - if (suite_context.pool == ODP_POOL_INVALID) - return -1; - suite_context.pktio = odp_pktio_lookup("loop"); - if (suite_context.pktio == ODP_PKTIO_INVALID) - return -1; - - return ipsec_config(*inst); -} - -odp_suiteinfo_t ipsec_suites[] = { - {"IPsec-plain-in", ipsec_suite_plain_init, ipsec_suite_term, - ipsec_in_suite}, - {"IPsec-sched-in", ipsec_suite_sched_init, ipsec_suite_term, - ipsec_in_suite}, - ODP_SUITE_INFO_NULL, -}; - -int main(int argc, char *argv[]) -{ - int ret; - - /* parse common options: */ - if (odp_cunit_parse_options(&argc, argv)) - return -1; - - odp_cunit_register_global_init(ipsec_sync_init); - odp_cunit_register_global_term(ipsec_term); - - ret = odp_cunit_register(ipsec_suites); - if (ret == 0) - ret = odp_cunit_run(); - - return ret; -} diff --git a/test/validation/api/ipsec/ipsec_inline_in.sh b/test/validation/api/ipsec/ipsec_inline_in.sh new file mode 100755 index 000000000..c1d1fed36 --- /dev/null +++ b/test/validation/api/ipsec/ipsec_inline_in.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/ipsec" +$TEST_DIR/ipsec_main$EXEEXT inline-in diff --git a/test/validation/api/ipsec/ipsec_inline_out.c b/test/validation/api/ipsec/ipsec_inline_out.c deleted file mode 100644 index 89035f37b..000000000 --- a/test/validation/api/ipsec/ipsec_inline_out.c +++ /dev/null @@ -1,54 +0,0 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include "ipsec.h" - -static int ipsec_sync_init(odp_instance_t *inst) -{ - int rc; - - suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; - suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_INLINE; - - rc = ipsec_init(inst, ODP_IPSEC_OP_MODE_INLINE); - if (rc != 0) - return rc; - - suite_context.pool = odp_pool_lookup("packet_pool"); - if (suite_context.pool == ODP_POOL_INVALID) - return -1; - suite_context.pktio = odp_pktio_lookup("loop"); - if (suite_context.pktio == ODP_PKTIO_INVALID) - return -1; - - return ipsec_config(*inst); -} - -odp_suiteinfo_t ipsec_suites[] = { - {"IPsec-plain-out", ipsec_suite_plain_init, ipsec_suite_term, - ipsec_out_suite}, - {"IPsec-sched-out", ipsec_suite_sched_init, ipsec_suite_term, - ipsec_out_suite}, - ODP_SUITE_INFO_NULL, -}; - -int main(int argc, char *argv[]) -{ - int ret; - - /* parse common options: */ - if (odp_cunit_parse_options(&argc, argv)) - return -1; - - odp_cunit_register_global_init(ipsec_sync_init); - odp_cunit_register_global_term(ipsec_term); - - ret = odp_cunit_register(ipsec_suites); - if (ret == 0) - ret = odp_cunit_run(); - - return ret; -} diff --git a/test/validation/api/ipsec/ipsec_inline_out.sh b/test/validation/api/ipsec/ipsec_inline_out.sh new file mode 100755 index 000000000..91ebccac7 --- /dev/null +++ b/test/validation/api/ipsec/ipsec_inline_out.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/ipsec" +$TEST_DIR/ipsec_main$EXEEXT inline-out diff --git a/test/validation/api/ipsec/ipsec_main.c b/test/validation/api/ipsec/ipsec_main.c new file mode 100644 index 000000000..2c343f063 --- /dev/null +++ b/test/validation/api/ipsec/ipsec_main.c @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited + * Copyright (c) 2024 Nokia + */ + +#include +#include +#include "ipsec.h" + +odp_suiteinfo_t ipsec_sync_suites[] = { + {"IPsec-in", ipsec_suite_sync_init, ipsec_suite_term, ipsec_in_suite}, + {"IPsec-out", ipsec_suite_sync_init, ipsec_suite_term, ipsec_out_suite}, + ODP_SUITE_INFO_NULL, +}; + +odp_suiteinfo_t ipsec_async_suites[] = { + {"IPsec-plain-in", ipsec_suite_plain_init, ipsec_suite_term, ipsec_in_suite}, + {"IPsec-sched-in", ipsec_suite_sched_init, ipsec_suite_term, ipsec_in_suite}, + {"IPsec-plain-out", ipsec_suite_plain_init, ipsec_suite_term, ipsec_out_suite}, + {"IPsec-sched-out", ipsec_suite_sched_init, ipsec_suite_term, ipsec_out_suite}, + ODP_SUITE_INFO_NULL, +}; + +odp_suiteinfo_t ipsec_inline_in_suites[] = { + {"IPsec-plain-in", ipsec_suite_plain_init, ipsec_suite_term, ipsec_in_suite}, + {"IPsec-sched-in", ipsec_suite_sched_init, ipsec_suite_term, ipsec_in_suite}, + ODP_SUITE_INFO_NULL, +}; + +odp_suiteinfo_t ipsec_inline_out_suites[] = { + {"IPsec-plain-out", ipsec_suite_plain_init, ipsec_suite_term, ipsec_out_suite}, + {"IPsec-sched-out", ipsec_suite_sched_init, ipsec_suite_term, ipsec_out_suite}, + ODP_SUITE_INFO_NULL, +}; + +static void usage_exit(void) +{ + fprintf(stderr, "Usage: ipsec_main {sync|async|inline-in|inline-out}\n"); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + char *test_mode; + odp_suiteinfo_t *suites = NULL; + int ret; + + if (odp_cunit_parse_options(&argc, argv)) + return EXIT_FAILURE; + + if (argc < 2) + usage_exit(); + test_mode = argv[1]; + + if ((!strcmp(test_mode, "sync"))) { + suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_SYNC; + suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_SYNC; + suites = ipsec_sync_suites; + } else if ((!strcmp(test_mode, "async"))) { + suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; + suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; + suites = ipsec_async_suites; + } else if ((!strcmp(test_mode, "inline-in"))) { + suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_INLINE; + suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; + suites = ipsec_inline_in_suites; + } else if ((!strcmp(test_mode, "inline-out"))) { + suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; + suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_INLINE; + suites = ipsec_inline_out_suites; + } else { + usage_exit(); + } + + odp_cunit_register_global_init(ipsec_init); + odp_cunit_register_global_term(ipsec_term); + ret = odp_cunit_register(suites); + + if (ret == 0) + ret = odp_cunit_run(); + + return ret; +} diff --git a/test/validation/api/ipsec/ipsec_sync.c b/test/validation/api/ipsec/ipsec_sync.c deleted file mode 100644 index 68aa76e4c..000000000 --- a/test/validation/api/ipsec/ipsec_sync.c +++ /dev/null @@ -1,49 +0,0 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include "ipsec.h" - -static int ipsec_sync_init(odp_instance_t *inst) -{ - int rc; - - suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_SYNC; - suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_SYNC; - - rc = ipsec_init(inst, ODP_IPSEC_OP_MODE_SYNC); - if (rc != 0) - return rc; - - suite_context.pool = odp_pool_lookup("packet_pool"); - if (suite_context.pool == ODP_POOL_INVALID) - return -1; - - return ipsec_config(*inst); -} - -odp_suiteinfo_t ipsec_suites[] = { - {"IPsec-in", ipsec_suite_sync_init, ipsec_in_term, ipsec_in_suite}, - {"IPsec-out", ipsec_suite_sync_init, ipsec_out_term, ipsec_out_suite}, - ODP_SUITE_INFO_NULL, -}; - -int main(int argc, char *argv[]) -{ - int ret; - - /* parse common options: */ - if (odp_cunit_parse_options(&argc, argv)) - return -1; - - odp_cunit_register_global_init(ipsec_sync_init); - odp_cunit_register_global_term(ipsec_term); - - ret = odp_cunit_register(ipsec_suites); - if (ret == 0) - ret = odp_cunit_run(); - - return ret; -} diff --git a/test/validation/api/ipsec/ipsec_sync.sh b/test/validation/api/ipsec/ipsec_sync.sh new file mode 100755 index 000000000..2653ddc78 --- /dev/null +++ b/test/validation/api/ipsec/ipsec_sync.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/ipsec" +$TEST_DIR/ipsec_main$EXEEXT sync From abf2bb60003a7782b0b2f5c4cc3e904ff3409aab Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Thu, 4 Jan 2024 15:38:19 +0200 Subject: [PATCH 040/147] validation: init: combine tests to single test binary Do not compile a separate binary for each test case. This speeds up LTO build a bit. Signed-off-by: Janne Peltonen Reviewed-by: Matias Elo --- test/validation/api/Makefile.am | 18 +++++----- test/validation/api/init/.gitignore | 10 +----- test/validation/api/init/Makefile.am | 36 +++++++------------ test/validation/api/init/init_abort.sh | 3 ++ test/validation/api/init/init_defaults.sh | 3 ++ .../api/init/init_feature_disabled.sh | 3 ++ .../api/init/init_feature_enabled.sh | 3 ++ test/validation/api/init/init_log.sh | 3 ++ test/validation/api/init/init_log_thread.sh | 3 ++ test/validation/api/init/init_main.c | 21 ++++++----- test/validation/api/init/init_num_thr.sh | 3 ++ .../api/init/init_test_param_init.sh | 3 ++ .../api/init/init_test_term_abnormal.sh | 3 ++ 13 files changed, 62 insertions(+), 50 deletions(-) create mode 100755 test/validation/api/init/init_abort.sh create mode 100755 test/validation/api/init/init_defaults.sh create mode 100755 test/validation/api/init/init_feature_disabled.sh create mode 100755 test/validation/api/init/init_feature_enabled.sh create mode 100755 test/validation/api/init/init_log.sh create mode 100755 test/validation/api/init/init_log_thread.sh create mode 100755 test/validation/api/init/init_num_thr.sh create mode 100755 test/validation/api/init/init_test_param_init.sh create mode 100755 test/validation/api/init/init_test_term_abnormal.sh diff --git a/test/validation/api/Makefile.am b/test/validation/api/Makefile.am index abafa1e10..b109c1b38 100644 --- a/test/validation/api/Makefile.am +++ b/test/validation/api/Makefile.am @@ -46,15 +46,15 @@ TESTS = \ errno/errno_main$(EXEEXT) \ event/event_main$(EXEEXT) \ hash/hash_main$(EXEEXT) \ - init/init_defaults$(EXEEXT) \ - init/init_abort$(EXEEXT) \ - init/init_log$(EXEEXT) \ - init/init_log_thread$(EXEEXT) \ - init/init_num_thr$(EXEEXT) \ - init/init_feature_enabled$(EXEEXT) \ - init/init_feature_disabled$(EXEEXT) \ - init/init_test_param_init$(EXEEXT) \ - init/init_test_term_abnormal$(EXEEXT) \ + init/init_defaults.sh \ + init/init_abort.sh \ + init/init_log.sh \ + init/init_log_thread.sh \ + init/init_num_thr.sh \ + init/init_feature_enabled.sh \ + init/init_feature_disabled.sh \ + init/init_test_param_init.sh \ + init/init_test_term_abnormal.sh \ ipsec/ipsec_sync.sh \ ipsec/ipsec_async.sh \ ipsec/ipsec_inline_in.sh \ diff --git a/test/validation/api/init/.gitignore b/test/validation/api/init/.gitignore index 4e14eb040..f0ce98bdd 100644 --- a/test/validation/api/init/.gitignore +++ b/test/validation/api/init/.gitignore @@ -1,9 +1 @@ -init_defaults -init_abort -init_log -init_log_thread -init_num_thr -init_feature_enabled -init_feature_disabled -init_test_param_init -init_test_term_abnormal +init_main diff --git a/test/validation/api/init/Makefile.am b/test/validation/api/init/Makefile.am index 8226b87ce..7465f683f 100644 --- a/test/validation/api/init/Makefile.am +++ b/test/validation/api/init/Makefile.am @@ -1,27 +1,15 @@ include ../Makefile.inc -# Keep init test cases in separate binaries. Some implementations may not allow -# the same application process to call odp_init_global() multiple times. -test_PROGRAMS = init_defaults init_abort init_log init_num_thr \ - init_feature_enabled init_feature_disabled init_log_thread \ - init_test_param_init init_test_term_abnormal +test_PROGRAMS = init_main +init_main_SOURCES = init_main.c -init_defaults_CPPFLAGS = -DINIT_TEST=0 $(AM_CPPFLAGS) -init_abort_CPPFLAGS = -DINIT_TEST=1 $(AM_CPPFLAGS) -init_log_CPPFLAGS = -DINIT_TEST=2 $(AM_CPPFLAGS) -init_num_thr_CPPFLAGS = -DINIT_TEST=3 $(AM_CPPFLAGS) -init_feature_enabled_CPPFLAGS = -DINIT_TEST=4 $(AM_CPPFLAGS) -init_feature_disabled_CPPFLAGS = -DINIT_TEST=5 $(AM_CPPFLAGS) -init_log_thread_CPPFLAGS = -DINIT_TEST=6 $(AM_CPPFLAGS) -init_test_param_init_CPPFLAGS = -DINIT_TEST=7 $(AM_CPPFLAGS) -init_test_term_abnormal_CPPFLAGS = -DINIT_TEST=8 $(AM_CPPFLAGS) - -init_defaults_SOURCES = init_main.c -init_abort_SOURCES = init_main.c -init_log_SOURCES = init_main.c -init_num_thr_SOURCES = init_main.c -init_feature_enabled_SOURCES = init_main.c -init_feature_disabled_SOURCES = init_main.c -init_log_thread_SOURCES = init_main.c -init_test_param_init_SOURCES = init_main.c -init_test_term_abnormal_SOURCES = init_main.c +EXTRA_DIST = \ + init_defaults.sh \ + init_abort.sh \ + init_log.sh \ + init_num_thr.sh \ + init_feature_enabled.sh \ + init_feature_disabled.sh \ + init_log_thread.sh \ + init_test_param_init.sh \ + init_test_term_abnormal.sh diff --git a/test/validation/api/init/init_abort.sh b/test/validation/api/init/init_abort.sh new file mode 100755 index 000000000..27796fcf9 --- /dev/null +++ b/test/validation/api/init/init_abort.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 1 diff --git a/test/validation/api/init/init_defaults.sh b/test/validation/api/init/init_defaults.sh new file mode 100755 index 000000000..2215a65a9 --- /dev/null +++ b/test/validation/api/init/init_defaults.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 0 diff --git a/test/validation/api/init/init_feature_disabled.sh b/test/validation/api/init/init_feature_disabled.sh new file mode 100755 index 000000000..e538429b6 --- /dev/null +++ b/test/validation/api/init/init_feature_disabled.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 5 diff --git a/test/validation/api/init/init_feature_enabled.sh b/test/validation/api/init/init_feature_enabled.sh new file mode 100755 index 000000000..18237cbf9 --- /dev/null +++ b/test/validation/api/init/init_feature_enabled.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 4 diff --git a/test/validation/api/init/init_log.sh b/test/validation/api/init/init_log.sh new file mode 100755 index 000000000..cf4177ed5 --- /dev/null +++ b/test/validation/api/init/init_log.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 2 diff --git a/test/validation/api/init/init_log_thread.sh b/test/validation/api/init/init_log_thread.sh new file mode 100755 index 000000000..b0bb02220 --- /dev/null +++ b/test/validation/api/init/init_log_thread.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 6 diff --git a/test/validation/api/init/init_main.c b/test/validation/api/init/init_main.c index f2a141b5d..ab1db421b 100644 --- a/test/validation/api/init/init_main.c +++ b/test/validation/api/init/init_main.c @@ -1,8 +1,6 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2019-2023, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2019-2024 Nokia */ #include @@ -297,14 +295,21 @@ static int fill_testinfo(odp_testinfo_t *info, unsigned int test_case) int main(int argc, char *argv[]) { int ret; - - if (fill_testinfo(&init_suite[0], INIT_TEST)) - return -1; + int test_id; /* Parse common options */ if (odp_cunit_parse_options(&argc, argv)) return -1; + if (argc < 2) { + ODPH_ERR("Usage: init_main \n"); + return -1; + } + test_id = atoi(argv[1]); + + if (fill_testinfo(&init_suite[0], test_id)) + return -1; + /* Prevent default ODP init */ odp_cunit_register_global_init(NULL); odp_cunit_register_global_term(NULL); diff --git a/test/validation/api/init/init_num_thr.sh b/test/validation/api/init/init_num_thr.sh new file mode 100755 index 000000000..3889d2a14 --- /dev/null +++ b/test/validation/api/init/init_num_thr.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 3 diff --git a/test/validation/api/init/init_test_param_init.sh b/test/validation/api/init/init_test_param_init.sh new file mode 100755 index 000000000..afb4d17a2 --- /dev/null +++ b/test/validation/api/init/init_test_param_init.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 7 diff --git a/test/validation/api/init/init_test_term_abnormal.sh b/test/validation/api/init/init_test_term_abnormal.sh new file mode 100755 index 000000000..b3edd8391 --- /dev/null +++ b/test/validation/api/init/init_test_term_abnormal.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 8 From b70cd853b1b16c35bf30aa36d148e0e9672988b6 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 12 Jan 2024 16:15:11 +0200 Subject: [PATCH 041/147] linux-gen: packet: move packet header cos field Move odp_packet_hdr_t.cos field to a gap on the previous 64-byte cache line. This cache line is always written during packet reception, so writing CoS here (when classifier is enabled) can reduce classification overhead. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- platform/linux-generic/include/odp_packet_internal.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index f7904c3bc..0b03aa211 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -107,6 +107,9 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { uint16_t tailroom; + /* Classifier handle index */ + uint16_t cos; + /* Used as classifier destination queue, in IPsec inline input processing and as Tx * completion event queue. */ odp_queue_t dst_queue; @@ -131,9 +134,6 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { /* Classifier mark */ uint16_t cls_mark; - /* Classifier handle index */ - uint16_t cos; - /* Offset to payload start */ uint16_t payload_offset; From 3c7e781b8fec98007d1604e63064f068eec227e9 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 15 Jan 2024 15:40:32 +0200 Subject: [PATCH 042/147] validation: sched: run tests only on worker threads There is no guarantee that the main thread (ODP_THREAD_CONTROL) won't end up running on the same core with a worker thread. To avoid test slowdowns run multithreaded scheduler tests only on worker threads. The 'scheduler_test_order_wait_2_threads' test still runs also on control thread due to specific test logic. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- test/validation/api/scheduler/scheduler.c | 59 ++++++----------------- 1 file changed, 14 insertions(+), 45 deletions(-) diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index dac4becbd..9beb6c72d 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2024, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -1355,7 +1355,7 @@ static void chaos_run(unsigned int qtype) test_globals_t *globals; thread_args_t *args; odp_shm_t shm; - int i, rc, num_thr; + int i, rc; void *arg_ptr; odp_schedule_sync_t sync[] = {ODP_SCHED_SYNC_PARALLEL, ODP_SCHED_SYNC_ATOMIC, @@ -1418,16 +1418,10 @@ static void chaos_run(unsigned int qtype) CU_ASSERT_FATAL(rc == 0); } - /* Test runs also on the main thread */ - num_thr = globals->num_workers - 1; arg_ptr = args; - if (num_thr > 0) - odp_cunit_thread_create(num_thr, chaos_thread, &arg_ptr, 0, 0); + odp_cunit_thread_create(globals->num_workers, chaos_thread, &arg_ptr, 0, 0); - chaos_thread(args); - - if (num_thr > 0) - odp_cunit_thread_join(num_thr); + odp_cunit_thread_join(globals->num_workers); if (CHAOS_DEBUG) printf("Thread %d returning from chaos threads..cleaning up\n", @@ -1830,7 +1824,6 @@ static void parallel_execute(odp_schedule_sync_t sync, int num_queues, test_globals_t *globals; thread_args_t *args; void *arg_ptr; - int num; shm = odp_shm_lookup(GLOBALS_SHM_NAME); CU_ASSERT_FATAL(shm != ODP_SHM_INVALID); @@ -1860,18 +1853,11 @@ static void parallel_execute(odp_schedule_sync_t sync, int num_queues, odp_schedule_print(); /* Create and launch worker threads */ - - /* Test runs also on the main thread */ - num = globals->num_workers - 1; arg_ptr = args; - if (num > 0) - odp_cunit_thread_create(num, schedule_common_, &arg_ptr, 0, 0); - - schedule_common_(args); + odp_cunit_thread_create(globals->num_workers, schedule_common_, &arg_ptr, 0, 0); /* Wait for worker threads to terminate */ - if (num > 0) - odp_cunit_thread_join(num); + odp_cunit_thread_join(globals->num_workers); /* Cleanup ordered queues for next pass */ if (sync == ODP_SCHED_SYNC_ORDERED) @@ -2590,7 +2576,7 @@ static void scheduler_test_sched_and_plain(odp_schedule_sync_t sync) uint64_t wait = odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS); uint32_t events_per_queue = BUFS_PER_QUEUE / 2; uint32_t prev_seq; - int first, num; + int first; void *arg_ptr; CU_ASSERT_FATAL(!odp_schedule_capability(&sched_capa)); @@ -2673,16 +2659,10 @@ static void scheduler_test_sched_and_plain(odp_schedule_sync_t sync) } CU_ASSERT_FATAL(seq > 2); - /* Test runs also on the main thread */ - num = globals->num_workers - 1; arg_ptr = args; - if (num > 0) - odp_cunit_thread_create(num, sched_and_plain_thread, &arg_ptr, 0, 0); - - sched_and_plain_thread(args); + odp_cunit_thread_create(globals->num_workers, sched_and_plain_thread, &arg_ptr, 0, 0); - if (num > 0) - odp_cunit_thread_join(num); + odp_cunit_thread_join(globals->num_workers); /* Check plain queue sequence numbers and free events */ first = 1; @@ -2979,15 +2959,10 @@ static void scheduler_fifo_mt(odp_schedule_sync_t sync, int multi) for (i = 0; i < num_thr; i++) arg[i] = i; - if (num_thr > 1) - odp_cunit_thread_create(num_thr - 1, scheduler_fifo_test, (void **)&arg[1], 1, 0); - - /* Main thread runs as thread 0 */ - scheduler_fifo_test(0); + odp_cunit_thread_create(num_thr, scheduler_fifo_test, (void **)&arg[0], 1, 0); /* Wait for worker threads to terminate */ - if (num_thr > 1) - odp_cunit_thread_join(num_thr - 1); + odp_cunit_thread_join(num_thr); } static void scheduler_fifo_mt_parallel_single(void) @@ -3083,7 +3058,7 @@ static void scheduler_test_atomicity(void) odp_pool_t pool; odp_queue_t queue; odp_queue_param_t queue_param; - int i, num; + int i; void *arg_ptr; shm = odp_shm_lookup(GLOBALS_SHM_NAME); @@ -3121,18 +3096,12 @@ static void scheduler_test_atomicity(void) odp_atomic_init_u32(&globals->atomicity_q.state, 0); /* Create and launch worker threads */ - /* Test runs also on the main thread */ args->num_workers = globals->num_workers; - num = globals->num_workers - 1; arg_ptr = args; - if (num > 0) - odp_cunit_thread_create(num, atomicity_test_run, &arg_ptr, 0, 0); - - atomicity_test_run(args); + odp_cunit_thread_create(globals->num_workers, atomicity_test_run, &arg_ptr, 0, 0); /* Wait for worker threads to terminate */ - if (num > 0) - odp_cunit_thread_join(num); + odp_cunit_thread_join(globals->num_workers); odp_queue_destroy(globals->atomicity_q.handle); } From 3878be0ab31947fd7fa66a22126178f41f2bee2b Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 16 Jan 2024 09:42:01 +0200 Subject: [PATCH 043/147] validation: timer: shorten timeouts to speed up testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Increase timer resolution and shorten various timer intervals to make the test faster. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- test/validation/api/timer/timer.c | 40 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index 281a8d763..ea784fdb1 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -192,8 +192,8 @@ static int timer_global_init(odp_instance_t *instance) global_mem->periodic_support = capa.periodic.max_pools > 0; - /* By default 20 msec resolution */ - res_ns = 20 * ODP_TIME_MSEC_IN_NS; + /* By default 2 msec resolution */ + res_ns = 2 * ODP_TIME_MSEC_IN_NS; if (res_ns < capa.max_res.res_ns) res_ns = capa.max_res.res_ns; @@ -1131,42 +1131,42 @@ static void timer_single_shot(odp_queue_type_t queue_type, odp_timer_tick_type_t static void timer_plain_rel_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, START, TIMEOUT, 2, 500 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, START, TIMEOUT, 2, 50 * MSEC); } static void timer_plain_abs_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, START, TIMEOUT, 2, 500 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, START, TIMEOUT, 2, 50 * MSEC); } static void timer_plain_rel_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, START, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, START, CANCEL, 5, 100 * MSEC); } static void timer_plain_abs_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, START, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, START, CANCEL, 5, 100 * MSEC); } static void timer_plain_rel_restart_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, RESTART, TIMEOUT, 2, 600 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, RESTART, TIMEOUT, 2, 60 * MSEC); } static void timer_plain_abs_restart_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, RESTART, TIMEOUT, 2, 600 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, RESTART, TIMEOUT, 2, 60 * MSEC); } static void timer_plain_rel_restart_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, RESTART, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, RESTART, CANCEL, 5, 100 * MSEC); } static void timer_plain_abs_restart_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, RESTART, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, RESTART, CANCEL, 5, 100 * MSEC); } static void timer_plain_abs_wait_3sec(void) @@ -1176,42 +1176,42 @@ static void timer_plain_abs_wait_3sec(void) static void timer_sched_rel_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, START, TIMEOUT, 2, 500 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, START, TIMEOUT, 2, 50 * MSEC); } static void timer_sched_abs_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, START, TIMEOUT, 2, 500 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, START, TIMEOUT, 2, 50 * MSEC); } static void timer_sched_rel_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, START, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, START, CANCEL, 5, 100 * MSEC); } static void timer_sched_abs_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, START, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, START, CANCEL, 5, 100 * MSEC); } static void timer_sched_rel_restart_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, RESTART, TIMEOUT, 2, 600 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, RESTART, TIMEOUT, 2, 60 * MSEC); } static void timer_sched_abs_restart_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, RESTART, TIMEOUT, 2, 600 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, RESTART, TIMEOUT, 2, 60 * MSEC); } static void timer_sched_rel_restart_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, RESTART, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, RESTART, CANCEL, 5, 100 * MSEC); } static void timer_sched_abs_restart_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, RESTART, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, RESTART, CANCEL, 5, 100 * MSEC); } static void timer_sched_abs_wait_3sec(void) @@ -2862,9 +2862,9 @@ static void timer_test_periodic(odp_queue_type_t queue_type, int use_first, int int num_tmo; int done; const int num = 200; - /* Test frequency: 1x 100Hz, or 1x min/max_base_freq */ + /* Test frequency: 1x 1000 Hz, or 1x min/max_base_freq */ const uint64_t multiplier = 1; - odp_fract_u64_t base_freq = {100, 0, 0}; + odp_fract_u64_t base_freq = {1000, 0, 0}; odp_timer_clk_src_t clk_src = test_global->clk_src; memset(&timer_capa, 0, sizeof(odp_timer_capability_t)); From 91ad7b18d650c66a49987caea3b63e340e3ad2df Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 16 Jan 2024 09:42:03 +0200 Subject: [PATCH 044/147] test: sched_latency: lower the number of test rounds in make check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lower the number of scheduling rounds during 'make check' to make 'make check' a bit faster. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- test/performance/odp_sched_latency_run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/performance/odp_sched_latency_run.sh b/test/performance/odp_sched_latency_run.sh index 372fdb166..b051c1a4e 100755 --- a/test/performance/odp_sched_latency_run.sh +++ b/test/performance/odp_sched_latency_run.sh @@ -19,7 +19,7 @@ run() if [ $(nproc) -lt $1 ]; then echo "Not enough CPU cores. Skipping test." else - $TEST_DIR/odp_sched_latency${EXEEXT} -c $1 || exit $? + $TEST_DIR/odp_sched_latency${EXEEXT} -c $1 -d 1 || exit $? fi } From 23da2fc78b6e4343f3050743e169a6864c6c9023 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 16 Jan 2024 09:42:04 +0200 Subject: [PATCH 045/147] test: dma_perf: fix potential rounding errors and kilo prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use floating point arithmetic when truncation to integer may cause unexpected rounding errors. This also makes sub-second measurement intervals possible. Fix also the kilo prefix (k) in statistics printout. Reviewed-by: Jere Leppänen Signed-off-by: Janne Peltonen --- test/performance/odp_dma_perf.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/performance/odp_dma_perf.c b/test/performance/odp_dma_perf.c index f843843c8..939e1ddaf 100644 --- a/test/performance/odp_dma_perf.c +++ b/test/performance/odp_dma_perf.c @@ -1692,7 +1692,7 @@ static void print_humanised(uint64_t value, const char *type) else if (value > MEGAS) printf("%.2f M%s\n", (double)value / MEGAS, type); else if (value > KILOS) - printf("%.2f K%s\n", (double)value / KILOS, type); + printf("%.2f k%s\n", (double)value / KILOS, type); else printf("%" PRIu64 " %s\n", value, type); } @@ -1702,8 +1702,8 @@ static void print_stats(const prog_config_t *config) const stats_t *stats; uint64_t data_cnt = config->num_in_segs * config->src_seg_len, tot_completed = 0U, tot_tm = 0U, tot_trs_tm = 0U, tot_trs_cc = 0U, tot_trs_cnt = 0U, tot_min_tm = UINT64_MAX, - tot_max_tm = 0U, tot_min_cc = UINT64_MAX, tot_max_cc = 0U, avg_start_cc, avg_wait_cc, - avg_tot_tm; + tot_max_tm = 0U, tot_min_cc = UINT64_MAX, tot_max_cc = 0U, avg_start_cc, avg_wait_cc; + double avg_tot_tm; printf("\n======================\n\n" "DMA performance test done\n\n" @@ -1771,11 +1771,12 @@ static void print_stats(const prog_config_t *config) stats->trs_cnt > 0U ? stats->trs_cc / stats->trs_cnt : 0U, stats->trs_cnt > 0U ? stats->min_trs_cc : 0U, stats->trs_cnt > 0U ? stats->max_trs_cc : 0U); - print_humanised(stats->completed / (stats->tot_tm / ODP_TIME_SEC_IN_NS), + print_humanised(stats->completed / + ((double)stats->tot_tm / ODP_TIME_SEC_IN_NS), "OPS"); printf(" speed: "); print_humanised(stats->completed * data_cnt / - (stats->tot_tm / ODP_TIME_SEC_IN_NS), "B/s"); + ((double)stats->tot_tm / ODP_TIME_SEC_IN_NS), "B/s"); } avg_start_cc = stats->start_cnt > 0U ? stats->start_cc / stats->start_cnt : 0U; @@ -1818,7 +1819,7 @@ static void print_stats(const prog_config_t *config) printf("\n"); } - avg_tot_tm = tot_tm / config->num_workers / ODP_TIME_SEC_IN_NS; + avg_tot_tm = (double)tot_tm / config->num_workers / ODP_TIME_SEC_IN_NS; printf(" total:\n" " average time per transfer: %" PRIu64 " (min: %" PRIu64 ", max: %" PRIu64 ") ns\n" From b4ab2c455f34eaa9dc3ccda390d98671c2a78caa Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 16 Jan 2024 09:42:06 +0200 Subject: [PATCH 046/147] test: dma_perf: allow decimal run time parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow decimal run time parameter to enable sub-second run times. Also reduce the wait time for queue cleanup at the end. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- test/performance/odp_dma_perf.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/performance/odp_dma_perf.c b/test/performance/odp_dma_perf.c index 939e1ddaf..cc86775cd 100644 --- a/test/performance/odp_dma_perf.c +++ b/test/performance/odp_dma_perf.c @@ -204,7 +204,7 @@ typedef struct prog_config_s { uint32_t src_seg_len; uint32_t dst_seg_len; uint32_t num_inflight; - uint32_t time_sec; + double time_sec; uint32_t num_sessions; uint32_t src_cache_size; uint32_t dst_cache_size; @@ -599,7 +599,7 @@ static parse_result_t parse_options(int argc, char **argv, prog_config_t *config config->num_inflight = atoi(optarg); break; case 'T': - config->time_sec = atoi(optarg); + config->time_sec = atof(optarg); break; case 'c': config->num_workers = atoi(optarg); @@ -1243,7 +1243,7 @@ static void drain_compl_events(ODP_UNUSED sd_t *sd) odp_event_t ev; while (true) { - ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS)); + ev = odp_schedule(NULL, odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS)); if (ev == ODP_EVENT_INVALID) break; @@ -1916,8 +1916,12 @@ int main(int argc, char **argv) goto out_test; } - if (prog_conf->time_sec) { - sleep(prog_conf->time_sec); + if (prog_conf->time_sec > 0.001) { + struct timespec ts; + + ts.tv_sec = prog_conf->time_sec; + ts.tv_nsec = (prog_conf->time_sec - ts.tv_sec) * ODP_TIME_SEC_IN_NS; + nanosleep(&ts, NULL); odp_atomic_store_u32(&prog_conf->is_running, 0U); } From 08deb85bf38ca47dad7010633a8e9605c9d44e37 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 16 Jan 2024 09:42:08 +0200 Subject: [PATCH 047/147] test: dma_perf: reduce test run time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce test run time during make check. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- test/performance/odp_dma_perf_run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/performance/odp_dma_perf_run.sh b/test/performance/odp_dma_perf_run.sh index f5d567740..31948e40a 100755 --- a/test/performance/odp_dma_perf_run.sh +++ b/test/performance/odp_dma_perf_run.sh @@ -10,7 +10,7 @@ BIN_NAME=odp_dma_perf SEGC=0 SEGS=1024 INFL=1 -TIME=1 +TIME=0.1 TESTS_RUN=0 check_result() From d4c20010f21354739a6dbe580c6f0a3c0ea68466 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 16 Jan 2024 09:42:09 +0200 Subject: [PATCH 048/147] test: dmafwd: allow decimal run time parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow decimal run time parameter to enable sub-second run times. Also reduce the wait time for queue cleanup at the end. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- test/performance/odp_dmafwd.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test/performance/odp_dmafwd.c b/test/performance/odp_dmafwd.c index 088fc69e2..3f89bf707 100644 --- a/test/performance/odp_dmafwd.c +++ b/test/performance/odp_dmafwd.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -154,7 +155,7 @@ typedef struct prog_config_s { uint32_t trs_cache_size; uint32_t compl_cache_size; uint32_t stash_cache_size; - uint32_t time_sec; + double time_sec; odp_stash_type_t stash_type; int num_thrs; uint8_t num_ifs; @@ -507,7 +508,7 @@ static parse_result_t parse_options(int argc, char **argv, prog_config_t *config config->cache_size = atoi(optarg); break; case 'T': - config->time_sec = atoi(optarg); + config->time_sec = atof(optarg); break; case 'h': print_usage(&config->dyn_defs); @@ -760,7 +761,7 @@ static void drain_events(thread_config_t *config ODP_UNUSED) transfer_t *trs; while (true) { - ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS * 2U)); + ev = odp_schedule(NULL, odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS)); if (ev == ODP_EVENT_INVALID) break; @@ -1438,8 +1439,12 @@ int main(int argc, char **argv) goto out_test; } - if (prog_conf->time_sec) { - sleep(prog_conf->time_sec); + if (prog_conf->time_sec > 0.001) { + struct timespec ts; + + ts.tv_sec = prog_conf->time_sec; + ts.tv_nsec = (prog_conf->time_sec - ts.tv_sec) * ODP_TIME_SEC_IN_NS; + nanosleep(&ts, NULL); odp_atomic_store_u32(&prog_conf->is_running, 0U); } else { while (odp_atomic_load_u32(&prog_conf->is_running)) From 831da6172428ba0a16861c951917963e56c9dd75 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 16 Jan 2024 09:42:11 +0200 Subject: [PATCH 049/147] test: dmafwd: reduce test run time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce test run time during make check. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- test/performance/odp_dmafwd_run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/performance/odp_dmafwd_run.sh b/test/performance/odp_dmafwd_run.sh index fa629bd0c..ebb9b153a 100755 --- a/test/performance/odp_dmafwd_run.sh +++ b/test/performance/odp_dmafwd_run.sh @@ -10,7 +10,7 @@ PERF_TEST_DIR=${TEST_SRC_DIR}/../../${PERF_TEST_DIR} BIN_NAME=odp_dmafwd BATCH=10 -TIME=2 +TIME=0.1 TESTS_RUN=0 check_env() From 58c5733ecd79a40287b5c82e18d7bda8e7d318e5 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 16 Jan 2024 09:42:12 +0200 Subject: [PATCH 050/147] test: performance: cpu_bench: reduce scheduler draining wait time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Speed up the program slightly by reducing the wait time for scheduled queue draining. Signed-off-by: Janne Peltonen Reviewed-by: Jere Leppänen --- test/performance/odp_cpu_bench.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/performance/odp_cpu_bench.c b/test/performance/odp_cpu_bench.c index 0ce8aa3fb..39eff620d 100644 --- a/test/performance/odp_cpu_bench.c +++ b/test/performance/odp_cpu_bench.c @@ -335,7 +335,7 @@ static int run_thread(void *arg) odp_event_t ev; ev = odp_schedule(NULL, - odp_schedule_wait_time(ODP_TIME_SEC_IN_NS)); + odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS)); if (ev == ODP_EVENT_INVALID) break; From 3517348179bf0ebc7bb3c4901b23ff9c8e07a17e Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 19 Jan 2024 09:59:01 +0200 Subject: [PATCH 051/147] validation: sched: run 2-thread order wait test with one worker too Run the 2-thread odp_schedule_order_wait() test regardless of the number of worker CPUs. The test anyway uses just single worker thread and the main (control) thread. This enables the test in systems with just one or two CPUs. Signed-off-by: Janne Peltonen Reviewed-by: Matias Elo --- test/validation/api/scheduler/scheduler.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index 9beb6c72d..87069f7fe 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -2481,16 +2481,6 @@ static void scheduler_test_order_wait_2_threads(void) CU_ASSERT(odp_queue_destroy(queue) == 0); } -static int check_2_workers(void) -{ - if (globals->num_workers < 2) { - printf("\nTest: scheduler_test_order_wait_2_threads: SKIPPED\n"); - return ODP_TEST_INACTIVE; - } - - return ODP_TEST_ACTIVE; -} - static int sched_and_plain_thread(void *arg) { odp_event_t ev1, ev2; @@ -3689,7 +3679,7 @@ odp_testinfo_t scheduler_basic_suite[] = { ODP_TEST_INFO(scheduler_test_pause_enqueue), ODP_TEST_INFO(scheduler_test_ordered_lock), ODP_TEST_INFO(scheduler_test_order_wait_1_thread), - ODP_TEST_INFO_CONDITIONAL(scheduler_test_order_wait_2_threads, check_2_workers), + ODP_TEST_INFO(scheduler_test_order_wait_2_threads), ODP_TEST_INFO_CONDITIONAL(scheduler_test_flow_aware, check_flow_aware_support), ODP_TEST_INFO(scheduler_test_parallel), From dd5a231bc227bd29023fcee8c4a794a0192029b0 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Fri, 19 Jan 2024 09:59:02 +0200 Subject: [PATCH 052/147] validation: time: relax global time sync tolerance with 1 CPU When running the 2-thread time_test_global_sync_control() test in a system with only one CPU core, the control and worker threads run in the same core, which shows up as time differences of the order of the thread scheduling interval of the underlying OS. Increase the allowed error when the threads run in the same CPU core. Signed-off-by: Janne Peltonen Reviewed-by: Matias Elo --- test/validation/api/time/time.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/validation/api/time/time.c b/test/validation/api/time/time.c index ef406a025..cfef7f619 100644 --- a/test/validation/api/time/time.c +++ b/test/validation/api/time/time.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2019-2023, Nokia + * Copyright (c) 2019-2024, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -24,6 +24,7 @@ #define TIME_SAMPLES 2 #define TIME_TOLERANCE_NS 1000000 #define TIME_TOLERANCE_CI_NS 40000000 +#define TIME_TOLERANCE_1CPU_NS 40000000 #define GLOBAL_SHM_NAME "GlobalTimeTest" #define YEAR_IN_NS (365 * 24 * ODP_TIME_HOUR_IN_NS) @@ -858,8 +859,7 @@ static void time_test_global_sync(const int ctrl) odph_thread_common_param_t thr_common; odph_thread_param_t thr_param; odph_thread_t thread_tbl[MAX_WORKERS]; - const uint64_t tolerance = - odp_cunit_ci() ? TIME_TOLERANCE_CI_NS : TIME_TOLERANCE_NS; + uint64_t tolerance = odp_cunit_ci() ? TIME_TOLERANCE_CI_NS : TIME_TOLERANCE_NS; const int num = ctrl ? 2 : global_mem->num_threads; if (num < 2) { @@ -879,14 +879,23 @@ static void time_test_global_sync(const int ctrl) if (ctrl) { /* Test sync between one control and one worker thread. */ + int control_cpu; + int worker_cpu; + odp_cpumask_default_control(&cpumask, 1); thr_common.cpumask = &cpumask; thr_param.thr_type = ODP_THREAD_CONTROL; + control_cpu = odp_cpumask_first(&cpumask); int r = odph_thread_create(&thread_tbl[thr++], &thr_common, &thr_param, 1); CU_ASSERT_FATAL(r == 1); odp_cpumask_default_worker(&cpumask, 1); + worker_cpu = odp_cpumask_first(&cpumask); + if (control_cpu == worker_cpu) { + printf(" single CPU, relaxing tolerance. "); + tolerance = TIME_TOLERANCE_1CPU_NS; + } } else { /* Test sync between num worker threads. */ odp_cpumask_default_worker(&cpumask, num); From 5eca9c479e80615993c64163bd5ee692675c6c61 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Mon, 15 Jan 2024 16:24:31 +0200 Subject: [PATCH 053/147] test: sched_perf: add user area touch Added options to read (-u) and modify (-U) user area content per event reception. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/performance/odp_sched_perf.c | 84 +++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 15 deletions(-) diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c index 4cd4c3ac4..70c91092e 100644 --- a/test/performance/odp_sched_perf.c +++ b/test/performance/odp_sched_perf.c @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2024, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -60,6 +60,9 @@ typedef struct test_options_t { uint32_t ctx_size; uint32_t ctx_rd_words; uint32_t ctx_rw_words; + uint32_t uarea_rd; + uint32_t uarea_rw; + uint32_t uarea_size; uint64_t wait_ns; int verbose; @@ -135,6 +138,8 @@ static void print_usage(void) " -l, --ctx_rw_words Number of queue context words (uint64_t) to modify on every event. Default: 0.\n" " -n, --rd_words Number of event data words (uint64_t) to read before enqueueing it. Default: 0.\n" " -m, --rw_words Number of event data words (uint64_t) to modify before enqueueing it. Default: 0.\n" + " -u, --uarea_rd Number of user area words (uint64_t) to read on every event. Default: 0.\n" + " -U, --uarea_rw Number of user area words (uint64_t) to modify on every event. Default: 0.\n" " -p, --pool_type Pool type. 0: buffer, 1: packet. Default: 0.\n" " -v, --verbose Verbose output.\n" " -h, --help This help\n" @@ -167,13 +172,15 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) {"ctx_rw_words", required_argument, NULL, 'l'}, {"rd_words", required_argument, NULL, 'n'}, {"rw_words", required_argument, NULL, 'm'}, + {"uarea_rd", required_argument, NULL, 'u'}, + {"uarea_rw", required_argument, NULL, 'U'}, {"pool_type", required_argument, NULL, 'p'}, {"verbose", no_argument, NULL, 'v'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:q:L:H:d:e:s:g:j:b:t:f:a:w:k:l:n:m:p:vh"; + static const char *shortopts = "+c:q:L:H:d:e:s:g:j:b:t:f:a:w:k:l:n:m:p:u:U:vh"; test_options->num_cpu = 1; test_options->num_queue = 1; @@ -192,6 +199,8 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) test_options->ctx_rw_words = 0; test_options->rd_words = 0; test_options->rw_words = 0; + test_options->uarea_rd = 0; + test_options->uarea_rw = 0; test_options->wait_ns = 0; test_options->verbose = 0; @@ -253,6 +262,12 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) case 'm': test_options->rw_words = atoi(optarg); break; + case 'u': + test_options->uarea_rd = atoi(optarg); + break; + case 'U': + test_options->uarea_rw = atoi(optarg); + break; case 'p': pool_type = atoi(optarg); break; @@ -345,6 +360,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) ctx_size = ROUNDUP(ctx_size, ODP_CACHE_LINE_SIZE); test_options->ctx_size = ctx_size; + test_options->uarea_size = 8 * (test_options->uarea_rd + test_options->uarea_rw); return ret; } @@ -385,7 +401,7 @@ static int create_pool(test_global_t *global) odp_pool_capability_t pool_capa; odp_pool_param_t pool_param; odp_pool_t pool; - uint32_t max_num, max_size; + uint32_t max_num, max_size, max_uarea; test_options_t *test_options = &global->test_options; uint32_t num_cpu = test_options->num_cpu; uint32_t num_queue = test_options->num_queue; @@ -403,6 +419,7 @@ static int create_pool(test_global_t *global) uint32_t event_size = 16; int touch_data = test_options->touch_data; uint32_t ctx_size = test_options->ctx_size; + uint32_t uarea_size = test_options->uarea_size; if (touch_data) { event_size = test_options->rd_words + test_options->rw_words; @@ -434,22 +451,22 @@ static int create_pool(test_global_t *global) printf(" max burst size %u\n", max_burst); printf(" total events %u\n", tot_event); printf(" event size %u bytes", event_size); - if (touch_data) { - printf(" (rd: %u, rw: %u)\n", - 8 * test_options->rd_words, - 8 * test_options->rw_words); - } else { - printf("\n"); - } + if (touch_data) + printf(" (rd: %u, rw: %u)", 8 * test_options->rd_words, 8 * test_options->rw_words); + printf("\n"); printf(" context size %u bytes", ctx_size); if (test_options->ctx_rd_words || test_options->ctx_rw_words) { - printf(" (rd: %u, rw: %u)\n", + printf(" (rd: %u, rw: %u)", 8 * test_options->ctx_rd_words, 8 * test_options->ctx_rw_words); - } else { - printf("\n"); } + printf("\n"); + + printf(" user area size %u bytes", uarea_size); + if (uarea_size) + printf(" (rd: %u, rw: %u)", 8 * test_options->uarea_rd, 8 * test_options->uarea_rw); + printf("\n"); if (odp_pool_capability(&pool_capa)) { ODPH_ERR("Error: pool capa failed\n"); @@ -460,11 +477,12 @@ static int create_pool(test_global_t *global) printf(" pool type buffer\n"); max_num = pool_capa.buf.max_num; max_size = pool_capa.buf.max_size; - + max_uarea = pool_capa.buf.max_uarea_size; } else { printf(" pool type packet\n"); max_num = pool_capa.pkt.max_num; max_size = pool_capa.pkt.max_seg_len; + max_uarea = pool_capa.pkt.max_uarea_size; } if (max_num && tot_event > max_num) { @@ -477,18 +495,25 @@ static int create_pool(test_global_t *global) return -1; } + if (uarea_size > max_uarea) { + ODPH_ERR("Error: max supported user area size %u\n", max_uarea); + return -1; + } + odp_pool_param_init(&pool_param); if (test_options->pool_type == ODP_POOL_BUFFER) { pool_param.type = ODP_POOL_BUFFER; pool_param.buf.num = tot_event; pool_param.buf.size = event_size; pool_param.buf.align = 8; + pool_param.buf.uarea_size = uarea_size; } else { pool_param.type = ODP_POOL_PACKET; pool_param.pkt.num = tot_event; pool_param.pkt.len = event_size; pool_param.pkt.seg_len = event_size; pool_param.pkt.align = 8; + pool_param.pkt.uarea_size = uarea_size; } pool = odp_pool_create("sched perf", &pool_param); @@ -861,6 +886,28 @@ static int destroy_groups(test_global_t *global) return 0; } +static uint64_t rw_uarea(odp_event_t ev[], int num, uint32_t rd_words, uint32_t rw_words) +{ + uint64_t *data; + int i; + uint32_t j; + uint64_t sum = 0; + + for (i = 0; i < num; i++) { + data = odp_event_user_area(ev[i]); + + for (j = 0; j < rd_words; j++) + sum += data[j]; + + for (; j < rd_words + rw_words; j++) { + sum += data[j]; + data[j] += 1; + } + } + + return sum; +} + static inline uint64_t rw_ctx_data(void *ctx, uint32_t offset, uint32_t rd_words, uint32_t rw_words) { @@ -929,12 +976,16 @@ static int test_sched(void *arg) uint32_t ctx_size = test_options->ctx_size; uint32_t ctx_rd_words = test_options->ctx_rd_words; uint32_t ctx_rw_words = test_options->ctx_rw_words; + const uint32_t uarea_size = test_options->uarea_size; + const uint32_t uarea_rd = test_options->uarea_rd; + const uint32_t uarea_rw = test_options->uarea_rw; odp_pool_type_t pool_type = test_options->pool_type; int touch_ctx = ctx_rd_words || ctx_rw_words; uint32_t ctx_offset = 0; uint32_t sched_retries = 0; uint64_t data_sum = 0; uint64_t ctx_sum = 0; + uint64_t uarea_sum = 0; uint64_t wait_ns = test_options->wait_ns; odp_event_t ev[max_burst]; @@ -998,6 +1049,9 @@ static int test_sched(void *arg) events += num; i = 0; + if (odp_unlikely(uarea_size)) + uarea_sum += rw_uarea(ev, num, uarea_rd, uarea_rw); + if (odp_unlikely(ctx_size)) { queue_context_t *ctx = odp_queue_context(queue); @@ -1085,7 +1139,7 @@ static int test_sched(void *arg) global->stat[thr].nsec = nsec; global->stat[thr].cycles = cycles; global->stat[thr].waits = waits; - global->stat[thr].dummy_sum = data_sum + ctx_sum; + global->stat[thr].dummy_sum = data_sum + ctx_sum + uarea_sum; global->stat[thr].failed = ret; if (odp_atomic_fetch_dec_u32(&global->num_worker) == 1) { From af4e94014b7a4130f0761ac070df7b1eae6ddf4e Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 16 Jan 2024 15:12:03 +0200 Subject: [PATCH 054/147] validation: align: add tests Add validation tests for align module macros and defines. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale Reviewed-by: Petri Savolainen --- test/m4/configure.m4 | 1 + test/validation/api/Makefile.am | 4 +- test/validation/api/align/.gitignore | 1 + test/validation/api/align/Makefile.am | 4 + test/validation/api/align/align.c | 156 ++++++++++++++++++++++++++ 5 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 test/validation/api/align/.gitignore create mode 100644 test/validation/api/align/Makefile.am create mode 100644 test/validation/api/align/align.c diff --git a/test/m4/configure.m4 b/test/m4/configure.m4 index 4335476ba..84d2fd753 100644 --- a/test/m4/configure.m4 +++ b/test/m4/configure.m4 @@ -17,6 +17,7 @@ AC_CONFIG_FILES([test/common/Makefile test/miscellaneous/Makefile test/performance/Makefile test/validation/Makefile + test/validation/api/align/Makefile test/validation/api/atomic/Makefile test/validation/api/barrier/Makefile test/validation/api/buffer/Makefile diff --git a/test/validation/api/Makefile.am b/test/validation/api/Makefile.am index b109c1b38..49cd0f4aa 100644 --- a/test/validation/api/Makefile.am +++ b/test/validation/api/Makefile.am @@ -1,4 +1,5 @@ -ODP_MODULES = atomic \ +ODP_MODULES = align \ + atomic \ barrier \ buffer \ chksum \ @@ -34,6 +35,7 @@ include $(top_srcdir)/test/Makefile.inc TESTS_ENVIRONMENT += TEST_DIR=${top_builddir}/test/validation/api TESTS = \ + align/align_main$(EXEEXT) \ atomic/atomic_main$(EXEEXT) \ barrier/barrier_main$(EXEEXT) \ buffer/buffer_main$(EXEEXT) \ diff --git a/test/validation/api/align/.gitignore b/test/validation/api/align/.gitignore new file mode 100644 index 000000000..3031151a1 --- /dev/null +++ b/test/validation/api/align/.gitignore @@ -0,0 +1 @@ +align_main diff --git a/test/validation/api/align/Makefile.am b/test/validation/api/align/Makefile.am new file mode 100644 index 000000000..dd6e36bcd --- /dev/null +++ b/test/validation/api/align/Makefile.am @@ -0,0 +1,4 @@ +include ../Makefile.inc + +test_PROGRAMS = align_main +align_main_SOURCES = align.c diff --git a/test/validation/api/align/align.c b/test/validation/api/align/align.c new file mode 100644 index 000000000..ddbd1a2fe --- /dev/null +++ b/test/validation/api/align/align.c @@ -0,0 +1,156 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2024 Nokia + */ + +#include +#include + +#include +#include + +/* Test struct without gaps */ +typedef struct ODP_PACKED { + uint8_t a; + uint8_t b; + uint16_t c; + uint32_t d; +} test_type_t; + +/* Test struct with gaps */ +typedef struct ODP_PACKED { + uint8_t a; + uint16_t b; + uint8_t c; + uint32_t d; +} test_type_2_t; + +static void test_aligned(void) +{ + uint8_t align_2 ODP_ALIGNED(2); + uint16_t align_4 ODP_ALIGNED(4); + uint32_t align_8 ODP_ALIGNED(8); + uint64_t align_16 ODP_ALIGNED(16); + + CU_ASSERT((uintptr_t)&align_2 % 2 == 0); + CU_ASSERT((uintptr_t)&align_4 % 4 == 0); + CU_ASSERT((uintptr_t)&align_8 % 8 == 0); + CU_ASSERT((uintptr_t)&align_16 % 16 == 0); +} + +static void test_packed(void) +{ + uint32_t offset; + + offset = 0; + CU_ASSERT(offsetof(test_type_t, a) == offset); + + offset += sizeof(uint8_t); + CU_ASSERT(offsetof(test_type_t, b) == offset); + + offset += sizeof(uint8_t); + CU_ASSERT(offsetof(test_type_t, c) == offset); + + offset += sizeof(uint16_t); + CU_ASSERT(offsetof(test_type_t, d) == offset); + + offset = 0; + CU_ASSERT(offsetof(test_type_2_t, a) == offset); + + offset += sizeof(uint8_t); + CU_ASSERT(offsetof(test_type_2_t, b) == offset); + + offset += sizeof(uint16_t); + CU_ASSERT(offsetof(test_type_2_t, c) == offset); + + offset += sizeof(uint8_t); + CU_ASSERT(offsetof(test_type_2_t, d) == offset); +} + +static void test_offsetof(void) +{ + CU_ASSERT(ODP_OFFSETOF(test_type_t, a) == offsetof(test_type_t, a)); + CU_ASSERT(ODP_OFFSETOF(test_type_t, b) == offsetof(test_type_t, b)); + CU_ASSERT(ODP_OFFSETOF(test_type_t, c) == offsetof(test_type_t, c)); + CU_ASSERT(ODP_OFFSETOF(test_type_t, d) == offsetof(test_type_t, d)); + CU_ASSERT(ODP_OFFSETOF(test_type_2_t, a) == offsetof(test_type_2_t, a)); + CU_ASSERT(ODP_OFFSETOF(test_type_2_t, b) == offsetof(test_type_2_t, b)); + CU_ASSERT(ODP_OFFSETOF(test_type_2_t, c) == offsetof(test_type_2_t, c)); + CU_ASSERT(ODP_OFFSETOF(test_type_2_t, d) == offsetof(test_type_2_t, d)); +} + +static void test_field_sizeof(void) +{ + test_type_t tt; + + CU_ASSERT(ODP_FIELD_SIZEOF(test_type_t, a) == sizeof(tt.a)); + CU_ASSERT(ODP_FIELD_SIZEOF(test_type_t, b) == sizeof(tt.b)); + CU_ASSERT(ODP_FIELD_SIZEOF(test_type_t, c) == sizeof(tt.c)); + CU_ASSERT(ODP_FIELD_SIZEOF(test_type_t, d) == sizeof(tt.d)); +} + +static void test_cache_line_size(void) +{ + CU_ASSERT(ODP_CACHE_LINE_SIZE > 0); + CU_ASSERT(ODP_CACHE_LINE_SIZE % 2 == 0); +} + +static void test_page_size(void) +{ + CU_ASSERT(ODP_PAGE_SIZE > 0); + CU_ASSERT(ODP_PAGE_SIZE % 2 == 0); +} + +static void test_aligned_cache(void) +{ + uint8_t arr[123] ODP_ALIGNED_CACHE; + + CU_ASSERT((uintptr_t)arr % ODP_CACHE_LINE_SIZE == 0); +} + +static void test_aligned_page(void) +{ + uint8_t arr[123] ODP_ALIGNED_PAGE; + + CU_ASSERT((uintptr_t)arr % ODP_PAGE_SIZE == 0); +} + +static void test_cache_line_roundup(void) +{ + CU_ASSERT(ODP_CACHE_LINE_ROUNDUP(123) % ODP_CACHE_LINE_SIZE == 0); + CU_ASSERT(ODP_CACHE_LINE_ROUNDUP(ODP_CACHE_LINE_SIZE) == ODP_CACHE_LINE_SIZE); + CU_ASSERT(ODP_CACHE_LINE_ROUNDUP(0) == 0); +} + +odp_testinfo_t align_suite[] = { + ODP_TEST_INFO(test_aligned), + ODP_TEST_INFO(test_packed), + ODP_TEST_INFO(test_offsetof), + ODP_TEST_INFO(test_field_sizeof), + ODP_TEST_INFO(test_cache_line_size), + ODP_TEST_INFO(test_page_size), + ODP_TEST_INFO(test_aligned_cache), + ODP_TEST_INFO(test_aligned_page), + ODP_TEST_INFO(test_cache_line_roundup), + ODP_TEST_INFO_NULL, +}; + +odp_suiteinfo_t align_suites[] = { + {"align", NULL, NULL, align_suite}, + ODP_SUITE_INFO_NULL +}; + +int main(int argc, char *argv[]) +{ + int ret; + + /* Parse common options */ + if (odp_cunit_parse_options(&argc, argv)) + return -1; + + ret = odp_cunit_register(align_suites); + + if (ret == 0) + ret = odp_cunit_run(); + + return ret; +} From e1a65f2fe33354dad14bfb082a66772d2c01f56a Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 17 Jan 2024 09:57:07 +0200 Subject: [PATCH 055/147] validation: hints: add tests Add validation tests for hints module macros and attributes. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale Reviewed-by: Petri Savolainen --- test/m4/configure.m4 | 1 + test/validation/api/Makefile.am | 2 + test/validation/api/hints/.gitignore | 1 + test/validation/api/hints/Makefile.am | 4 ++ test/validation/api/hints/hints.c | 92 +++++++++++++++++++++++++++ 5 files changed, 100 insertions(+) create mode 100644 test/validation/api/hints/.gitignore create mode 100644 test/validation/api/hints/Makefile.am create mode 100644 test/validation/api/hints/hints.c diff --git a/test/m4/configure.m4 b/test/m4/configure.m4 index 84d2fd753..b979f400d 100644 --- a/test/m4/configure.m4 +++ b/test/m4/configure.m4 @@ -30,6 +30,7 @@ AC_CONFIG_FILES([test/common/Makefile test/validation/api/errno/Makefile test/validation/api/event/Makefile test/validation/api/hash/Makefile + test/validation/api/hints/Makefile test/validation/api/init/Makefile test/validation/api/ipsec/Makefile test/validation/api/lock/Makefile diff --git a/test/validation/api/Makefile.am b/test/validation/api/Makefile.am index 49cd0f4aa..558c2c8bb 100644 --- a/test/validation/api/Makefile.am +++ b/test/validation/api/Makefile.am @@ -11,6 +11,7 @@ ODP_MODULES = align \ errno \ event \ hash \ + hints \ init \ ipsec \ lock \ @@ -48,6 +49,7 @@ TESTS = \ errno/errno_main$(EXEEXT) \ event/event_main$(EXEEXT) \ hash/hash_main$(EXEEXT) \ + hints/hints_main$(EXEEXT) \ init/init_defaults.sh \ init/init_abort.sh \ init/init_log.sh \ diff --git a/test/validation/api/hints/.gitignore b/test/validation/api/hints/.gitignore new file mode 100644 index 000000000..586f429bc --- /dev/null +++ b/test/validation/api/hints/.gitignore @@ -0,0 +1 @@ +hints_main diff --git a/test/validation/api/hints/Makefile.am b/test/validation/api/hints/Makefile.am new file mode 100644 index 000000000..bcc77f606 --- /dev/null +++ b/test/validation/api/hints/Makefile.am @@ -0,0 +1,4 @@ +include ../Makefile.inc + +test_PROGRAMS = hints_main +hints_main_SOURCES = hints.c diff --git a/test/validation/api/hints/hints.c b/test/validation/api/hints/hints.c new file mode 100644 index 000000000..4c049f33b --- /dev/null +++ b/test/validation/api/hints/hints.c @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2024 Nokia + */ + +#include +#include + +#include +#include + +ODP_NORETURN static void test_noreturn(void) +{ + abort(); +} + +int test_weak(void); + +ODP_WEAK_SYMBOL int test_weak(void) +{ + return 0; +} + +ODP_COLD_CODE static int test_cold(void) +{ + return -1; +} + +ODP_HOT_CODE static int test_hot(void) +{ + return 1; +} + +ODP_PRINTF_FORMAT(2, 3) +static int test_printf_format(int level ODP_UNUSED, const char *fmt ODP_UNUSED, ...) +{ + return 0; +} + +static void test_hints(void) +{ + volatile int val = 1; + + if (odp_unlikely(!val)) + test_noreturn(); + + test_weak(); + test_cold(); + + if (odp_likely(val)) + test_hot(); + + test_printf_format(0, "test"); +} + +static void test_prefetch(void) +{ + const int rounds = 10; + uint64_t data[rounds]; + + for (int i = 0; i < rounds; i++) + odp_prefetch(&data[i]); + + for (int i = 0; i < rounds; i++) + odp_prefetch_store(&data[i]); +} + +odp_testinfo_t hints_suite[] = { + ODP_TEST_INFO(test_hints), + ODP_TEST_INFO(test_prefetch), + ODP_TEST_INFO_NULL, +}; + +odp_suiteinfo_t align_suites[] = { + {"hints", NULL, NULL, hints_suite}, + ODP_SUITE_INFO_NULL +}; + +int main(int argc, char *argv[]) +{ + int ret; + + /* Parse common options */ + if (odp_cunit_parse_options(&argc, argv)) + return -1; + + ret = odp_cunit_register(align_suites); + + if (ret == 0) + ret = odp_cunit_run(); + + return ret; +} From cc974b71adf04dcc5e61ba46e39b92a1de92c8d8 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 18 Jan 2024 09:24:04 +0200 Subject: [PATCH 056/147] validation: byteorder: add tests Add validation tests for byteorder module APIs. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale Reviewed-by: Petri Savolainen --- test/m4/configure.m4 | 1 + test/validation/api/Makefile.am | 2 + test/validation/api/byteorder/.gitignore | 1 + test/validation/api/byteorder/Makefile.am | 4 + test/validation/api/byteorder/byteorder.c | 107 ++++++++++++++++++++++ 5 files changed, 115 insertions(+) create mode 100644 test/validation/api/byteorder/.gitignore create mode 100644 test/validation/api/byteorder/Makefile.am create mode 100644 test/validation/api/byteorder/byteorder.c diff --git a/test/m4/configure.m4 b/test/m4/configure.m4 index b979f400d..7dbaa7fcd 100644 --- a/test/m4/configure.m4 +++ b/test/m4/configure.m4 @@ -21,6 +21,7 @@ AC_CONFIG_FILES([test/common/Makefile test/validation/api/atomic/Makefile test/validation/api/barrier/Makefile test/validation/api/buffer/Makefile + test/validation/api/byteorder/Makefile test/validation/api/chksum/Makefile test/validation/api/classification/Makefile test/validation/api/comp/Makefile diff --git a/test/validation/api/Makefile.am b/test/validation/api/Makefile.am index 558c2c8bb..d9e7aa86e 100644 --- a/test/validation/api/Makefile.am +++ b/test/validation/api/Makefile.am @@ -2,6 +2,7 @@ ODP_MODULES = align \ atomic \ barrier \ buffer \ + byteorder \ chksum \ classification \ comp \ @@ -40,6 +41,7 @@ TESTS = \ atomic/atomic_main$(EXEEXT) \ barrier/barrier_main$(EXEEXT) \ buffer/buffer_main$(EXEEXT) \ + byteorder/byteorder_main$(EXEEXT) \ chksum/chksum_main$(EXEEXT) \ classification/classification_main$(EXEEXT) \ comp/comp_main$(EXEEXT) \ diff --git a/test/validation/api/byteorder/.gitignore b/test/validation/api/byteorder/.gitignore new file mode 100644 index 000000000..0b0d91053 --- /dev/null +++ b/test/validation/api/byteorder/.gitignore @@ -0,0 +1 @@ +byteorder_main diff --git a/test/validation/api/byteorder/Makefile.am b/test/validation/api/byteorder/Makefile.am new file mode 100644 index 000000000..186f4ecbc --- /dev/null +++ b/test/validation/api/byteorder/Makefile.am @@ -0,0 +1,4 @@ +include ../Makefile.inc + +test_PROGRAMS = byteorder_main +byteorder_main_SOURCES = byteorder.c diff --git a/test/validation/api/byteorder/byteorder.c b/test/validation/api/byteorder/byteorder.c new file mode 100644 index 000000000..087dfce95 --- /dev/null +++ b/test/validation/api/byteorder/byteorder.c @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2024 Nokia + */ + +#include +#include + +#include + +static void test_defines(void) +{ + /* Endianness */ + CU_ASSERT(ODP_BIG_ENDIAN || ODP_LITTLE_ENDIAN); + + if (ODP_BIG_ENDIAN) { + CU_ASSERT(ODP_BYTE_ORDER == ODP_BIG_ENDIAN); + CU_ASSERT(!ODP_LITTLE_ENDIAN); + } + + if (ODP_LITTLE_ENDIAN) { + CU_ASSERT(ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN); + CU_ASSERT(!ODP_BIG_ENDIAN); + } + + /* Bitfield endianness */ + CU_ASSERT(ODP_BIG_ENDIAN_BITFIELD || ODP_LITTLE_ENDIAN_BITFIELD); + + if (ODP_BIG_ENDIAN_BITFIELD) { + CU_ASSERT(ODP_BITFIELD_ORDER == ODP_BIG_ENDIAN_BITFIELD); + CU_ASSERT(!ODP_LITTLE_ENDIAN_BITFIELD); + } + + if (ODP_LITTLE_ENDIAN_BITFIELD) { + CU_ASSERT(ODP_BITFIELD_ORDER == ODP_LITTLE_ENDIAN_BITFIELD); + CU_ASSERT(!ODP_BIG_ENDIAN_BITFIELD); + } +} + +static void test_types(void) +{ + const uint16_t u16_val = 0x1234; + const uint32_t u32_val = 0x12345678; + const uint64_t u64_val = 0x1234567890123456; + const uint16_t u16_val_conv = 0x3412; + const uint32_t u32_val_conv = 0x78563412; + const uint64_t u64_val_conv = 0x5634129078563412; + odp_u16be_t be16 = odp_cpu_to_be_16(u16_val); + odp_u32be_t be32 = odp_cpu_to_be_32(u32_val); + odp_u64be_t be64 = odp_cpu_to_be_64(u64_val); + odp_u16le_t le16 = odp_cpu_to_le_16(u16_val); + odp_u32le_t le32 = odp_cpu_to_le_32(u32_val); + odp_u64le_t le64 = odp_cpu_to_le_64(u64_val); + odp_u16sum_t sum16 = u16_val; + odp_u32sum_t sum32 = u16_val; + + CU_ASSERT(sum16 == sum32); + + if (ODP_BIG_ENDIAN) { + CU_ASSERT(be16 == u16_val); + CU_ASSERT(be32 == u32_val); + CU_ASSERT(be64 == u64_val); + CU_ASSERT(le16 == u16_val_conv); + CU_ASSERT(le32 == u32_val_conv); + CU_ASSERT(le64 == u64_val_conv); + } else { + CU_ASSERT(le16 == u16_val); + CU_ASSERT(le32 == u32_val); + CU_ASSERT(le64 == u64_val); + CU_ASSERT(be16 == u16_val_conv); + CU_ASSERT(be32 == u32_val_conv); + CU_ASSERT(be64 == u64_val_conv); + } + + CU_ASSERT(odp_be_to_cpu_16(be16) == u16_val); + CU_ASSERT(odp_be_to_cpu_32(be32) == u32_val); + CU_ASSERT(odp_be_to_cpu_64(be64) == u64_val); + CU_ASSERT(odp_le_to_cpu_16(le16) == u16_val); + CU_ASSERT(odp_le_to_cpu_32(le32) == u32_val); + CU_ASSERT(odp_le_to_cpu_64(le64) == u64_val); +} + +odp_testinfo_t byteorder_suite[] = { + ODP_TEST_INFO(test_defines), + ODP_TEST_INFO(test_types), + ODP_TEST_INFO_NULL, +}; + +odp_suiteinfo_t byteorder_suites[] = { + {"byteorder", NULL, NULL, byteorder_suite}, + ODP_SUITE_INFO_NULL +}; + +int main(int argc, char *argv[]) +{ + int ret; + + /* Parse common options */ + if (odp_cunit_parse_options(&argc, argv)) + return -1; + + ret = odp_cunit_register(byteorder_suites); + + if (ret == 0) + ret = odp_cunit_run(); + + return ret; +} From 5bebc1cc273f353575032ebed85e866bcf111ff9 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 16 Jan 2024 09:45:09 +0200 Subject: [PATCH 057/147] linux-gen: time: improve TSC frequency measurement Speed up TSC frequency measurement and thus ODP startup by reducing the measurement duration. Make the result more accurate despite the decreased duration by using a busyloop instead of nanosleep and by taking the median of several measurements. Using a busyloop instead of nanosleep also fixes the measurement in (old) systems that have constant but not invariant TSC which may stop or change frequency in C-states. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale --- .../linux-generic/arch/x86/odp_time_cpu.c | 96 +++++++++++++------ 1 file changed, 66 insertions(+), 30 deletions(-) diff --git a/platform/linux-generic/arch/x86/odp_time_cpu.c b/platform/linux-generic/arch/x86/odp_time_cpu.c index aa00ac04e..ab897296d 100644 --- a/platform/linux-generic/arch/x86/odp_time_cpu.c +++ b/platform/linux-generic/arch/x86/odp_time_cpu.c @@ -1,7 +1,6 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2024 Nokia */ #include @@ -14,42 +13,77 @@ #include #include +#include +#include -/* Measure TSC frequency. Frequency information registers are defined for x86, - * but those are often not enumerated. */ +static int nwait(uint64_t nsec) +{ + struct timespec ts1, ts2; + uint64_t diff; + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) + return 1; + + do { + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) + return 1; + + diff = (ts2.tv_sec - ts1.tv_sec) * ODP_TIME_SEC_IN_NS + + ts2.tv_nsec - ts1.tv_nsec; + } while (diff < nsec); + + return 0; +} + +static void sort(uint64_t values[], int num) +{ + for (int n = 0; n < num; n++) { + for (int i = n + 1; i < num; i++) { + if (values[i] < values[n]) { + uint64_t tmp = values[i]; + + values[i] = values[n]; + values[n] = tmp; + } + } + } +} + +static uint64_t median(uint64_t values[], int num) +{ + sort(values, num); + if (num % 2 == 0) + return (values[num / 2 - 1] + values[num / 2]) / 2; + else + return values[num / 2]; +} + +/* Measure TSC frequency. */ uint64_t _odp_time_cpu_global_freq(void) { - struct timespec sleep, ts1, ts2; - uint64_t t1, t2, ts_nsec, cycles, hz; + struct timespec ts1, ts2; + uint64_t t1, t2, ts_nsec, cycles; int i; - uint64_t avg = 0; - int rounds = 3; + const int rounds = 6; /* first round is warmup */ int warm_up = 1; + uint64_t hz[rounds]; for (i = 0; i < rounds; i++) { - sleep.tv_sec = 0; + uint64_t wait_nsec = ODP_TIME_SEC_IN_NS / 50; if (warm_up) - sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 1000; - else - sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 4; + wait_nsec = ODP_TIME_SEC_IN_NS / 1000; - if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) { - _ODP_ERR("clock_gettime() failed\n"); - return 0; - } + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) + goto err_out; t1 = _odp_time_cpu_global(); - if (nanosleep(&sleep, NULL) < 0) { - _ODP_ERR("nanosleep() failed\n"); - return 0; - } + if (nwait(wait_nsec)) + goto err_out; - if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) { - _ODP_ERR("clock_gettime() failed\n"); - return 0; - } + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) + goto err_out; t2 = _odp_time_cpu_global(); @@ -58,13 +92,15 @@ uint64_t _odp_time_cpu_global_freq(void) cycles = t2 - t1; - hz = (cycles * ODP_TIME_SEC_IN_NS) / ts_nsec; + hz[i] = (cycles * ODP_TIME_SEC_IN_NS) / ts_nsec; if (warm_up) warm_up = 0; - else - avg += hz; } - return avg / (rounds - 1); + return median(&hz[1], rounds - 1); + +err_out: + _ODP_ERR("clock_gettime() failed (%s)\n", strerror(errno)); + return 0; } From 132cfb7f3d6cdac60ba3a9f7d5fca1d580e8e157 Mon Sep 17 00:00:00 2001 From: Janne Peltonen Date: Tue, 16 Jan 2024 09:45:11 +0200 Subject: [PATCH 058/147] linux-gen: sysinfo: do not save cpu frequency if it is not static Do not read and save the current CPU frequency if the frequency is not marked static in the config file. In that case the saved frequency information is not used. Signed-off-by: Janne Peltonen Reviewed-by: Tuomas Taipale --- platform/linux-generic/odp_system_info.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 52f1000f1..d8aa272cc 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -386,8 +386,9 @@ int _odp_system_info_init(void) num_cpus); /* Read and save all CPU frequencies for static mode */ - for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) - odp_global_ro.system_info.cpu_hz[i] = cpu_hz_current(i); + if (odp_global_ro.system_info.cpu_hz_static) + for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) + odp_global_ro.system_info.cpu_hz[i] = cpu_hz_current(i); /* By default, read max frequency from a cpufreq file */ for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) { From f74bd07adc50885b54036496b2f9993ebffb5bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jere=20Lepp=C3=A4nen?= Date: Tue, 21 Nov 2023 11:25:00 +0200 Subject: [PATCH 059/147] linux-gen: remove unnecessary pthread includes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove inclusion of pthread.h from files that don't actually need it. Signed-off-by: Jere Leppänen Reviewed-by: Petri Savolainen --- platform/linux-generic/include/odp_global_data.h | 1 - platform/linux-generic/odp_cpumask.c | 1 - platform/linux-generic/odp_cpumask_task.c | 1 - platform/linux-generic/odp_system_info.c | 1 - 4 files changed, 4 deletions(-) diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index f883cefd9..f00e155de 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -21,7 +21,6 @@ extern "C" { #include #include -#include #include #include diff --git a/platform/linux-generic/odp_cpumask.c b/platform/linux-generic/odp_cpumask.c index d0a9953f7..7d7575f51 100644 --- a/platform/linux-generic/odp_cpumask.c +++ b/platform/linux-generic/odp_cpumask.c @@ -7,7 +7,6 @@ #include #include -#include #include #include diff --git a/platform/linux-generic/odp_cpumask_task.c b/platform/linux-generic/odp_cpumask_task.c index 0807e231e..a579b2e7e 100644 --- a/platform/linux-generic/odp_cpumask_task.c +++ b/platform/linux-generic/odp_cpumask_task.c @@ -12,7 +12,6 @@ #include #include -#include #include int odp_cpumask_default_worker(odp_cpumask_t *mask, int max_num) diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index d8aa272cc..bea77fb23 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -26,7 +26,6 @@ #include #include -#include #include #include #include From 90dd758b73034ac12caf29e7ddab4ad6d3cbbbaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jere=20Lepp=C3=A4nen?= Date: Mon, 13 Mar 2023 19:05:25 +0200 Subject: [PATCH 060/147] linux-gen: timer: return time to the next timeout from timer_run() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In timer_run(), return the time in nsec to the next timeout. If timer_run() expires a timer, return zero. Signed-off-by: Jere Leppänen Reviewed-by: Petri Savolainen --- .../include/odp_timer_internal.h | 8 +++-- platform/linux-generic/odp_timer.c | 30 ++++++++++++------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 01ee4a0f3..9c9852de9 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -48,13 +48,15 @@ ODP_STATIC_ASSERT(sizeof(odp_timeout_hdr_t) <= ODP_CACHE_LINE_SIZE, /* A larger decrement value should be used after receiving events compared to * an 'empty' call. */ -void _odp_timer_run_inline(int dec); +uint64_t _odp_timer_run_inline(int dec); /* Static inline wrapper to minimize modification of schedulers. */ -static inline void timer_run(int dec) +static inline uint64_t timer_run(int dec) { if (odp_global_rw->inline_timers) - _odp_timer_run_inline(dec); + return _odp_timer_run_inline(dec); + + return UINT64_MAX; } #endif diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index daf187390..0b582e18b 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -761,11 +761,12 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) } } -static inline void timer_pool_scan(timer_pool_t *tp, uint64_t tick) +static inline uint64_t timer_pool_scan(timer_pool_t *tp, uint64_t tick) { tick_buf_t *array = &tp->tick_buf[0]; uint32_t high_wm = odp_atomic_load_acq_u32(&tp->high_wm); uint32_t i; + uint64_t min = UINT64_MAX; _ODP_ASSERT(high_wm <= tp->param.num_timers); for (i = 0; i < high_wm; i++) { @@ -780,18 +781,23 @@ static inline void timer_pool_scan(timer_pool_t *tp, uint64_t tick) if (odp_unlikely(exp_tck <= tick)) { /* Attempt to expire timer */ timer_expire(tp, i, tick); + min = 0; + } else { + min = _ODP_MIN(min, exp_tck - tick); } } + + return min; } /****************************************************************************** * Inline timer processing *****************************************************************************/ -static inline void timer_pool_scan_inline(int num, odp_time_t now) +static inline uint64_t timer_pool_scan_inline(int num, odp_time_t now) { timer_pool_t *tp; - uint64_t new_tick, old_tick, nsec; + uint64_t new_tick, old_tick, ticks_to_next_expire, nsec, min = UINT64_MAX; int64_t diff; int i; @@ -832,26 +838,29 @@ static inline void timer_pool_scan_inline(int num, odp_time_t now) odp_atomic_store_u32(&tp->notify_overrun, 2); } } - timer_pool_scan(tp, nsec); + ticks_to_next_expire = timer_pool_scan(tp, nsec); + min = _ODP_MIN(min, ticks_to_next_expire); } } + + return min; } -void _odp_timer_run_inline(int dec) +uint64_t _odp_timer_run_inline(int dec) { odp_time_t now; int num = timer_global->highest_tp_idx + 1; int poll_interval = timer_global->poll_interval; if (num == 0) - return; + return UINT64_MAX; /* Rate limit how often this thread checks the timer pools. */ if (poll_interval > 1) { timer_local.run_cnt -= dec; if (timer_local.run_cnt > 0) - return; + return UINT64_MAX; timer_local.run_cnt = poll_interval; } @@ -862,7 +871,7 @@ void _odp_timer_run_inline(int dec) if (odp_time_cmp(period, timer_global->poll_interval_time) < 0) - return; + return UINT64_MAX; timer_local.last_run = now; } @@ -870,13 +879,14 @@ void _odp_timer_run_inline(int dec) if (CONFIG_TIMER_PROFILE_INLINE) { odp_time_t t1 = odp_time_local_strict(); - timer_pool_scan_inline(num, now); + uint64_t ret = timer_pool_scan_inline(num, now); odp_time_t t2 = odp_time_local_strict(); timer_local.prof_nsec += odp_time_diff_ns(t2, t1); timer_local.prof_rounds++; + return ret; } else { - timer_pool_scan_inline(num, now); + return timer_pool_scan_inline(num, now); } } From 22c7067de77ff0ad39c01cd2525547eb3a9e6772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jere=20Lepp=C3=A4nen?= Date: Tue, 14 Mar 2023 11:29:42 +0200 Subject: [PATCH 061/147] linux-gen: timer: allow forcing a scan in _odp_timer_run_inline() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow forcing a timer scan in timer_run() / _odp_timer_run_inline() by providing a special parameter value TIMER_SCAN_FORCE. Signed-off-by: Jere Leppänen Reviewed-by: Petri Savolainen --- .../linux-generic/include/odp_timer_internal.h | 6 ++++++ platform/linux-generic/odp_timer.c | 16 +++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 9c9852de9..38192d917 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -22,6 +22,12 @@ #include #include +/* + * Use as the argument to timer_run() to force a scan and to ignore rate + * limit. + */ +#define TIMER_SCAN_FORCE INT32_MAX + /** * Internal Timeout header */ diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 0b582e18b..c8ea31078 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -794,7 +794,7 @@ static inline uint64_t timer_pool_scan(timer_pool_t *tp, uint64_t tick) * Inline timer processing *****************************************************************************/ -static inline uint64_t timer_pool_scan_inline(int num, odp_time_t now) +static inline uint64_t timer_pool_scan_inline(int num, odp_time_t now, int force) { timer_pool_t *tp; uint64_t new_tick, old_tick, ticks_to_next_expire, nsec, min = UINT64_MAX; @@ -823,7 +823,7 @@ static inline uint64_t timer_pool_scan_inline(int num, odp_time_t now) old_tick = odp_atomic_load_u64(&tp->cur_tick); diff = new_tick - old_tick; - if (diff < 1) + if (diff < 1 && !force) continue; if (odp_atomic_cas_u64(&tp->cur_tick, &old_tick, new_tick)) { @@ -850,7 +850,8 @@ uint64_t _odp_timer_run_inline(int dec) { odp_time_t now; int num = timer_global->highest_tp_idx + 1; - int poll_interval = timer_global->poll_interval; + int force = (dec == TIMER_SCAN_FORCE); + int poll_interval = force ? 0 : timer_global->poll_interval; if (num == 0) return UINT64_MAX; @@ -875,18 +876,23 @@ uint64_t _odp_timer_run_inline(int dec) timer_local.last_run = now; } + if (force) { + timer_local.run_cnt = poll_interval; + timer_local.last_run = now; + } + /* Check the timer pools. */ if (CONFIG_TIMER_PROFILE_INLINE) { odp_time_t t1 = odp_time_local_strict(); - uint64_t ret = timer_pool_scan_inline(num, now); + uint64_t ret = timer_pool_scan_inline(num, now, force); odp_time_t t2 = odp_time_local_strict(); timer_local.prof_nsec += odp_time_diff_ns(t2, t1); timer_local.prof_rounds++; return ret; } else { - return timer_pool_scan_inline(num, now); + return timer_pool_scan_inline(num, now, force); } } From 5cc40ab1c038831d9d9fafa97a6b6222d8141432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jere=20Lepp=C3=A4nen?= Date: Tue, 14 Mar 2023 11:24:38 +0200 Subject: [PATCH 062/147] linux-gen: sched: improve timer behavior when power saving sleep is enabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When using inline timers and power saving sleep, limit sleep duration according to the next timeout. Signed-off-by: Jere Leppänen Reviewed-by: Petri Savolainen --- config/odp-linux-generic.conf | 7 +++--- platform/linux-generic/odp_schedule_basic.c | 24 +++++++++++++++------ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index 63ac31fe2..2d27752b2 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -240,8 +240,9 @@ sched_basic: { # When waiting for events during a schedule call, save power by # sleeping in the poll loop. First, run schedule loop normally for # poll_time_nsec nanoseconds. If there are no events to schedule in that - # time, continue polling, but sleep for sleep_time_nsec nanoseconds on - # each round. + # time, continue polling, but sleep on each round. Sleep time is + # sleep_time_nsec nanoseconds, or the time to the next timer expiration, + # whichever is smaller. Timer pools are scanned just before sleep. # # During sleep, the thread is not polling for packet input or timers. # Each thread measures time and sleeps independently of other threads. @@ -257,7 +258,7 @@ sched_basic: { # Time in nsec to sleep # - # Actual sleep time may vary. + # Must be less than one second. Actual sleep time may vary. sleep_time_nsec = 0 } } diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 7bd8cbfed..3b0a4dce9 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -44,6 +44,7 @@ #include #include +#include /* No synchronization context */ #define NO_SYNC_CONTEXT ODP_SCHED_SYNC_PARALLEL @@ -297,7 +298,7 @@ typedef struct { struct { uint32_t poll_time; - struct timespec sleep_time; + uint64_t sleep_time; } powersave; /* Scheduler interface config options (not used in fast path) */ @@ -545,8 +546,8 @@ static int read_config_file(sched_global_t *sched) } val = _ODP_MAX(0, val); - sched->powersave.sleep_time.tv_sec = val / 1000000000; - sched->powersave.sleep_time.tv_nsec = val % 1000000000; + val = _ODP_MIN((int)ODP_TIME_SEC_IN_NS - 1, val); + sched->powersave.sleep_time = val; _ODP_PRINT(" %s: %i\n", str, val); _ODP_PRINT(" dynamic load balance: %s\n", sched->load_balance ? "ON" : "OFF"); @@ -1672,7 +1673,7 @@ static inline int schedule_loop_sleep(odp_queue_t *out_queue, uint64_t wait, timer_run(2); break; } - timer_run(1); + uint64_t next = timer_run(sleep ? TIMER_SCAN_FORCE : 1); if (first) { start = odp_time_local(); @@ -1683,8 +1684,19 @@ static inline int schedule_loop_sleep(odp_queue_t *out_queue, uint64_t wait, continue; } - if (sleep) - nanosleep(&sched->powersave.sleep_time, NULL); + if (sleep && next) { + uint64_t sleep_nsec = _ODP_MIN(sched->powersave.sleep_time, next); + + if (wait != ODP_SCHED_WAIT) { + uint64_t nsec_to_end = odp_time_diff_ns(end, current); + + sleep_nsec = _ODP_MIN(sleep_nsec, nsec_to_end); + } + + struct timespec ts = { 0, sleep_nsec }; + + nanosleep(&ts, NULL); + } if (wait != ODP_SCHED_WAIT || !sleep) { current = odp_time_local(); From 810928383f33680cfb93504a55c20a214ce5ef4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jere=20Lepp=C3=A4nen?= Date: Wed, 17 Jan 2024 11:29:46 +0200 Subject: [PATCH 063/147] linux-gen: sched: avoid odp_time_cmp() in schedule_loop_sleep() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorganize some conditionals in schedule_loop_sleep() in order to make them clearer, and to avoid calling odp_time_cmp() needlessly in some situations. Signed-off-by: Jere Leppänen Reviewed-by: Petri Savolainen --- platform/linux-generic/odp_schedule_basic.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 3b0a4dce9..379f1f828 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -1698,16 +1698,13 @@ static inline int schedule_loop_sleep(odp_queue_t *out_queue, uint64_t wait, nanosleep(&ts, NULL); } - if (wait != ODP_SCHED_WAIT || !sleep) { + if (!sleep || wait != ODP_SCHED_WAIT) current = odp_time_local(); - if (odp_time_cmp(start_sleep, current) < 0) - sleep = 1; - } - if (wait == ODP_SCHED_WAIT) - continue; + if (!sleep && odp_time_cmp(start_sleep, current) < 0) + sleep = 1; - if (odp_time_cmp(end, current) < 0) + if (wait != ODP_SCHED_WAIT && odp_time_cmp(end, current) < 0) break; } From b9cde2247d718e80a0af21e64eb2a45214a77376 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Mon, 29 Jan 2024 12:27:32 +0000 Subject: [PATCH 064/147] test: dma_perf: refactor timestamping Swap to use `odp_time_local_strict()` for timestamping for less overhead. Additionally, drop some redundant calculations with synchronous DMA and memory copy options. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/odp_dma_perf.c | 61 +++++++++++++++++---------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/test/performance/odp_dma_perf.c b/test/performance/odp_dma_perf.c index cc86775cd..b67c1fa23 100644 --- a/test/performance/odp_dma_perf.c +++ b/test/performance/odp_dma_perf.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2021-2023 Nokia + * Copyright (c) 2021-2024 Nokia */ /** @@ -109,7 +109,7 @@ typedef struct { odp_dma_transfer_param_t trs_param; odp_dma_compl_param_t compl_param; odp_ticketlock_t lock; - uint64_t trs_start_tm; + odp_time_t trs_start_tm; uint64_t trs_start_cc; uint64_t trs_poll_cnt; odp_bool_t is_running; @@ -873,20 +873,21 @@ static void free_memory(const sd_t *sd) static void run_transfer(odp_dma_t handle, trs_info_t *info, stats_t *stats, ver_fn_t ver_fn) { - uint64_t start_tm, end_tm, start_cc, end_cc, trs_tm, trs_cc, start_cc_diff; + odp_time_t start_tm, end_tm; + uint64_t start_cc, end_cc, trs_tm, trs_cc; odp_dma_result_t res; int ret; - start_tm = odp_time_local_strict_ns(); + start_tm = odp_time_local_strict(); start_cc = odp_cpu_cycles(); ret = odp_dma_transfer(handle, &info->trs_param, &res); end_cc = odp_cpu_cycles(); - end_tm = odp_time_local_strict_ns(); + end_tm = odp_time_local_strict(); if (odp_unlikely(ret <= 0)) { ++stats->start_errs; } else { - trs_tm = end_tm - start_tm; + trs_tm = odp_time_diff_ns(end_tm, start_tm); stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm); stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm); stats->trs_tm += trs_tm; @@ -895,10 +896,9 @@ static void run_transfer(odp_dma_t handle, trs_info_t *info, stats_t *stats, ver stats->min_trs_cc = ODPH_MIN(trs_cc, stats->min_trs_cc); stats->trs_cc += trs_cc; ++stats->trs_cnt; - start_cc_diff = odp_cpu_cycles_diff(end_cc, start_cc); - stats->max_start_cc = ODPH_MAX(start_cc_diff, stats->max_start_cc); - stats->min_start_cc = ODPH_MIN(start_cc_diff, stats->min_start_cc); - stats->start_cc += start_cc_diff; + stats->max_start_cc = stats->max_trs_cc; + stats->min_start_cc = stats->min_trs_cc; + stats->start_cc += trs_cc; ++stats->start_cnt; if (odp_unlikely(!res.success)) { @@ -969,7 +969,8 @@ static odp_bool_t configure_poll_compl(sd_t *sd) static void poll_transfer(sd_t *sd, trs_info_t *info, stats_t *stats) { - uint64_t start_cc, end_cc, trs_tm, trs_cc, wait_cc, start_tm, start_cc_diff; + uint64_t start_cc, end_cc, trs_tm, trs_cc, wait_cc, start_cc_diff; + odp_time_t start_tm; odp_dma_t handle = sd->dma.handle; odp_dma_result_t res; int ret; @@ -994,7 +995,7 @@ static void poll_transfer(sd_t *sd, trs_info_t *info, stats_t *stats) if (ret == 0) return; - trs_tm = odp_time_global_strict_ns() - info->trs_start_tm; + trs_tm = odp_time_diff_ns(odp_time_global_strict(), info->trs_start_tm); stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm); stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm); stats->trs_tm += trs_tm; @@ -1019,7 +1020,7 @@ static void poll_transfer(sd_t *sd, trs_info_t *info, stats_t *stats) if (sd->prep_trs_fn != NULL) sd->prep_trs_fn(sd, info); - start_tm = odp_time_global_strict_ns(); + start_tm = odp_time_global_strict(); start_cc = odp_cpu_cycles(); ret = odp_dma_transfer_start(handle, &info->trs_param, &info->compl_param); end_cc = odp_cpu_cycles(); @@ -1148,7 +1149,8 @@ static odp_bool_t configure_event_compl(sd_t *sd) static odp_bool_t start_initial_transfers(sd_t *sd) { - uint64_t start_tm, start_cc; + odp_time_t start_tm; + uint64_t start_cc; trs_info_t *info; int ret; @@ -1158,7 +1160,7 @@ static odp_bool_t start_initial_transfers(sd_t *sd) if (sd->prep_trs_fn != NULL) sd->prep_trs_fn(sd, info); - start_tm = odp_time_global_strict_ns(); + start_tm = odp_time_global_strict(); start_cc = odp_cpu_cycles(); ret = odp_dma_transfer_start(sd->dma.handle, &info->trs_param, &info->compl_param); @@ -1176,7 +1178,8 @@ static odp_bool_t start_initial_transfers(sd_t *sd) static void wait_compl_event(sd_t *sd, stats_t *stats) { - uint64_t start_cc, end_cc, wait_cc, trs_tm, trs_cc, start_tm, start_cc_diff; + uint64_t start_cc, end_cc, wait_cc, trs_tm, trs_cc, start_cc_diff; + odp_time_t start_tm; odp_event_t ev; odp_dma_result_t res; trs_info_t *info; @@ -1193,7 +1196,7 @@ static void wait_compl_event(sd_t *sd, stats_t *stats) odp_dma_compl_result(odp_dma_compl_from_event(ev), &res); info = res.user_ptr; - trs_tm = odp_time_global_strict_ns() - info->trs_start_tm; + trs_tm = odp_time_diff_ns(odp_time_global_strict(), info->trs_start_tm); stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm); stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm); stats->trs_tm += trs_tm; @@ -1220,7 +1223,7 @@ static void wait_compl_event(sd_t *sd, stats_t *stats) if (sd->prep_trs_fn != NULL) sd->prep_trs_fn(sd, info); - start_tm = odp_time_global_strict_ns(); + start_tm = odp_time_global_strict(); start_cc = odp_cpu_cycles(); ret = odp_dma_transfer_start(sd->dma.handle, &info->trs_param, &info->compl_param); end_cc = odp_cpu_cycles(); @@ -1252,7 +1255,8 @@ static void drain_compl_events(ODP_UNUSED sd_t *sd) static void run_memcpy(trs_info_t *info, stats_t *stats, ver_fn_t ver_fn) { - uint64_t start_tm, end_tm, start_cc, end_cc, trs_tm, trs_cc, start_cc_diff; + odp_time_t start_tm; + uint64_t start_cc, end_cc, trs_tm, trs_cc; const odp_dma_transfer_param_t *param = &info->trs_param; uint32_t tot_len, src_len, dst_len, min_len, len, i = 0U, j = 0U, src_off = 0U, dst_off = 0U, src_rem, dst_rem; @@ -1267,7 +1271,7 @@ static void run_memcpy(trs_info_t *info, stats_t *stats, ver_fn_t ver_fn) dst_len = param->dst_seg->len; min_len = ODPH_MIN(src_len, dst_len); len = min_len; - start_tm = odp_time_local_strict_ns(); + start_tm = odp_time_local_strict(); start_cc = odp_cpu_cycles(); while (tot_len > 0U) { @@ -1297,8 +1301,7 @@ static void run_memcpy(trs_info_t *info, stats_t *stats, ver_fn_t ver_fn) } end_cc = odp_cpu_cycles(); - end_tm = odp_time_local_strict_ns(); - trs_tm = end_tm - start_tm; + trs_tm = odp_time_diff_ns(odp_time_local_strict(), start_tm); stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm); stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm); stats->trs_tm += trs_tm; @@ -1307,10 +1310,9 @@ static void run_memcpy(trs_info_t *info, stats_t *stats, ver_fn_t ver_fn) stats->min_trs_cc = ODPH_MIN(trs_cc, stats->min_trs_cc); stats->trs_cc += trs_cc; ++stats->trs_cnt; - start_cc_diff = odp_cpu_cycles_diff(end_cc, start_cc); - stats->max_start_cc = ODPH_MAX(start_cc_diff, stats->max_start_cc); - stats->min_start_cc = ODPH_MIN(start_cc_diff, stats->min_start_cc); - stats->start_cc += start_cc_diff; + stats->max_start_cc = stats->max_trs_cc; + stats->min_start_cc = stats->min_trs_cc; + stats->start_cc += trs_cc; ++stats->start_cnt; ++stats->completed; @@ -1550,7 +1552,7 @@ static int transfer(void *args) stats_t *stats = &thr_config->stats; test_api_t *api = &prog_conf->api; odp_thrmask_t mask; - uint64_t start_tm, end_tm; + odp_time_t start_tm; odp_barrier_wait(&prog_config->init_barrier); @@ -1564,13 +1566,12 @@ static int transfer(void *args) } } - start_tm = odp_time_local_strict_ns(); + start_tm = odp_time_local_strict(); while (odp_atomic_load_u32(&prog_config->is_running)) api->wait_fn(sd, stats); - end_tm = odp_time_local_strict_ns(); - thr_config->stats.tot_tm = end_tm - start_tm; + thr_config->stats.tot_tm = odp_time_diff_ns(odp_time_local_strict(), start_tm); if (api->drain_fn != NULL) api->drain_fn(sd); From 3092328672057ba298180a644242b111ecf7358f Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Thu, 1 Feb 2024 09:30:55 +0200 Subject: [PATCH 065/147] github_ci: improve checkpatch testing for push operations Previously, in case of push operations only the latest commit was tested to avoid issues with invalid commit ranges in force pushes. Read force push status from GitHub events and test full commit range with normal push operations. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- .github/workflows/ci-pipeline.yml | 2 +- scripts/ci-checkpatches.sh | 18 +++++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 72b84eff3..7ae7348ef 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -28,7 +28,7 @@ jobs: run: | AFTER=${{ github.event.after }} BEFORE=${{ github.event.before }} - if [ -z "${BEFORE//0}" ] || [ -z "${AFTER//0}" ]; then + if [ -z "${BEFORE//0}" ] || [ -z "${AFTER//0}" ] || ${{ github.event.forced }}; then COMMIT_RANGE="" else COMMIT_RANGE="${BEFORE}..${AFTER}" diff --git a/scripts/ci-checkpatches.sh b/scripts/ci-checkpatches.sh index 383045cd7..4ce077342 100755 --- a/scripts/ci-checkpatches.sh +++ b/scripts/ci-checkpatches.sh @@ -2,22 +2,14 @@ PATCHES=$1 echo "Run checkpatch for ${PATCHES}" -# Generate patches provided with $1. -# In case of force push and range is broken -# validate only the latest commit if it's not merge commit. +# Generate patches provided with $1. If commit range is not available validate +# only the latest commit. if [ "$PATCHES" = "" ]; then git format-patch -1 -M HEAD; - perl ./scripts/checkpatch.pl *.patch; - exit $? +else + git format-patch ${PATCHES} fi -git show --summary HEAD| grep -q '^Merge:'; -if [ $? -ne 0 ]; then - git format-patch -1 -M HEAD; - perl ./scripts/checkpatch.pl *.patch; - exit $? -fi - -git format-patch ${PATCHES} perl ./scripts/checkpatch.pl *.patch; +exit $? From 51f6ec6920d22d9318a5a053b0985642fdf129f5 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Fri, 22 Dec 2023 13:40:09 +0000 Subject: [PATCH 066/147] test: performance: add pool latency tester `odp_pool_latency` is a simple pool latency tester for profiling pool allocation and free latencies under different load patterns. Test scenario consists of a configurable pattern (allocations/frees/delays) that is run continuously. All the currently specified pool types are available for testing. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/.gitignore | 1 + test/performance/Makefile.am | 2 + test/performance/odp_pool_latency.c | 1382 +++++++++++++++++++++++++++ 3 files changed, 1385 insertions(+) create mode 100644 test/performance/odp_pool_latency.c diff --git a/test/performance/.gitignore b/test/performance/.gitignore index 08a4d5609..46d9e9c2c 100644 --- a/test/performance/.gitignore +++ b/test/performance/.gitignore @@ -20,6 +20,7 @@ odp_mem_perf odp_packet_gen odp_pktio_ordered odp_pktio_perf +odp_pool_latency odp_pool_perf odp_queue_perf odp_random diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am index 7b0adbe09..bc239980d 100644 --- a/test/performance/Makefile.am +++ b/test/performance/Makefile.am @@ -12,6 +12,7 @@ EXECUTABLES = odp_atomic_perf \ odp_lock_perf \ odp_mem_perf \ odp_pktio_perf \ + odp_pool_latency \ odp_pool_perf \ odp_queue_perf \ odp_stash_perf \ @@ -81,6 +82,7 @@ odp_sched_latency_SOURCES = odp_sched_latency.c odp_sched_pktio_SOURCES = odp_sched_pktio.c odp_scheduling_SOURCES = odp_scheduling.c odp_pktio_perf_SOURCES = odp_pktio_perf.c +odp_pool_latency_SOURCES = odp_pool_latency.c odp_pool_perf_SOURCES = odp_pool_perf.c odp_queue_perf_SOURCES = odp_queue_perf.c odp_random_SOURCES = odp_random.c diff --git a/test/performance/odp_pool_latency.c b/test/performance/odp_pool_latency.c new file mode 100644 index 000000000..800a8dab1 --- /dev/null +++ b/test/performance/odp_pool_latency.c @@ -0,0 +1,1382 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2024 Nokia + */ + +/** + * @example odp_pool_latency.c + * + * Pool latency tester. Allocate from different kind of pools with a varying set of configurations + * and record latencies. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include + +#include +#include + +#define PROG_NAME "odp_pool_latency" +#define DELIMITER "," +#define ALLOC '+' +#define FREE '-' +#define TOP 't' +#define BOTTOM 'b' +#define DELAY 'd' + +enum { + BUFFER = 0U, + PACKET, + TMO, + VECTOR +}; + +enum { + SINGLE = 0U, + MANY +}; + +#define DEF_ALLOC 1U +#define DEF_FREE 1U +#define DEF_DIR TOP +#define DEF_TYPE BUFFER +#define DEF_CNT 32768U +#define DEF_SIZE 1024U +#define DEF_POLICY MANY +#define DEF_ROUNDS 100000U +#define DEF_IGNORE 0U +#define DEF_WORKERS 1U +#define DEF_UA_SIZE 0U + +#define MAX_PATTERN_LEN 32U +#define MAX_WORKERS ((uint32_t)(ODP_THREAD_COUNT_MAX - 1)) +#define MAX_RETRIES 10U + +#define COND_MIN(a, b) ((a) > 0U ? ODPH_MIN((a), (b)) : (b)) +#define UA_DATA 0xAA + +ODP_STATIC_ASSERT(MAX_PATTERN_LEN < UINT8_MAX, "Too long pattern length"); + +typedef struct { + uint32_t num_evs_buf; + uint32_t num_evs_pkt; + uint32_t num_evs_tmo; + uint32_t num_evs_vec; + uint32_t data_size_buf; + uint32_t data_size_pkt; + uint32_t data_size_vec; + uint32_t cache_size_buf; + uint32_t cache_size_pkt; + uint32_t cache_size_tmo; + uint32_t cache_size_vec; +} dynamic_defs_t; + +typedef enum { + PRS_OK, + PRS_NOK, + PRS_TERM +} parse_result_t; + +typedef struct { + uint64_t tot_tm; + uint64_t alloc_tm; + uint64_t max_alloc_tm; + uint64_t min_alloc_tm; + uint64_t max_alloc_rnd; + uint64_t min_alloc_rnd; + uint64_t alloc_cnt; + uint64_t alloc_b_cnt; + uint64_t uarea_tm; + uint64_t max_uarea_tm; + uint64_t min_uarea_tm; + uint64_t max_uarea_rnd; + uint64_t min_uarea_rnd; + uint64_t free_tm; + uint64_t max_free_tm; + uint64_t min_free_tm; + uint64_t max_free_rnd; + uint64_t min_free_rnd; + uint64_t free_b_cnt; + uint64_t reallocs; + uint64_t alloc_errs; + uint64_t pattern_errs; + uint8_t max_alloc_pt; + uint8_t min_alloc_pt; + uint8_t max_uarea_pt; + uint8_t min_uarea_pt; + uint8_t max_free_pt; + uint8_t min_free_pt; +} stats_t; + +typedef struct { + uint32_t val; + uint8_t op; + uint8_t opt; +} alloc_elem_t; + +typedef struct prog_config_s prog_config_t; + +typedef struct ODP_ALIGNED_CACHE { + stats_t stats; + odp_pool_t pool; + void *data; + prog_config_t *prog_config; + odp_shm_t shm; + uint32_t data_size; + uint32_t uarea_size; +} worker_config_t; + +typedef uint32_t (*alloc_fn_t)(worker_config_t *config, void *data, uint32_t idx, uint32_t num, + uint64_t round, uint8_t pattern, odp_bool_t is_saved); +typedef void (*free_fn_t)(void *data, uint32_t idx, uint32_t num, stats_t *stats, + uint64_t round, uint8_t pattern, odp_bool_t is_saved); + +typedef struct prog_config_s { + odph_thread_t thread_tbl[MAX_WORKERS]; + worker_config_t worker_config[MAX_WORKERS]; + alloc_elem_t alloc_elems[MAX_PATTERN_LEN]; + dynamic_defs_t dyn_defs; + odp_instance_t odp_instance; + odp_cpumask_t worker_mask; + odp_barrier_t init_barrier; + odp_barrier_t term_barrier; + alloc_fn_t alloc_fn; + free_fn_t free_fn; + int64_t cache_size; + uint32_t num_data_elems; + uint32_t seg_len; + uint32_t handle_size; + uint32_t num_evs; + uint32_t data_size; + uint32_t num_rounds; + uint32_t num_ignore; + uint32_t num_workers; + uint32_t uarea_size; + uint8_t num_elems; + uint8_t type; + uint8_t policy; +} prog_config_t; + +static prog_config_t *prog_conf; + +static void init_config(prog_config_t *config) +{ + alloc_elem_t *alloc_elem; + odp_pool_capability_t capa; + odp_pool_param_t param; + worker_config_t *worker; + + memset(config, 0, sizeof(*config)); + alloc_elem = &config->alloc_elems[0]; + alloc_elem->val = DEF_ALLOC; + alloc_elem->op = ALLOC; + alloc_elem = &config->alloc_elems[1]; + alloc_elem->val = DEF_FREE; + alloc_elem->op = FREE; + alloc_elem->opt = DEF_DIR; + config->num_elems = 2U; + + if (odp_pool_capability(&capa) == 0) { + config->dyn_defs.num_evs_buf = COND_MIN(capa.buf.max_num, DEF_CNT); + config->dyn_defs.num_evs_pkt = COND_MIN(capa.pkt.max_num, DEF_CNT); + config->dyn_defs.num_evs_tmo = COND_MIN(capa.tmo.max_num, DEF_CNT); + config->dyn_defs.num_evs_vec = COND_MIN(capa.vector.max_num, DEF_CNT); + config->dyn_defs.data_size_buf = COND_MIN(capa.buf.max_size, DEF_SIZE); + config->dyn_defs.data_size_pkt = COND_MIN(capa.pkt.max_len, DEF_SIZE); + config->dyn_defs.data_size_vec = COND_MIN(capa.vector.max_size, DEF_SIZE); + odp_pool_param_init(¶m); + config->dyn_defs.cache_size_buf = param.buf.cache_size; + config->dyn_defs.cache_size_pkt = param.pkt.cache_size; + config->dyn_defs.cache_size_tmo = param.tmo.cache_size; + config->dyn_defs.cache_size_vec = param.vector.cache_size; + } + + config->cache_size = -1; + config->num_rounds = DEF_ROUNDS; + config->num_ignore = DEF_IGNORE; + config->num_workers = DEF_WORKERS; + config->uarea_size = DEF_UA_SIZE; + config->type = DEF_TYPE; + config->policy = DEF_POLICY; + + for (uint32_t i = 0U; i < MAX_WORKERS; ++i) { + worker = &config->worker_config[i]; + worker->stats.min_alloc_tm = UINT64_MAX; + worker->stats.min_uarea_tm = UINT64_MAX; + worker->stats.min_free_tm = UINT64_MAX; + worker->pool = ODP_POOL_INVALID; + worker->shm = ODP_SHM_INVALID; + } +} + +static void parse_burst_pattern(prog_config_t *config, const char *optarg) +{ + char *tmp_str = strdup(optarg), *tmp, op, opt; + uint8_t num_elems = 0U; + alloc_elem_t *elem; + uint32_t val; + int ret; + + if (tmp_str == NULL) + return; + + tmp = strtok(tmp_str, DELIMITER); + + while (tmp && num_elems < MAX_PATTERN_LEN) { + elem = &config->alloc_elems[num_elems]; + ret = sscanf(tmp, "%c%u%c", &op, &val, &opt); + + if (ret == 2 || ret == 3) { + if (op == ALLOC || (op == FREE && (opt == TOP || opt == BOTTOM)) || + op == DELAY) { + if (op == FREE) + elem->opt = opt; + + elem->val = val; + elem->op = op; + ++num_elems; + } + } + + tmp = strtok(NULL, DELIMITER); + } + + free(tmp_str); + config->num_elems = num_elems; +} + +static void print_usage(const dynamic_defs_t *dyn_defs) +{ + printf("\n" + "Pool latency tester. Allocate from different kind of pools with a varying set of\n" + "configurations and record latencies.\n" + "\n" + "Usage: " PROG_NAME " [OPTIONS]\n"); + printf("\n" + " E.g. " PROG_NAME "\n" + " " PROG_NAME " -b %c7" DELIMITER "%c1%c" DELIMITER "%c3" DELIMITER "%c9%c\n", + ALLOC, FREE, TOP, ALLOC, FREE, BOTTOM); + printf(" " PROG_NAME " -b %c10" DELIMITER "%c1000" DELIMITER "%c10%c -t 1 -d 2048 " + "-p 0 -w 64\n", ALLOC, DELAY, FREE, TOP); + printf("\n" + "Optional OPTIONS:\n" + "\n" + " -b, --burst_pattern Burst pattern for allocations, frees and delays per round,\n" + " delimited by '%s', no spaces. Allocations are indicated\n" + " with a '%c' prefix, frees with a '%c' prefix. The location\n" + " of frees are indicated from the top of a previously\n" + " allocated array of events with a '%c' suffix and from the\n" + " bottom with a '%c' suffix. Delays are indicated with a\n" + " '%c' prefix, followed by a delay in nanoseconds.\n" + " Allocations and frees should be equal in the aggregate and\n" + " frees should never outnumber allocations at any instant.\n" + " '%c%u%s%c%u%c' by default. Maximum pattern length is %u.\n" + " -t, --type Pool type. %u by default.\n" + " 0: buffer\n" + " 1: packet\n" + " 2: timeout\n" + " 3: vector\n" + " -e, --event_count Number of events. Defaults:\n" + " buffer: %u\n" + " packet: %u\n" + " timeout: %u\n" + " vector: %u\n" + " -d, --data_size Data size in bytes, ignored in case of timeout pools, with\n" + " vector pools, defines the vector size.\n" + " Defaults:\n" + " buffer: %u\n" + " packet: %u\n" + " vector: %u\n" + " -p, --policy Pool allocation policy. %u by default.\n" + " Policies:\n" + " 0: One pool shared by workers\n" + " 1: One pool per worker\n" + " -r, --round_count Number of rounds to run. %u by default.\n" + " -i, --ignore_rounds Ignore an amount of initial rounds. %u by default.\n" + " -c, --worker_count Number of workers. %u by default.\n" + " -C, --cache_size Maximum cache size for pools. Defaults:\n" + " buffer: %u\n" + " packet: %u\n" + " timeout: %u\n" + " vector: %u\n" + " -w, --write_uarea Write data to allocated event user areas. 0 bytes disables\n" + " user area write. %u by default.\n" + " -h, --help This help.\n" + "\n", DELIMITER, ALLOC, FREE, TOP, BOTTOM, DELAY, ALLOC, DEF_ALLOC, DELIMITER, FREE, + DEF_FREE, DEF_DIR, MAX_PATTERN_LEN, DEF_TYPE, dyn_defs->num_evs_buf, + dyn_defs->num_evs_pkt, dyn_defs->num_evs_tmo, dyn_defs->num_evs_vec, + dyn_defs->data_size_buf, dyn_defs->data_size_pkt, dyn_defs->data_size_vec, + DEF_POLICY, DEF_ROUNDS, DEF_IGNORE, DEF_WORKERS, dyn_defs->cache_size_buf, + dyn_defs->cache_size_pkt, dyn_defs->cache_size_tmo, dyn_defs->cache_size_vec, + DEF_UA_SIZE); +} + +static parse_result_t check_options(prog_config_t *config) +{ + odp_pool_capability_t pool_capa; + uint32_t max_workers, num_pools; + alloc_elem_t *elem; + int64_t num_tot = 0; + odp_shm_capability_t shm_capa; + uint64_t shm_size; + + if (config->type != BUFFER && config->type != PACKET && config->type != TMO && + config->type != VECTOR) { + ODPH_ERR("Invalid pool type: %u\n", config->type); + return PRS_NOK; + } + + if (odp_pool_capability(&pool_capa) < 0) { + ODPH_ERR("Error querying pool capabilities\n"); + return PRS_NOK; + } + + max_workers = ODPH_MIN(MAX_WORKERS, (uint32_t)odp_cpumask_default_worker(NULL, 0)); + + if (config->num_workers == 0U || config->num_workers > max_workers) { + ODPH_ERR("Invalid worker count: %u (min: 1, max: %u)\n", config->num_workers, + max_workers); + return PRS_NOK; + } + + (void)odp_cpumask_default_worker(&config->worker_mask, config->num_workers); + num_pools = config->policy == SINGLE ? 1U : config->num_workers; + + if (config->type == BUFFER) { + if (config->num_evs == 0U) + config->num_evs = config->dyn_defs.num_evs_buf; + + if (config->data_size == 0U) + config->data_size = config->dyn_defs.data_size_buf; + + if (config->cache_size == -1) + config->cache_size = config->dyn_defs.cache_size_buf; + + if (config->num_evs > pool_capa.buf.max_num) { + ODPH_ERR("Invalid event count: %u (max: %u)\n", config->num_evs, + pool_capa.buf.max_num); + return PRS_NOK; + } + + if (config->data_size > pool_capa.buf.max_size) { + ODPH_ERR("Invalid data size: %u (max: %u)\n", config->data_size, + pool_capa.buf.max_size); + return PRS_NOK; + } + + if (config->cache_size < pool_capa.buf.min_cache_size || + config->cache_size > pool_capa.buf.max_cache_size) { + ODPH_ERR("Invalid cache size: %" PRIi64 " (min: %u, max: %u)\n", + config->cache_size, pool_capa.buf.min_cache_size, + pool_capa.buf.max_cache_size); + return PRS_NOK; + } + + if (num_pools > pool_capa.buf.max_pools) { + ODPH_ERR("Invalid pool count: %u (max: %u)\n", num_pools, + pool_capa.buf.max_pools); + return PRS_NOK; + } + + config->handle_size = sizeof(odp_buffer_t); + config->uarea_size = ODPH_MIN(config->uarea_size, pool_capa.buf.max_uarea_size); + } else if (config->type == PACKET) { + if (config->num_evs == 0U) + config->num_evs = config->dyn_defs.num_evs_pkt; + + if (config->data_size == 0U) + config->data_size = config->dyn_defs.data_size_pkt; + + if (config->cache_size == -1) + config->cache_size = config->dyn_defs.cache_size_pkt; + + if (config->num_evs > pool_capa.pkt.max_num) { + ODPH_ERR("Invalid event count: %u (max: %u)\n", config->num_evs, + pool_capa.pkt.max_num); + return PRS_NOK; + } + + if (config->data_size > pool_capa.pkt.max_len) { + ODPH_ERR("Invalid data size: %u (max: %u)\n", config->data_size, + pool_capa.pkt.max_len); + return PRS_NOK; + } + + if (config->cache_size < pool_capa.pkt.min_cache_size || + config->cache_size > pool_capa.pkt.max_cache_size) { + ODPH_ERR("Invalid cache size: %" PRIi64 " (min: %u, max: %u)\n", + config->cache_size, pool_capa.pkt.min_cache_size, + pool_capa.pkt.max_cache_size); + return PRS_NOK; + } + + if (num_pools > pool_capa.pkt.max_pools) { + ODPH_ERR("Invalid pool count: %u (max: %u)\n", num_pools, + pool_capa.pkt.max_pools); + return PRS_NOK; + } + + config->seg_len = pool_capa.pkt.max_seg_len > config->data_size ? + config->data_size : pool_capa.pkt.max_seg_len; + config->handle_size = sizeof(odp_packet_t); + config->uarea_size = ODPH_MIN(config->uarea_size, pool_capa.pkt.max_uarea_size); + } else if (config->type == TMO) { + if (config->num_evs == 0U) + config->num_evs = config->dyn_defs.num_evs_tmo; + + if (config->cache_size == -1) + config->cache_size = config->dyn_defs.cache_size_tmo; + + if (config->num_evs > pool_capa.tmo.max_num) { + ODPH_ERR("Invalid event count: %u (max: %u)\n", config->num_evs, + pool_capa.tmo.max_num); + return PRS_NOK; + } + + if (config->cache_size < pool_capa.tmo.min_cache_size || + config->cache_size > pool_capa.tmo.max_cache_size) { + ODPH_ERR("Invalid cache size: %" PRIi64 " (min: %u, max: %u)\n", + config->cache_size, pool_capa.tmo.min_cache_size, + pool_capa.tmo.max_cache_size); + return PRS_NOK; + } + + if (num_pools > pool_capa.tmo.max_pools) { + ODPH_ERR("Invalid pool count: %u (max: %u)\n", num_pools, + pool_capa.tmo.max_pools); + return PRS_NOK; + } + + config->handle_size = sizeof(odp_timeout_t); + config->uarea_size = ODPH_MIN(config->uarea_size, pool_capa.tmo.max_uarea_size); + } else { + if (config->num_evs == 0U) + config->num_evs = config->dyn_defs.num_evs_vec; + + if (config->data_size == 0U) + config->data_size = config->dyn_defs.data_size_vec; + + if (config->cache_size == -1) + config->cache_size = config->dyn_defs.cache_size_vec; + + if (config->num_evs > pool_capa.vector.max_num) { + ODPH_ERR("Invalid event count: %u (max: %u)\n", config->num_evs, + pool_capa.vector.max_num); + return PRS_NOK; + } + + if (config->data_size > pool_capa.vector.max_size) { + ODPH_ERR("Invalid vector size: %u (max: %u)\n", config->data_size, + pool_capa.vector.max_size); + return PRS_NOK; + } + + if (config->cache_size < pool_capa.vector.min_cache_size || + config->cache_size > pool_capa.vector.max_cache_size) { + ODPH_ERR("Invalid cache size: %" PRIi64 " (min: %u, max: %u)\n", + config->cache_size, pool_capa.vector.min_cache_size, + pool_capa.vector.max_cache_size); + return PRS_NOK; + } + + if (num_pools > pool_capa.vector.max_pools) { + ODPH_ERR("Invalid pool count: %u (max: %u)\n", num_pools, + pool_capa.vector.max_pools); + return PRS_NOK; + } + + config->handle_size = sizeof(odp_packet_vector_t); + config->uarea_size = ODPH_MIN(config->uarea_size, pool_capa.vector.max_uarea_size); + } + + if (config->num_elems == 0U) { + ODPH_ERR("Invalid burst pattern, no elements\n"); + return PRS_NOK; + } + + for (uint8_t i = 0U; i < config->num_elems; ++i) { + elem = &config->alloc_elems[i]; + + if (elem->op == ALLOC) + num_tot += elem->val; + else if (elem->op == FREE) + num_tot -= elem->val; + + if (num_tot < 0) { + ODPH_ERR("Invalid burst pattern, frees exceed allocations " + "instantaneously\n"); + return PRS_NOK; + } + + config->num_data_elems += (elem->op == ALLOC ? elem->val : 0U); + } + + if (num_tot != 0) { + ODPH_ERR("Invalid burst pattern, cumulative sum not zero: %" PRId64 "\n", num_tot); + return PRS_NOK; + } + + if (odp_shm_capability(&shm_capa) < 0) { + ODPH_ERR("Error querying SHM capabilities\n"); + return PRS_NOK; + } + + if (shm_capa.max_blocks < config->num_workers + 1U) { + ODPH_ERR("Invalid amount of SHM blocks: %u (max: %u)\n", config->num_workers + 1U, + shm_capa.max_blocks); + return PRS_NOK; + } + + shm_size = config->num_data_elems * config->handle_size; + + if (shm_capa.max_size != 0U && shm_size > shm_capa.max_size) { + ODPH_ERR("Invalid total SHM block size: %" PRIu64 " (max: %" PRIu64 ")\n", + shm_size, shm_capa.max_size); + return PRS_NOK; + } + + if (config->policy != SINGLE && config->policy != MANY) { + ODPH_ERR("Invalid pool policy: %u\n", config->policy); + return PRS_NOK; + } + + if (config->num_rounds == 0U) { + ODPH_ERR("Invalid round count: %u (min: 1)\n", config->num_rounds); + return PRS_NOK; + } + + if (config->num_ignore >= config->num_rounds) { + ODPH_ERR("Invalid round ignorance count: %u (max: %u)\n", config->num_ignore, + config->num_rounds - 1U); + return PRS_NOK; + } + + return PRS_OK; +} + +static parse_result_t parse_options(int argc, char **argv, prog_config_t *config) +{ + int opt, long_index; + + static const struct option longopts[] = { + { "burst_pattern", required_argument, NULL, 'b' }, + { "type", required_argument, NULL, 't' }, + { "event_count", required_argument, NULL, 'e' }, + { "data_size", required_argument, NULL, 'd' }, + { "policy", required_argument, NULL, 'p' }, + { "round_count", required_argument, NULL, 'r' }, + { "ignore_rounds", required_argument, NULL, 'i' }, + { "worker_count", required_argument, NULL, 'c' }, + { "cache_size", required_argument, NULL, 'C' }, + { "write_uarea", required_argument, NULL, 'w' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + static const char *shortopts = "b:t:e:d:p:r:i:c:C:w:h"; + + init_config(config); + + while (1) { + opt = getopt_long(argc, argv, shortopts, longopts, &long_index); + + if (opt == -1) + break; + + switch (opt) { + case 'b': + parse_burst_pattern(config, optarg); + break; + case 't': + config->type = atoi(optarg); + break; + case 'e': + config->num_evs = atoi(optarg); + break; + case 'd': + config->data_size = atoi(optarg); + break; + case 'p': + config->policy = atoi(optarg); + break; + case 'r': + config->num_rounds = atoi(optarg); + break; + case 'i': + config->num_ignore = atoi(optarg); + break; + case 'c': + config->num_workers = atoi(optarg); + break; + case 'C': + config->cache_size = atoi(optarg); + break; + case 'w': + config->uarea_size = atoi(optarg); + break; + case 'h': + print_usage(&config->dyn_defs); + return PRS_TERM; + case '?': + default: + print_usage(&config->dyn_defs); + return PRS_NOK; + } + } + + return check_options(config); +} + +static inline void save_alloc_stats(odp_time_t t1, odp_time_t t2, uint32_t num_alloc, + uint64_t round, uint8_t pattern, stats_t *stats) +{ + const uint64_t tm_diff = odp_time_diff_ns(t2, t1); + + stats->alloc_tm += tm_diff; + stats->alloc_cnt += num_alloc; + ++stats->alloc_b_cnt; + + if (tm_diff > stats->max_alloc_tm) { + stats->max_alloc_tm = tm_diff; + stats->max_alloc_rnd = round; + stats->max_alloc_pt = pattern; + } + + if (tm_diff < stats->min_alloc_tm) { + stats->min_alloc_tm = tm_diff; + stats->min_alloc_rnd = round; + stats->min_alloc_pt = pattern; + } +} + +static inline void write_to_uarea(uint8_t *data, uint32_t size) +{ + memset(data, UA_DATA, size); +} + +static inline void save_uarea_stats(odp_time_t t1, odp_time_t t2, uint64_t round, uint8_t pattern, + stats_t *stats) +{ + const uint64_t tm_diff = odp_time_diff_ns(t2, t1); + + stats->uarea_tm += tm_diff; + + if (tm_diff > stats->max_uarea_tm) { + stats->max_uarea_tm = tm_diff; + stats->max_uarea_rnd = round; + stats->max_uarea_pt = pattern; + } + + if (tm_diff < stats->min_uarea_tm) { + stats->min_uarea_tm = tm_diff; + stats->min_uarea_rnd = round; + stats->min_uarea_pt = pattern; + } +} + +static inline void save_free_stats(odp_time_t t1, odp_time_t t2, uint64_t round, uint8_t pattern, + stats_t *stats) +{ + const uint64_t tm_diff = odp_time_diff_ns(t2, t1); + + stats->free_tm += tm_diff; + ++stats->free_b_cnt; + + if (tm_diff > stats->max_free_tm) { + stats->max_free_tm = tm_diff; + stats->max_free_rnd = round; + stats->max_free_pt = pattern; + } + + if (tm_diff < stats->min_free_tm) { + stats->min_free_tm = tm_diff; + stats->min_free_rnd = round; + stats->min_free_pt = pattern; + } + + stats->max_free_tm = ODPH_MAX(tm_diff, stats->max_free_tm); + stats->min_free_tm = ODPH_MIN(tm_diff, stats->min_free_tm); +} + +static uint32_t allocate_buffers(worker_config_t *config, void *data, uint32_t idx, uint32_t num, + uint64_t round, uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_pool_t pool = config->pool; + uint32_t retries = MAX_RETRIES; + odp_buffer_t *bufs = &((odp_buffer_t *)data)[idx]; + uint32_t num_alloc, num_tot = 0U; + int ret; + stats_t *stats = &config->stats; + + while (retries-- > 0U && num_tot < num) { + num_alloc = num - num_tot; + t1 = odp_time_local_strict(); + ret = odp_buffer_alloc_multi(pool, &bufs[num_tot], num_alloc); + t2 = odp_time_local_strict(); + + if (odp_unlikely(ret < 0)) { + ++stats->alloc_errs; + break; + } + + if (odp_unlikely((uint32_t)ret < num_alloc)) + ++stats->reallocs; + + num_tot += ret; + + if (odp_likely(is_saved)) + save_alloc_stats(t1, t2, ret, round, pattern, stats); + } + + if (config->uarea_size > 0U) { + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < num_tot; ++i) + write_to_uarea(odp_buffer_user_area(bufs[i]), config->uarea_size); + + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_uarea_stats(t1, t2, round, pattern, stats); + } + + return num_tot; +} + +static void free_buffers(void *data, uint32_t idx, uint32_t num, stats_t *stats, uint64_t round, + uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_buffer_t *bufs = &((odp_buffer_t *)data)[idx]; + + t1 = odp_time_local_strict(); + odp_buffer_free_multi(bufs, num); + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_free_stats(t1, t2, round, pattern, stats); +} + +static uint32_t allocate_packets(worker_config_t *config, void *data, uint32_t idx, uint32_t num, + uint64_t round, uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_pool_t pool = config->pool; + uint32_t retries = MAX_RETRIES, data_size = config->data_size; + odp_packet_t *pkts = &((odp_packet_t *)data)[idx]; + uint32_t num_alloc, num_tot = 0U; + int ret; + stats_t *stats = &config->stats; + + while (retries-- > 0U && num_tot < num) { + num_alloc = num - num_tot; + t1 = odp_time_local_strict(); + ret = odp_packet_alloc_multi(pool, data_size, &pkts[num_tot], num_alloc); + t2 = odp_time_local_strict(); + + if (odp_unlikely(ret < 0)) { + ++stats->alloc_errs; + break; + } + + if (odp_unlikely((uint32_t)ret < num_alloc)) + ++stats->reallocs; + + num_tot += ret; + + if (odp_likely(is_saved)) + save_alloc_stats(t1, t2, ret, round, pattern, stats); + } + + if (config->uarea_size > 0U) { + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < num_tot; ++i) + write_to_uarea(odp_packet_user_area(pkts[i]), config->uarea_size); + + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_uarea_stats(t1, t2, round, pattern, stats); + } + + return num_tot; +} + +static void free_packets(void *data, uint32_t idx, uint32_t num, stats_t *stats, uint64_t round, + uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_packet_t *pkts = &((odp_packet_t *)data)[idx]; + + t1 = odp_time_local_strict(); + odp_packet_free_multi(pkts, num); + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_free_stats(t1, t2, round, pattern, stats); +} + +static uint32_t allocate_timeouts(worker_config_t *config, void *data, uint32_t idx, uint32_t num, + uint64_t round, uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_pool_t pool = config->pool; + uint32_t retries = MAX_RETRIES; + odp_timeout_t *tmos = &((odp_timeout_t *)data)[idx]; + uint32_t num_alloc, num_tot = 0U; + int ret; + stats_t *stats = &config->stats; + + while (retries-- > 0U && num_tot < num) { + num_alloc = num - num_tot; + t1 = odp_time_local_strict(); + ret = odp_timeout_alloc_multi(pool, &tmos[num_tot], num_alloc); + t2 = odp_time_local_strict(); + + if (odp_unlikely(ret < 0)) { + ++stats->alloc_errs; + break; + } + + if (odp_unlikely((uint32_t)ret < num_alloc)) + ++stats->reallocs; + + num_tot += ret; + + if (odp_likely(is_saved)) + save_alloc_stats(t1, t2, ret, round, pattern, stats); + } + + if (config->uarea_size > 0U) { + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < num_tot; ++i) + write_to_uarea(odp_timeout_user_area(tmos[i]), config->uarea_size); + + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_uarea_stats(t1, t2, round, pattern, stats); + } + + return num_tot; +} + +static void free_timeouts(void *data, uint32_t idx, uint32_t num, stats_t *stats, uint64_t round, + uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_timeout_t *tmos = &((odp_timeout_t *)data)[idx]; + + t1 = odp_time_local_strict(); + odp_timeout_free_multi(tmos, num); + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_free_stats(t1, t2, round, pattern, stats); +} + +static uint32_t allocate_vectors(worker_config_t *config, void *data, uint32_t idx, uint32_t num, + uint64_t round, uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_pool_t pool = config->pool; + uint32_t num_tot = 0U; + odp_packet_vector_t *vecs = &((odp_packet_vector_t *)data)[idx], vec; + stats_t *stats = &config->stats; + + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < num; ++i) { + vec = odp_packet_vector_alloc(pool); + + if (odp_unlikely(vec == ODP_PACKET_VECTOR_INVALID)) + break; + + vecs[num_tot++] = vec; + } + + t2 = odp_time_local_strict(); + + if (odp_unlikely(num_tot == 0)) + ++stats->alloc_errs; + else if (odp_likely(is_saved)) + save_alloc_stats(t1, t2, num_tot, round, pattern, stats); + + if (config->uarea_size > 0U) { + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < num_tot; ++i) + write_to_uarea(odp_packet_vector_user_area(vecs[i]), config->uarea_size); + + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_uarea_stats(t1, t2, round, pattern, stats); + } + + return num_tot; +} + +static void free_vectors(void *data, uint32_t idx, uint32_t num, stats_t *stats, uint64_t round, + uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_packet_vector_t *vecs = &((odp_packet_vector_t *)data)[idx]; + + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < num; ++i) + odp_packet_vector_free(vecs[i]); + + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_free_stats(t1, t2, round, pattern, stats); +} + +static odp_pool_t create_pool(const char *name, const odp_pool_param_t *params, uint8_t policy) +{ + static odp_pool_t pool = ODP_POOL_INVALID; + + if (policy == SINGLE && pool != ODP_POOL_INVALID) + return pool; + + pool = odp_pool_create(name, params); + + return pool; +} + +static odp_bool_t setup_worker_config(prog_config_t *config) +{ + odp_pool_param_t param; + odp_pool_t pool; + worker_config_t *worker; + odp_shm_t shm; + void *data; + + odp_pool_param_init(¶m); + + if (config->type == BUFFER) { + param.type = ODP_POOL_BUFFER; + param.buf.num = config->num_evs; + param.buf.size = config->data_size; + param.buf.uarea_size = config->uarea_size; + param.buf.cache_size = config->cache_size; + config->alloc_fn = allocate_buffers; + config->free_fn = free_buffers; + } else if (config->type == PACKET) { + param.type = ODP_POOL_PACKET; + param.pkt.num = config->num_evs; + param.pkt.len = config->data_size; + param.pkt.seg_len = config->seg_len; + param.pkt.uarea_size = config->uarea_size; + param.pkt.cache_size = config->cache_size; + config->alloc_fn = allocate_packets; + config->free_fn = free_packets; + } else if (config->type == TMO) { + param.type = ODP_POOL_TIMEOUT; + param.tmo.num = config->num_evs; + param.tmo.uarea_size = config->uarea_size; + param.tmo.cache_size = config->cache_size; + config->alloc_fn = allocate_timeouts; + config->free_fn = free_timeouts; + } else { + param.type = ODP_POOL_VECTOR; + param.vector.num = config->num_evs; + param.vector.max_size = config->data_size; + param.vector.uarea_size = config->uarea_size; + param.vector.cache_size = config->cache_size; + config->alloc_fn = allocate_vectors; + config->free_fn = free_vectors; + } + + for (uint32_t i = 0U; i < config->num_workers; ++i) { + pool = create_pool(PROG_NAME "_pool", ¶m, config->policy); + + if (pool == ODP_POOL_INVALID) { + ODPH_ERR("Error creating worker pool\n"); + return false; + } + + shm = odp_shm_reserve(PROG_NAME "_shm", + config->handle_size * config->num_data_elems, + ODP_CACHE_LINE_SIZE, 0U); + + if (shm == ODP_SHM_INVALID) { + ODPH_ERR("Error creating worker SHM\n"); + return false; + } + + data = odp_shm_addr(shm); + + if (data == NULL) { + ODPH_ERR("Error resolving worker SHM\n"); + return false; + } + + worker = &config->worker_config[i]; + worker->pool = pool; + worker->data = data; + worker->prog_config = config; + worker->shm = shm; + worker->data_size = config->data_size; + worker->uarea_size = config->uarea_size; + } + + return true; +} + +static int run_test(void *args) +{ + worker_config_t *config = args; + odp_time_t t1, t2; + uint32_t head_idx, cur_idx, num_ignore = config->prog_config->num_ignore, val, num_alloc, + idx; + odp_bool_t is_saved; + const uint8_t num_elems = config->prog_config->num_elems; + const alloc_elem_t *elems = config->prog_config->alloc_elems, *elem; + uint8_t op; + void *data = config->data; + const alloc_fn_t alloc_fn = config->prog_config->alloc_fn; + stats_t *stats = &config->stats; + const free_fn_t free_fn = config->prog_config->free_fn; + + odp_barrier_wait(&config->prog_config->init_barrier); + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < config->prog_config->num_rounds; ++i) { + head_idx = 0U; + cur_idx = head_idx; + is_saved = (num_ignore > 0U ? num_ignore-- : num_ignore) == 0U; + + for (uint8_t j = 0U; j < num_elems; ++j) { + elem = &elems[j]; + val = elem->val; + op = elem->op; + + if (op == ALLOC) { + num_alloc = alloc_fn(config, data, cur_idx, val, i, j, is_saved); + + if (odp_unlikely(num_alloc < val)) + ++stats->pattern_errs; + + cur_idx += num_alloc; + } else if (op == FREE) { + /* Due to potential pattern errors, there might not be expected + * amount of freeable events. */ + val = ODPH_MIN(val, cur_idx - head_idx); + + if (elem->opt == TOP) { + idx = head_idx; + head_idx += val; + } else { + cur_idx -= val; + idx = cur_idx; + } + + free_fn(data, idx, val, stats, i, j, is_saved); + } else { + odp_time_wait_ns(val); + } + } + } + + t2 = odp_time_local_strict(); + stats->tot_tm = odp_time_diff_ns(t2, t1); + odp_barrier_wait(&config->prog_config->term_barrier); + + return 0; +} + +static odp_bool_t setup_workers(prog_config_t *config) +{ + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_params[config->num_workers], *thr_param; + + odp_barrier_init(&config->init_barrier, config->num_workers + 1); + odp_barrier_init(&config->term_barrier, config->num_workers + 1); + odph_thread_common_param_init(&thr_common); + thr_common.instance = config->odp_instance; + thr_common.cpumask = &config->worker_mask; + + for (uint32_t i = 0; i < config->num_workers; ++i) { + thr_param = &thr_params[i]; + odph_thread_param_init(thr_param); + thr_param->start = run_test; + thr_param->thr_type = ODP_THREAD_WORKER; + thr_param->arg = &config->worker_config[i]; + } + + if ((uint32_t)odph_thread_create(config->thread_tbl, &thr_common, thr_params, + config->num_workers) != config->num_workers) { + ODPH_ERR("Error configuring worker threads\n"); + return false; + } + + odp_barrier_wait(&config->init_barrier); + + return true; +} + +static odp_bool_t setup_test(prog_config_t *config) +{ + return setup_worker_config(config) && setup_workers(config); +} + +static void stop_test(prog_config_t *config) +{ + odp_barrier_wait(&config->term_barrier); + (void)odph_thread_join(config->thread_tbl, config->num_workers); +} + +static void print_stats(const prog_config_t *config) +{ + const alloc_elem_t *elem; + const stats_t *stats; + uint64_t ev_rate, ave_b_alloc_tm, b_alloc_min, b_alloc_max, ave_b_free_tm, b_free_min, + b_free_max, ave_alloc_tm, ave_free_tm, ave_ua_b_tm, b_ua_min, b_ua_max, ave_ua_tm, + tot_b_alloc_tm = 0U, tot_b_free_tm = 0U, tot_alloc_tm = 0U, tot_free_tm = 0U, + tot_alloc_min = 0U, tot_alloc_max = 0U, tot_free_min = 0U, tot_free_max = 0U, + tot_b_ua_tm = 0U, tot_ua_tm = 0U, tot_ua_min = 0U, tot_ua_max = 0U; + + printf("\n==================\n\n" + "Pool latency test done\n\n" + " type: %s\n" + " event count: %u\n", config->type == BUFFER ? "buffer" : + config->type == PACKET ? "packet" : config->type == TMO ? "timeout" : "vector", + config->num_evs); + + if (config->type != TMO) + printf(" %s %u\n", config->type != VECTOR ? "data size: " : "vector size:", + config->data_size); + + printf(" pool policy: %s\n" + " round count: %u\n" + " ignore count: %u\n" + " cache size: %" PRIi64 "\n" + " user area: %u (B)\n" + " burst pattern:\n", config->policy == SINGLE ? "shared" : "per-worker", + config->num_rounds, config->num_ignore, config->cache_size, config->uarea_size); + + for (uint8_t i = 0U; i < config->num_elems; ++i) { + elem = &config->alloc_elems[i]; + printf(" %s %u%s\n", elem->op == ALLOC ? "allocate:" : + elem->op == FREE && elem->opt == TOP ? "free (t):" : + elem->op == FREE && elem->opt == BOTTOM ? "free (b):" : + "delay: ", elem->val, elem->op == DELAY ? " (ns)" : ""); + } + + printf("\n"); + + for (uint32_t i = 0U; i < config->num_workers; ++i) { + stats = &config->worker_config[i].stats; + ev_rate = stats->tot_tm > 0U ? + (double)stats->alloc_cnt / stats->tot_tm * ODP_TIME_SEC_IN_NS : 0U; + ave_b_alloc_tm = stats->alloc_b_cnt > 0U ? + stats->alloc_tm / stats->alloc_b_cnt : 0U; + b_alloc_min = ave_b_alloc_tm > 0U ? stats->min_alloc_tm : 0U; + b_alloc_max = ave_b_alloc_tm > 0U ? stats->max_alloc_tm : 0U; + ave_b_free_tm = stats->free_b_cnt > 0U ? + stats->free_tm / stats->free_b_cnt : 0U; + b_free_min = ave_b_free_tm > 0U ? stats->min_free_tm : 0U; + b_free_max = ave_b_free_tm > 0U ? stats->max_free_tm : 0U; + ave_alloc_tm = stats->alloc_cnt > 0U ? stats->alloc_tm / stats->alloc_cnt : 0U; + ave_free_tm = stats->alloc_cnt > 0U ? stats->free_tm / stats->alloc_cnt : 0U; + + printf(" worker %d:\n" + " significant events allocated/freed: %" PRIu64 "\n" + " allocation retries: %" PRIu64 "\n" + " allocation errors: %" PRIu64 "\n" + " pattern errors: %" PRIu64 "\n" + " run time: %" PRIu64 " (ns)\n" + " event rate %" PRIu64 " (evs/s)\n" + " average latency breakdown (ns):\n" + " per allocation burst: %" PRIu64 " (min: %" PRIu64 " (round: %" + PRIu64 ", pattern: %u), max: %" PRIu64 " (round: %" PRIu64 ", pattern: %u))" + "\n" + " per allocation: %" PRIu64 "\n" + " per free burst: %" PRIu64 " (min: %" PRIu64 " (round: %" + PRIu64 ", pattern: %u), max: %" PRIu64 " (round: %" PRIu64 ", pattern: %u))" + "\n" + " per free: %" PRIu64 "\n", i, stats->alloc_cnt, + stats->reallocs, stats->alloc_errs, stats->pattern_errs, stats->tot_tm, + ev_rate, ave_b_alloc_tm, b_alloc_min, stats->min_alloc_rnd, + stats->min_alloc_pt, b_alloc_max, stats->max_alloc_rnd, stats->max_alloc_pt, + ave_alloc_tm, ave_b_free_tm, b_free_min, stats->min_free_rnd, + stats->min_free_pt, b_free_max, stats->max_free_rnd, stats->max_free_pt, + ave_free_tm); + tot_b_alloc_tm += ave_b_alloc_tm; + tot_b_free_tm += ave_b_free_tm; + tot_alloc_tm += ave_alloc_tm; + tot_free_tm += ave_free_tm; + tot_alloc_min += b_alloc_min; + tot_alloc_max += b_alloc_max; + tot_free_min += b_free_min; + tot_free_max += b_free_max; + + if (config->uarea_size > 0U) { + ave_ua_b_tm = stats->alloc_b_cnt > 0U ? + stats->uarea_tm / stats->alloc_b_cnt : 0U; + ave_ua_tm = stats->alloc_cnt > 0U ? + stats->uarea_tm / stats->alloc_cnt : 0U; + b_ua_min = ave_ua_b_tm > 0U ? stats->min_uarea_tm : 0U; + b_ua_max = ave_ua_b_tm > 0U ? stats->max_uarea_tm : 0U; + printf(" per ua write burst: %" PRIu64 " (min: %" PRIu64 " (" + "round: %" PRIu64 ", pattern: %u), max: %" PRIu64 " (round: %" + PRIu64 ", pattern: %u))\n" + " per ua write: %" PRIu64 "\n", ave_ua_b_tm, + b_ua_min, stats->min_uarea_rnd, stats->min_uarea_pt, b_ua_max, + stats->max_uarea_rnd, stats->max_uarea_pt, ave_ua_tm); + tot_b_ua_tm += ave_ua_b_tm; + tot_ua_tm += ave_ua_tm; + tot_ua_min += b_ua_min; + tot_ua_max += b_ua_max; + } + + printf("\n"); + } + + printf(" total (ns):\n" + " per allocation burst: %" PRIu64 " (min: %" PRIu64 ", max: %" PRIu64 ")\n" + " per allocation: %" PRIu64 "\n" + " per free burst: %" PRIu64 " (min: %" PRIu64 ", max: %" PRIu64 ")\n" + " per free: %" PRIu64 "\n", + tot_b_alloc_tm / config->num_workers, tot_alloc_min / config->num_workers, + tot_alloc_max / config->num_workers, tot_alloc_tm / config->num_workers, + tot_b_free_tm / config->num_workers, tot_free_min / config->num_workers, + tot_free_max / config->num_workers, tot_free_tm / config->num_workers); + + if (config->uarea_size > 0U) { + printf(" per ua write burst: %" PRIu64 " (min: %" PRIu64 ", max: %" + PRIu64 ")\n" + " per ua write: %" PRIu64 "\n", + tot_b_ua_tm / config->num_workers, tot_ua_min / config->num_workers, + tot_ua_max / config->num_workers, tot_ua_tm / config->num_workers); + } + + printf("\n==================\n"); +} + +static void destroy_pool(odp_pool_t pool, uint8_t policy) +{ + static odp_bool_t is_destroyed; + + if (policy == SINGLE && is_destroyed) + return; + + (void)odp_pool_destroy(pool); + is_destroyed = true; +} + +static void teardown(const prog_config_t *config) +{ + const worker_config_t *worker; + + for (uint32_t i = 0U; i < config->num_workers; ++i) { + worker = &config->worker_config[i]; + + if (worker->pool != ODP_POOL_INVALID) + destroy_pool(worker->pool, config->policy); + + if (worker->shm != ODP_SHM_INVALID) + (void)odp_shm_free(worker->shm); + } +} + +int main(int argc, char **argv) +{ + odph_helper_options_t odph_opts; + odp_init_t init_param; + odp_instance_t odp_instance; + odp_shm_t shm_cfg = ODP_SHM_INVALID; + int ret = EXIT_SUCCESS; + parse_result_t parse_res; + + argc = odph_parse_options(argc, argv); + + if (odph_options(&odph_opts) == -1) { + ODPH_ERR("Error while reading ODP helper options, exiting\n"); + exit(EXIT_FAILURE); + } + + odp_init_param_init(&init_param); + init_param.mem_model = odph_opts.mem_model; + + if (odp_init_global(&odp_instance, NULL, NULL)) { + ODPH_ERR("ODP global init failed, exiting\n"); + exit(EXIT_FAILURE); + } + + if (odp_init_local(odp_instance, ODP_THREAD_CONTROL)) { + ODPH_ERR("ODP local init failed, exiting\n"); + exit(EXIT_FAILURE); + } + + shm_cfg = odp_shm_reserve(PROG_NAME "_cfg", sizeof(prog_config_t), ODP_CACHE_LINE_SIZE, + 0U); + + if (shm_cfg == ODP_SHM_INVALID) { + ODPH_ERR("Error reserving shared memory\n"); + ret = EXIT_FAILURE; + goto out; + } + + prog_conf = odp_shm_addr(shm_cfg); + + if (prog_conf == NULL) { + ODPH_ERR("Error resolving shared memory address\n"); + ret = EXIT_FAILURE; + goto out; + } + + parse_res = parse_options(argc, argv, prog_conf); + + if (parse_res == PRS_NOK) { + ret = EXIT_FAILURE; + goto out; + } + + if (parse_res == PRS_TERM) { + ret = EXIT_SUCCESS; + goto out; + } + + prog_conf->odp_instance = odp_instance; + + if (!setup_test(prog_conf)) { + ret = EXIT_FAILURE; + goto out_test; + } + + stop_test(prog_conf); + print_stats(prog_conf); + +out_test: + teardown(prog_conf); + +out: + if (shm_cfg != ODP_SHM_INVALID) + (void)odp_shm_free(shm_cfg); + + if (odp_term_local()) { + ODPH_ERR("ODP local terminate failed, exiting\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_global(odp_instance)) { + ODPH_ERR("ODP global terminate failed, exiting\n"); + exit(EXIT_FAILURE); + } + + return ret; +} From b98430032e88cd3f86df3c6d360adefe8f708cd2 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 2 Feb 2024 12:07:09 +0200 Subject: [PATCH 067/147] test: l2fwd: add option to enable packet input timestamping Add new command line option '-T, --input_ts' to enable packet input timestamping. The application code itself doesn't process packet timestamps, but the option can be used to measure the performance impact timestamping has to implementation's maximum packet throughput. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- test/performance/odp_l2fwd.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/test/performance/odp_l2fwd.c b/test/performance/odp_l2fwd.c index 4f4dc4af9..8efdec3de 100644 --- a/test/performance/odp_l2fwd.c +++ b/test/performance/odp_l2fwd.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2019-2023, Nokia + * Copyright (c) 2019-2024, Nokia * Copyright (c) 2020-2021, Marvell * All rights reserved. * @@ -117,11 +117,12 @@ typedef struct { uint32_t num_vec; /* Number of vectors per pool */ uint64_t vec_tmo_ns; /* Vector formation timeout in ns */ uint32_t vec_size; /* Vector size */ - int verbose; /* Verbose output */ - uint32_t packet_len; /* Maximum packet length supported */ - uint32_t seg_len; /* Pool segment length */ + int verbose; /* Verbose output */ + uint32_t packet_len; /* Maximum packet length supported */ + uint32_t seg_len; /* Pool segment length */ int promisc_mode; /* Promiscuous mode enabled */ int flow_aware; /* Flow aware scheduling enabled */ + uint8_t input_ts; /* Packet input timestamping enabled */ int mtu; /* Interface MTU */ int num_prio; odp_schedule_prio_t prio[MAX_PKTIOS]; /* Priority of input queues of an interface */ @@ -969,6 +970,14 @@ static int create_pktio(const char *dev, int idx, int num_rx, int num_tx, odp_po odp_pktio_config_init(&config); + if (gbl_args->appl.input_ts) { + if (!pktio_capa.config.pktin.bit.ts_all) { + ODPH_ERR("Packet input timestamping not supported: %s\n", dev); + return -1; + } + config.pktin.bit.ts_all = 1; + } + config.parser.layer = ODP_PROTO_LAYER_NONE; if (gbl_args->appl.error_check || gbl_args->appl.chksum) config.parser.layer = ODP_PROTO_LAYER_ALL; @@ -1520,6 +1529,7 @@ static void usage(char *progname) " -L, --seg_len Packet pool segment length\n" " (default equal to packet length).\n" " -f, --flow_aware Enable flow aware scheduling.\n" + " -T, --input_ts Enable packet input timestamping.\n" " -v, --verbose Verbose output.\n" " -h, --help Display help and exit.\n\n" "\n", DEFAULT_VEC_SIZE, DEFAULT_VEC_TMO, POOL_PKT_LEN); @@ -1569,12 +1579,13 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) {"packet_len", required_argument, NULL, 'l'}, {"seg_len", required_argument, NULL, 'L'}, {"flow_aware", no_argument, NULL, 'f'}, + {"input_ts", no_argument, NULL, 'T'}, {"verbose", no_argument, NULL, 'v'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:t:a:i:m:o:r:d:s:e:k:g:G:I:b:q:p:y:n:l:L:w:x:z:M:uPfvh"; + static const char *shortopts = "+c:t:a:i:m:o:r:d:s:e:k:g:G:I:b:q:p:y:n:l:L:w:x:z:M:uPfTvh"; appl_args->time = 0; /* loop forever if time to run is 0 */ appl_args->accuracy = 1; /* get and print pps stats second */ @@ -1600,6 +1611,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->vec_size = 0; appl_args->vec_tmo_ns = 0; appl_args->flow_aware = 0; + appl_args->input_ts = 0; appl_args->num_prio = 0; while (1) { @@ -1805,6 +1817,9 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) case 'f': appl_args->flow_aware = 1; break; + case 'T': + appl_args->input_ts = 1; + break; case 'v': appl_args->verbose = 1; break; @@ -1892,6 +1907,7 @@ static void print_info(void) "enabled" : "disabled"); printf("Flow aware: %s\n", appl_args->flow_aware ? "yes" : "no"); + printf("Input TS: %s\n", appl_args->input_ts ? "yes" : "no"); printf("Burst size: %i\n", appl_args->burst_rx); printf("RX queues per IF: %i\n", appl_args->rx_queues); printf("Number of pools: %i\n", appl_args->pool_per_if ? From c5af7f0e28749350a39bb09d20ce451020561a27 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 2 Feb 2024 13:42:27 +0200 Subject: [PATCH 068/147] validation: pktio: improve packet input timestamping test Add tests which compare packet input timestamps to odp_pktio_time(). Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- test/validation/api/pktio/pktio.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c index c9f8f98c3..deef4895a 100644 --- a/test/validation/api/pktio/pktio.c +++ b/test/validation/api/pktio/pktio.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2020-2023, Nokia + * Copyright (c) 2020-2024, Nokia * Copyright (c) 2020, Marvell * All rights reserved. * @@ -11,6 +11,7 @@ #include +#include #include #include "parser.h" #include "lso.h" @@ -3303,7 +3304,7 @@ static void pktio_test_pktin_ts(void) odp_packet_t pkt_tbl[TX_BATCH_LEN]; uint32_t pkt_seq[TX_BATCH_LEN]; uint64_t ns1, ns2; - uint64_t res, res_ns; + uint64_t res, res_ns, input_delay; odp_time_t ts_prev; odp_time_t ts; int num_rx = 0; @@ -3366,6 +3367,18 @@ static void pktio_test_pktin_ts(void) 1, TXRX_MODE_SINGLE, ODP_TIME_SEC_IN_NS, false); if (ret != 1) break; + + /* Compare to packet IO time to input timestamp */ + ts = odp_pktio_time(pktio_rx_info.id, NULL); + CU_ASSERT_FATAL(odp_packet_has_ts(pkt_tbl[i])); + ts_prev = odp_packet_ts(pkt_tbl[i]); + CU_ASSERT(odp_time_cmp(ts, ts_prev) >= 0); + input_delay = odp_time_diff_ns(ts, ts_prev); + if (input_delay > 100 * ODP_TIME_MSEC_IN_NS) { + printf(" Test packet %d input delay: %" PRIu64 "ns\n", i, input_delay); + CU_FAIL("Packet input delay too long"); + } + odp_time_wait_ns(PKTIO_TS_INTERVAL); } num_rx = i; From fb84928e0ea3db06d8085d3d35098af1aa288ff0 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 2 Feb 2024 14:08:39 +0200 Subject: [PATCH 069/147] example: packet_dump: use ODPH_ERR() for error prints Use ODPH_ERR() helper macro for all error prints. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- example/packet/odp_packet_dump.c | 52 +++++++++++++++----------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/example/packet/odp_packet_dump.c b/example/packet/odp_packet_dump.c index dc9f49dd6..a00baf51d 100644 --- a/example/packet/odp_packet_dump.c +++ b/example/packet/odp_packet_dump.c @@ -19,6 +19,7 @@ #include #include +#include #define MAX_PKTIOS 32 #define MAX_PKTIO_NAME 255 @@ -91,8 +92,7 @@ static int parse_int_list(char *str, int integer[], int max_num) str[len] = 0; if (i == max_num) { - printf("Error: maximum number of options is %i\n", - max_num); + ODPH_ERR("Maximum number of options is %i\n", max_num); return -1; } @@ -161,14 +161,13 @@ static int parse_options(int argc, char *argv[], test_global_t *global) str_len -= len + 1; if (i == MAX_PKTIOS) { - printf("Error: Too many interfaces\n"); + ODPH_ERR("Too many interfaces\n"); ret = -1; break; } if (len > MAX_PKTIO_NAME) { - printf("Error: Too long interface name %s\n", - str); + ODPH_ERR("Too long interface name %s\n", str); ret = -1; break; } @@ -202,7 +201,7 @@ static int parse_options(int argc, char *argv[], test_global_t *global) } if (global->opt.num_pktio == 0) { - printf("Error: At least one pktio interface needed.\n"); + ODPH_ERR("At least one pktio interface needed\n"); ret = -1; } @@ -225,7 +224,7 @@ static int open_pktios(test_global_t *global) num_pktio = global->opt.num_pktio; if (odp_pool_capability(&pool_capa)) { - printf("Error: Pool capability failed.\n"); + ODPH_ERR("Pool capability failed\n"); return -1; } @@ -241,7 +240,7 @@ static int open_pktios(test_global_t *global) global->pool = pool; if (pool == ODP_POOL_INVALID) { - printf("Error: Pool create.\n"); + ODPH_ERR("Pool create failed\n"); return -1; } @@ -258,7 +257,7 @@ static int open_pktios(test_global_t *global) pktio = odp_pktio_open(name, pool, &pktio_param); if (pktio == ODP_PKTIO_INVALID) { - printf("Error (%s): Pktio open failed.\n", name); + ODPH_ERR("Pktio open failed for %s\n", name); return -1; } @@ -280,7 +279,7 @@ static int open_pktios(test_global_t *global) pktin_param.num_queues = 1; if (odp_pktin_queue_config(pktio, &pktin_param)) { - printf("Error (%s): Pktin config failed.\n", name); + ODPH_ERR("Pktin config failed for %s\n", name); return -1; } } @@ -294,8 +293,7 @@ static int start_pktios(test_global_t *global) for (i = 0; i < global->opt.num_pktio; i++) { if (odp_pktio_start(global->pktio[i].pktio)) { - printf("Error (%s): Pktio start failed.\n", - global->opt.pktio_name[i]); + ODPH_ERR("Pktio start failed for %s\n", global->opt.pktio_name[i]); return -1; } @@ -318,8 +316,7 @@ static int stop_pktios(test_global_t *global) continue; if (odp_pktio_stop(pktio)) { - printf("Error (%s): Pktio stop failed.\n", - global->opt.pktio_name[i]); + ODPH_ERR("Pktio stop failed for %s\n", global->opt.pktio_name[i]); ret = -1; } } @@ -356,8 +353,7 @@ static int close_pktios(test_global_t *global) continue; if (odp_pktio_close(pktio)) { - printf("Error (%s): Pktio close failed.\n", - global->opt.pktio_name[i]); + ODPH_ERR("Pktio close failed for %s\n", global->opt.pktio_name[i]); ret = -1; } } @@ -368,7 +364,7 @@ static int close_pktios(test_global_t *global) return ret; if (odp_pool_destroy(pool)) { - printf("Error: Pool destroy failed.\n"); + ODPH_ERR("Pool destroy failed\n"); ret = -1; } @@ -499,7 +495,7 @@ static int print_packet(test_global_t *global, odp_packet_t pkt, pktio = odp_packet_input(pkt); if (odp_pktio_info(pktio, &pktio_info)) { - printf("Error: odp_pktio_info() failed\n"); + ODPH_ERR("Pktio info failed\n"); return -1; } @@ -651,7 +647,7 @@ static int receive_packets(test_global_t *global) continue; if (odp_event_type(ev) != ODP_EVENT_PACKET) { - printf("Bad event type: %i\n", odp_event_type(ev)); + ODPH_ERR("Bad event type: %i\n", odp_event_type(ev)); odp_event_free(ev); continue; } @@ -694,13 +690,13 @@ int main(int argc, char *argv[]) /* Init ODP before calling anything else */ if (odp_init_global(&instance, NULL, NULL)) { - printf("Error: Global init failed.\n"); + ODPH_ERR("Global init failed\n"); return -1; } /* Init this thread */ if (odp_init_local(instance, ODP_THREAD_CONTROL)) { - printf("Error: Local init failed.\n"); + ODPH_ERR("Local init failed\n"); return -1; } @@ -711,39 +707,39 @@ int main(int argc, char *argv[]) odp_sys_info_print(); if (open_pktios(global)) { - printf("Error: pktio open failed\n"); + ODPH_ERR("Pktio open failed\n"); return -1; } if (start_pktios(global)) { - printf("Error: pktio start failed\n"); + ODPH_ERR("Pktio start failed\n"); return -1; } if (receive_packets(global)) { - printf("Error: packet receive failed\n"); + ODPH_ERR("Packet receive failed\n"); return -1; } if (stop_pktios(global)) { - printf("Error: pktio stop failed\n"); + ODPH_ERR("Pktio stop failed\n"); return -1; } empty_queues(); if (close_pktios(global)) { - printf("Error: pktio close failed\n"); + ODPH_ERR("Pktio close failed\n"); return -1; } if (odp_term_local()) { - printf("Error: term local failed.\n"); + ODPH_ERR("Term local failed\n"); return -1; } if (odp_term_global(instance)) { - printf("Error: term global failed.\n"); + ODPH_ERR("Term global failed\n"); return -1; } From 2fda28a95fbe669c2cc5b3099ff3889201d1ddb6 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Fri, 2 Feb 2024 14:15:20 +0200 Subject: [PATCH 070/147] example: packet_dump: measure packet input delay Measure and print the delay between packet reception (odp_packet_ts()) and when the application receives the packet (odp_pktio_time()), when packet input timestamping is supported. Signed-off-by: Matias Elo Reviewed-by: Tuomas Taipale --- example/packet/odp_packet_dump.c | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/example/packet/odp_packet_dump.c b/example/packet/odp_packet_dump.c index a00baf51d..2e16861d9 100644 --- a/example/packet/odp_packet_dump.c +++ b/example/packet/odp_packet_dump.c @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2018 Linaro Limited + * Copyright (c) 2024 Nokia */ /** @@ -213,6 +214,7 @@ static int open_pktios(test_global_t *global) odp_pool_param_t pool_param; odp_pktio_param_t pktio_param; odp_pool_t pool; + odp_pktio_capability_t pktio_capa; odp_pool_capability_t pool_capa; odp_pktio_t pktio; odp_pktio_config_t pktio_config; @@ -263,10 +265,15 @@ static int open_pktios(test_global_t *global) global->pktio[i].pktio = pktio; + if (odp_pktio_capability(pktio, &pktio_capa)) { + ODPH_ERR("Pktio capability failed for %s\n", name); + return -1; + } + odp_pktio_print(pktio); odp_pktio_config_init(&pktio_config); - pktio_config.pktin.bit.ts_all = 1; + pktio_config.pktin.bit.ts_all = pktio_capa.config.pktin.bit.ts_all; pktio_config.parser.layer = ODP_PROTO_LAYER_ALL; odp_pktio_config(pktio, &pktio_config); @@ -431,9 +438,9 @@ static void print_data(odp_packet_t pkt, uint32_t offset, uint32_t len) static int print_packet(test_global_t *global, odp_packet_t pkt, uint64_t num_packet) { - odp_pktio_t pktio; + odp_pktio_t pktio = odp_packet_input(pkt); odp_pktio_info_t pktio_info; - odp_time_t time; + odp_time_t pktio_time, time; uint64_t sec, nsec; uint32_t offset; int i, type, match; @@ -450,10 +457,13 @@ static int print_packet(test_global_t *global, odp_packet_t pkt, int icmp = odp_packet_has_icmp(pkt); int ipv4 = odp_packet_has_ipv4(pkt); - if (odp_packet_has_ts(pkt)) + if (odp_packet_has_ts(pkt)) { + pktio_time = odp_pktio_time(pktio, NULL); time = odp_packet_ts(pkt); - else + } else { time = odp_time_local(); + pktio_time = ODP_TIME_NULL; + } /* Filter based on L3 type */ if (num_filter_l3) { @@ -492,7 +502,6 @@ static int print_packet(test_global_t *global, odp_packet_t pkt, nsec = odp_time_to_ns(time); sec = nsec / ODP_TIME_SEC_IN_NS; nsec = nsec - (sec * ODP_TIME_SEC_IN_NS); - pktio = odp_packet_input(pkt); if (odp_pktio_info(pktio, &pktio_info)) { ODPH_ERR("Pktio info failed\n"); @@ -501,6 +510,15 @@ static int print_packet(test_global_t *global, odp_packet_t pkt, printf("PACKET [%" PRIu64 "]\n", num_packet); printf(" time: %" PRIu64 ".%09" PRIu64 " sec\n", sec, nsec); + + if (odp_time_cmp(pktio_time, ODP_TIME_NULL)) { + nsec = odp_time_to_ns(pktio_time); + sec = nsec / ODP_TIME_SEC_IN_NS; + nsec = nsec - (sec * ODP_TIME_SEC_IN_NS); + printf(" pktio time: %" PRIu64 ".%09" PRIu64 " sec\n", sec, nsec); + printf(" input delay: %" PRIu64 " nsec\n", odp_time_diff_ns(pktio_time, time)); + } + printf(" interface name: %s\n", pktio_info.name); printf(" packet length: %u bytes\n", odp_packet_len(pkt)); From 50dfb2a5e61f08ff434467f889ef5abcac6eb8ca Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 5 Feb 2024 08:39:25 +0200 Subject: [PATCH 071/147] validation: crypto: add test for odp_crypto_packet_to_event() Previously, odp_crypto_packet_to_event() function was not tested by the validation suite. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- test/validation/api/crypto/crypto_op_test.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test/validation/api/crypto/crypto_op_test.c b/test/validation/api/crypto/crypto_op_test.c index 1c6944fdf..ae1465581 100644 --- a/test/validation/api/crypto/crypto_op_test.c +++ b/test/validation/api/crypto/crypto_op_test.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021-2023, Nokia + * Copyright (c) 2021-2024, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -50,7 +50,7 @@ int crypto_op(odp_packet_t pkt_in, odp_crypto_op_type_t op_type) { int rc; - odp_event_t event; + odp_event_t event, event2; odp_crypto_packet_result_t result; odp_event_subtype_t subtype; odp_packet_t orig_pkt_out; @@ -95,15 +95,19 @@ int crypto_op(odp_packet_t pkt_in, if (op_type != ODP_CRYPTO_OP_TYPE_BASIC) CU_ASSERT(*pkt_out == orig_pkt_out); - CU_ASSERT(ODP_EVENT_PACKET == - odp_event_type(odp_packet_to_event(*pkt_out))); - CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == - odp_event_subtype(odp_packet_to_event(*pkt_out))); - CU_ASSERT(ODP_EVENT_PACKET == - odp_event_types(odp_packet_to_event(*pkt_out), &subtype)); + + event = odp_packet_to_event(*pkt_out); + CU_ASSERT(ODP_EVENT_PACKET == odp_event_type(event)); + CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == odp_event_subtype(event)); + CU_ASSERT(ODP_EVENT_PACKET == odp_event_types(event, &subtype)); CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == subtype); CU_ASSERT(odp_packet_subtype(*pkt_out) == ODP_EVENT_PACKET_CRYPTO); + event2 = odp_crypto_packet_to_event(*pkt_out); + CU_ASSERT(ODP_EVENT_PACKET == odp_event_type(event2)); + CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == odp_event_subtype(event2)); + CU_ASSERT(odp_event_to_u64(event) == odp_event_to_u64(event2)); + rc = odp_crypto_result(&result, *pkt_out); if (rc < -1) CU_FAIL("Failed odp_crypto_result()"); From 4104a7127e3610bcea95cdd716da522fc853cc38 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Tue, 6 Feb 2024 06:47:08 +0000 Subject: [PATCH 072/147] test: pool_latency: fix potential overflow Fix a potential integer overflow reported by Coverity. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/odp_pool_latency.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/performance/odp_pool_latency.c b/test/performance/odp_pool_latency.c index 800a8dab1..a3eceb4f7 100644 --- a/test/performance/odp_pool_latency.c +++ b/test/performance/odp_pool_latency.c @@ -534,7 +534,7 @@ static parse_result_t check_options(prog_config_t *config) return PRS_NOK; } - shm_size = config->num_data_elems * config->handle_size; + shm_size = (uint64_t)config->num_data_elems * config->handle_size; if (shm_capa.max_size != 0U && shm_size > shm_capa.max_size) { ODPH_ERR("Invalid total SHM block size: %" PRIu64 " (max: %" PRIu64 ")\n", From abaed8f326c070b827ea3a7f57fe5343edb28c0c Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 16:28:53 +0200 Subject: [PATCH 073/147] linux-gen: macros: avoid nesting in MIN3 and MAX3 macros Add _ODP_MIN3() and reimplement _ODP_MAX3() without nested usage of _ODP_MAX(). Use different temporary variable names in macros. These avoid -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- .../include/odp_macros_internal.h | 32 ++++++++++++++----- platform/linux-generic/odp_ipsec.c | 4 +-- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/platform/linux-generic/include/odp_macros_internal.h b/platform/linux-generic/include/odp_macros_internal.h index abf017aec..047e550f9 100644 --- a/platform/linux-generic/include/odp_macros_internal.h +++ b/platform/linux-generic/include/odp_macros_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2022, Nokia + * Copyright (c) 2022-2024, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -26,19 +26,35 @@ extern "C" { #define _ODP_MIN(a, b) \ __extension__ ({ \ - __typeof__(a) tmp_a = (a); \ - __typeof__(b) tmp_b = (b); \ - tmp_a < tmp_b ? tmp_a : tmp_b; \ + __typeof__(a) min_a = (a); \ + __typeof__(b) min_b = (b); \ + min_a < min_b ? min_a : min_b; \ }) #define _ODP_MAX(a, b) \ __extension__ ({ \ - __typeof__(a) tmp_a = (a); \ - __typeof__(b) tmp_b = (b); \ - tmp_a > tmp_b ? tmp_a : tmp_b; \ + __typeof__(a) max_a = (a); \ + __typeof__(b) max_b = (b); \ + max_a > max_b ? max_a : max_b; \ }) -#define _ODP_MAX3(a, b, c) (_ODP_MAX(_ODP_MAX((a), (b)), (c))) +#define _ODP_MIN3(a, b, c) \ +__extension__ ({ \ + __typeof__(a) min3_a = (a); \ + __typeof__(b) min3_b = (b); \ + __typeof__(c) min3_c = (c); \ + (min3_a < min3_b ? (min3_a < min3_c ? min3_a : min3_c) : \ + (min3_b < min3_c ? min3_b : min3_c)); \ +}) + +#define _ODP_MAX3(a, b, c) \ +__extension__ ({ \ + __typeof__(a) max3_a = (a); \ + __typeof__(b) max3_b = (b); \ + __typeof__(c) max3_c = (c); \ + (max3_a > max3_b ? (max3_a > max3_c ? max3_a : max3_c) : \ + (max3_b > max3_c ? max3_b : max3_c)); \ +}) /* Macros to calculate ODP_ROUNDUP_POWER2_U32() in five rounds of shift * and OR operations. */ diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 8c97a0f55..ee402b935 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -2180,7 +2180,7 @@ static void ipsec_in_finalize(odp_packet_t pkt_in[], ipsec_op_t ops[], int num, int odp_ipsec_in(const odp_packet_t pkt_in[], int num_in, odp_packet_t pkt_out[], int *num_out, const odp_ipsec_in_param_t *param) { - int max_out = _ODP_MIN(_ODP_MIN(num_in, *num_out), MAX_BURST), num_crypto; + int max_out = _ODP_MIN3(num_in, *num_out, MAX_BURST), num_crypto; odp_packet_t crypto_pkts[MAX_BURST]; odp_crypto_packet_op_param_t crypto_param[MAX_BURST]; ipsec_op_t ops[MAX_BURST], *crypto_ops[MAX_BURST]; @@ -2288,7 +2288,7 @@ static void ipsec_out_finalize(odp_packet_t pkt_in[], ipsec_op_t ops[], int num, int odp_ipsec_out(const odp_packet_t pkt_in[], int num_in, odp_packet_t pkt_out[], int *num_out, const odp_ipsec_out_param_t *param) { - int max_out = _ODP_MIN(_ODP_MIN(num_in, *num_out), MAX_BURST), num_crypto; + int max_out = _ODP_MIN3(num_in, *num_out, MAX_BURST), num_crypto; odp_packet_t crypto_pkts[MAX_BURST]; odp_crypto_packet_op_param_t crypto_param[MAX_BURST]; ipsec_op_t ops[MAX_BURST], *crypto_ops[MAX_BURST]; From ba01ceedd26150713164b8d05b712ae48dc8c484 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 18:08:17 +0200 Subject: [PATCH 074/147] linux-gen: atomic: rename variables in macros Use different temporary variable names in nested macros. Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- .../arch/default/odp/api/abi/atomic_generic.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h index af435e495..c6ed86363 100644 --- a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h +++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h @@ -197,11 +197,11 @@ static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128 #define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \ __extension__ ({ \ int *_ret_ptr = ret_ptr; \ - odp_u128_t *_old_val = old_val; \ - odp_u128_t _new_val = new_val; \ - if (((_atom)->v.u64[0] == (_old_val)->u64[0]) && \ - ((_atom)->v.u64[1] == (_old_val)->u64[1])) { \ - (_atom)->v = (_new_val); \ + odp_u128_t *_cas_old = old_val; \ + odp_u128_t _cas_new = new_val; \ + if (((_atom)->v.u64[0] == (_cas_old)->u64[0]) && \ + ((_atom)->v.u64[1] == (_cas_old)->u64[1])) { \ + (_atom)->v = (_cas_new); \ *(_ret_ptr) = 1; \ } else { \ *(_ret_ptr) = 0; \ From 90c472f5d8e91429a386cf3604503beac0e1e32f Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 16:38:16 +0200 Subject: [PATCH 075/147] linux-gen: cls: avoid variable name shadowing Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- platform/linux-generic/odp_classification.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 0e6eea3ae..016a8f0c5 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -299,10 +299,11 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_ param.hash_proto); tbl_index = i * CLS_COS_QUEUE_MAX; for (j = 0; j < param.num_queue; j++) { - char name[ODP_QUEUE_NAME_LEN]; + char hq_name[ODP_QUEUE_NAME_LEN]; - snprintf(name, sizeof(name), "_odp_cos_hq_%u_%u", i, j); - queue = odp_queue_create(name, &cos->queue_param); + snprintf(hq_name, sizeof(hq_name), "_odp_cos_hq_%u_%u", + i, j); + queue = odp_queue_create(hq_name, &cos->queue_param); if (queue == ODP_QUEUE_INVALID) { /* unwind the queues */ _cls_queue_unwind(tbl_index, j); From 44f5fdf3d9d2fbf3f6c4c582380dc6057a64ec0c Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 16:48:41 +0200 Subject: [PATCH 076/147] linux-gen: scalable: avoid variable name shadowing Remove unnecessary local variable. Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- platform/linux-generic/odp_schedule_scalable_ordered.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/platform/linux-generic/odp_schedule_scalable_ordered.c b/platform/linux-generic/odp_schedule_scalable_ordered.c index f6655d7fa..f8568ce53 100644 --- a/platform/linux-generic/odp_schedule_scalable_ordered.c +++ b/platform/linux-generic/odp_schedule_scalable_ordered.c @@ -123,8 +123,6 @@ static void rwin_insert(reorder_window_t *rwin, /*readonly=*/false); rctx = NULL; do { - hc_t new; - new.head = old.head; new.chgi = old.chgi + 1; /* Changed value */ /* Update head & chgi, fail if any has changed */ From 1f4f82cc6b87936fa9ec9682785648c2e9bea9d4 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 17:00:27 +0200 Subject: [PATCH 077/147] linux-gen: pool: avoid variable name shadowing Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- platform/linux-generic/odp_pool.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 4bc10bb38..1e9767821 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -543,7 +543,6 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e static void init_buffers(pool_t *pool) { - uint64_t i; _odp_event_hdr_t *event_hdr; odp_buffer_hdr_t *buf_hdr; odp_packet_hdr_t *pkt_hdr; @@ -567,7 +566,7 @@ static void init_buffers(pool_t *pool) mask = pool->ring_mask; type = pool->type; - for (i = 0; i < pool->num + skipped_blocks ; i++) { + for (uint64_t i = 0; i < pool->num + skipped_blocks ; i++) { int skip = 0; addr = &pool->base_addr[i * pool->block_size]; @@ -1372,11 +1371,11 @@ static inline void event_free_to_pool(pool_t *pool, if (odp_unlikely((uint32_t)num > cache_num)) burst = cache_num; - _odp_event_hdr_t *event_hdr[burst]; + _odp_event_hdr_t *ev_hdr[burst]; - cache_pop(cache, event_hdr, burst); + cache_pop(cache, ev_hdr, burst); - ring_ptr_enq_multi(ring, mask, (void **)event_hdr, burst); + ring_ptr_enq_multi(ring, mask, (void **)ev_hdr, burst); if (CONFIG_POOL_STATISTICS && pool->params.stats.bit.free_ops) odp_atomic_inc_u64(&pool->stats.free_ops); } From 913bdb73cf38a9c954b61335d56b57b1a12b09fa Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 17:04:33 +0200 Subject: [PATCH 078/147] validation: random: avoid variable name shadowing Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/validation/api/random/random.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/validation/api/random/random.c b/test/validation/api/random/random.c index e8b4d962a..551fe775d 100644 --- a/test/validation/api/random/random.c +++ b/test/validation/api/random/random.c @@ -313,11 +313,10 @@ static void random_test_independence(odp_random_kind_t kind) for (int i = 0; i < 256; i++) { for (int j = 0; j < 256; j++) { - double expected = - (double)row[i] * (double)col[j] / (double)num; - double diff = - (double)freq[i][j] - expected; - chisq += diff * diff / expected; + double expect = (double)row[i] * (double)col[j] / (double)num; + double diff = (double)freq[i][j] - expect; + + chisq += diff * diff / expect; } } From f76c31f6f75bc6b55d44e225c05c8bfeab9fe7bf Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 17:10:10 +0200 Subject: [PATCH 079/147] validation: scheduler: avoid variable name shadowing Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/validation/api/scheduler/scheduler.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index 87069f7fe..8dddd8d8f 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -1469,8 +1469,7 @@ static int schedule_common_(void *arg) odp_pool_t pool; int locked; int num; - odp_event_t ev; - odp_buffer_t buf, buf_cpy; + odp_buffer_t buf; odp_queue_t from; globals = args->globals; @@ -1553,7 +1552,8 @@ static int schedule_common_(void *arg) odp_event_free(events[j]); } } else { - ev = odp_schedule(&from, ODP_SCHED_NO_WAIT); + odp_event_t ev = odp_schedule(&from, ODP_SCHED_NO_WAIT); + if (ev == ODP_EVENT_INVALID) continue; @@ -1564,6 +1564,7 @@ static int schedule_common_(void *arg) uint32_t ndx; uint32_t ndx_max; int rc; + odp_buffer_t buf_cpy; ndx_max = odp_queue_lock_count(from); CU_ASSERT_FATAL(ndx_max > 0); From 023ffe968c7ea36eb4c39488a40d359cd91dc1b4 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 17:15:11 +0200 Subject: [PATCH 080/147] validation: timer: avoid variable name shadowing Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/validation/api/timer/timer.c | 1 - 1 file changed, 1 deletion(-) diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index ea784fdb1..3678d0cb2 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -2403,7 +2403,6 @@ static int worker_entrypoint(void *arg ODP_UNUSED) } } else { uint64_t cur_tick; - uint64_t tck; int reset_timer = 0; if (tt[i].ev != ODP_EVENT_INVALID) { From eb1abb332f436d935e69f5702b4d4f512dd93727 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 17:19:01 +0200 Subject: [PATCH 081/147] helper: macros: rename variables Use different temporary variable names in macros. Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- helper/include/odp/helper/macros.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/helper/include/odp/helper/macros.h b/helper/include/odp/helper/macros.h index e30db7bab..1623f17e7 100644 --- a/helper/include/odp/helper/macros.h +++ b/helper/include/odp/helper/macros.h @@ -32,9 +32,9 @@ extern "C" { */ #define ODPH_MIN(a, b) \ __extension__ ({ \ - __typeof__(a) tmp_a = (a); \ - __typeof__(b) tmp_b = (b); \ - tmp_a < tmp_b ? tmp_a : tmp_b; \ + __typeof__(a) min_a = (a); \ + __typeof__(b) min_b = (b); \ + min_a < min_b ? min_a : min_b; \ }) /** @@ -42,9 +42,9 @@ extern "C" { */ #define ODPH_MAX(a, b) \ __extension__ ({ \ - __typeof__(a) tmp_a = (a); \ - __typeof__(b) tmp_b = (b); \ - tmp_a > tmp_b ? tmp_a : tmp_b; \ + __typeof__(a) max_a = (a); \ + __typeof__(b) max_b = (b); \ + max_a > max_b ? max_a : max_b; \ }) /** @@ -52,8 +52,8 @@ extern "C" { */ #define ODPH_ABS(v) \ __extension__ ({ \ - __typeof__(v) tmp_v = (v); \ - tmp_v < 0 ? -tmp_v : tmp_v; \ + __typeof__(v) abs_v = (v); \ + abs_v < 0 ? -abs_v : abs_v; \ }) /** From 17dfb7e0372f3f0ed2146068a0aae615d434da66 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 17:26:24 +0200 Subject: [PATCH 082/147] example: ipsec: avoid variable name shadowing Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- example/ipsec_crypto/odp_ipsec.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/example/ipsec_crypto/odp_ipsec.c b/example/ipsec_crypto/odp_ipsec.c index 31c866b56..92e32d301 100644 --- a/example/ipsec_crypto/odp_ipsec.c +++ b/example/ipsec_crypto/odp_ipsec.c @@ -975,10 +975,10 @@ pkt_disposition_e do_ipsec_out_seq(odp_packet_t *pkt, esp->seq_no = odp_cpu_to_be_32((*ctx->ipsec.esp_seq)++); } if (ctx->ipsec.tun_hdr_offset) { - odph_ipv4hdr_t *ip; + odph_ipv4hdr_t *ip_tun; - ip = (odph_ipv4hdr_t *)(ctx->ipsec.tun_hdr_offset + buf); - ip->id = odp_cpu_to_be_16((*ctx->ipsec.tun_hdr_id)++); + ip_tun = (odph_ipv4hdr_t *)(ctx->ipsec.tun_hdr_offset + buf); + ip_tun->id = odp_cpu_to_be_16((*ctx->ipsec.tun_hdr_id)++); } /* Issue crypto request */ From ef1f8d5ba4bb0838b0368e14b157aaf0edcbdefb Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 17:31:17 +0200 Subject: [PATCH 083/147] example: timer: avoid variable name shadowing Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- example/timer/odp_timer_accuracy.c | 15 +++++++-------- example/timer/odp_timer_test.c | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/example/timer/odp_timer_accuracy.c b/example/timer/odp_timer_accuracy.c index 347ad7697..e5df1c24e 100644 --- a/example/timer/odp_timer_accuracy.c +++ b/example/timer/odp_timer_accuracy.c @@ -767,7 +767,7 @@ static int start_timers(test_global_t *test_global) odp_timer_start_t start_param; if (mode == MODE_PERIODIC) { - odp_timer_periodic_start_t start_param; + odp_timer_periodic_start_t periodic_start; nsec = offset_ns + (j * burst_gap); @@ -781,13 +781,13 @@ static int start_timers(test_global_t *test_global) ctx->first_period = start_tick + odp_timer_ns_to_tick(timer_pool, test_global->period_dbl + 0.5); - start_param.freq_multiplier = test_global->opt.multiplier; - start_param.first_tick = 0; + periodic_start.freq_multiplier = test_global->opt.multiplier; + periodic_start.first_tick = 0; if (nsec) - start_param.first_tick = + periodic_start.first_tick = start_tick + odp_timer_ns_to_tick(timer_pool, nsec); - start_param.tmo_ev = ctx->event; - retval = odp_timer_periodic_start(ctx->timer, &start_param); + periodic_start.tmo_ev = ctx->event; + retval = odp_timer_periodic_start(ctx->timer, &periodic_start); } else { nsec = offset_ns + (i * period_ns) + (j * burst_gap); ctx->nsec = start_ns + nsec; @@ -875,7 +875,6 @@ static void print_nsec_error(const char *str, int64_t nsec, double res_ns, static void print_stat(test_global_t *test_global) { - uint64_t i; test_stat_t test_stat; test_stat_t *stat = &test_stat; uint64_t tot_timers; @@ -943,7 +942,7 @@ static void print_stat(test_global_t *test_global) fprintf(file, " Timer thread tmo(ns) diff(ns)\n"); - for (i = 0; i < tot_timers; i++) { + for (uint64_t i = 0; i < tot_timers; i++) { fprintf(file, "%8" PRIu64 " %7u %12" PRIu64 " %10" PRIi64 "\n", i, log[i].tid, log[i].tmo_ns, log[i].diff_ns); } diff --git a/example/timer/odp_timer_test.c b/example/timer/odp_timer_test.c index e96b4a87b..674d4f4ce 100644 --- a/example/timer/odp_timer_test.c +++ b/example/timer/odp_timer_test.c @@ -167,7 +167,7 @@ static void test_abs_timeouts(int thr, test_globals_t *gbls) ODPH_ABORT("Unexpected event type (%u) received\n", odp_event_type(ev)); } - odp_timeout_t tmo = odp_timeout_from_event(ev); + tmo = odp_timeout_from_event(ev); tick = odp_timeout_tick(tmo); ttp = odp_timeout_user_ptr(tmo); ttp->ev = ev; From a6db5e4dca8eec82ebdb394d579a3dd494f2ccce Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 17:40:43 +0200 Subject: [PATCH 084/147] example: tm: avoid variable name shadowing Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- example/traffic_mgmt/odp_traffic_mgmt.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/example/traffic_mgmt/odp_traffic_mgmt.c b/example/traffic_mgmt/odp_traffic_mgmt.c index 38c39bd5c..4ed4f2044 100644 --- a/example/traffic_mgmt/odp_traffic_mgmt.c +++ b/example/traffic_mgmt/odp_traffic_mgmt.c @@ -637,20 +637,19 @@ static int create_and_config_tm(void) tm_shaper_max_burst = tm_capa.per_level[0].max_burst; for (level = 1; level < tm_capa.max_levels; level++) { - odp_tm_level_capabilities_t *per_level = - &tm_capa.per_level[level]; + odp_tm_level_capabilities_t *level_capa = &tm_capa.per_level[level]; - if (per_level->min_rate > tm_shaper_min_rate) - tm_shaper_min_rate = per_level->min_rate; + if (level_capa->min_rate > tm_shaper_min_rate) + tm_shaper_min_rate = level_capa->min_rate; - if (per_level->min_burst > tm_shaper_min_burst) - tm_shaper_min_burst = per_level->min_burst; + if (level_capa->min_burst > tm_shaper_min_burst) + tm_shaper_min_burst = level_capa->min_burst; - if (per_level->max_rate < tm_shaper_max_rate) - tm_shaper_max_rate = per_level->max_rate; + if (level_capa->max_rate < tm_shaper_max_rate) + tm_shaper_max_rate = level_capa->max_rate; - if (per_level->max_burst < tm_shaper_max_burst) - tm_shaper_max_burst = per_level->max_burst; + if (level_capa->max_burst < tm_shaper_max_burst) + tm_shaper_max_burst = level_capa->max_burst; } if (tm_shaper_min_rate > tm_shaper_max_rate || From 6e6e1bcf519f5f268e95434d897fa1907c7cf0ab Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 17:43:41 +0200 Subject: [PATCH 085/147] test: crypto: avoid variable name shadowing Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/performance/odp_crypto.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c index 1ae09abab..a644da5e1 100644 --- a/test/performance/odp_crypto.c +++ b/test/performance/odp_crypto.c @@ -1241,7 +1241,7 @@ int main(int argc, char *argv[]) odp_pool_capability_t pool_capa; odp_crypto_capability_t crypto_capa; uint32_t max_seg_len; - unsigned i; + uint32_t i; /* Let helper collect its own arguments (e.g. --odph_proc) */ argc = odph_parse_options(argc, argv); @@ -1374,8 +1374,6 @@ int main(int argc, char *argv[]) run_measure_one_config(&test_run_arg); } } else { - unsigned int i; - for (i = 0; i < ODPH_ARRAY_SIZE(algs_config); i++) { test_run_arg.crypto_alg_config = algs_config + i; run_measure_one_config(&test_run_arg); From 1cf5ca392289c6a90d8bbf5134e66077e8ed36e5 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 17:48:06 +0200 Subject: [PATCH 086/147] test: dma_perf: avoid variable name shadowing Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/performance/odp_dma_perf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/performance/odp_dma_perf.c b/test/performance/odp_dma_perf.c index b67c1fa23..2f4ca490d 100644 --- a/test/performance/odp_dma_perf.c +++ b/test/performance/odp_dma_perf.c @@ -251,14 +251,14 @@ static void init_config(prog_config_t *config) stats = &config->thread_config[i].stats; memset(sd, 0, sizeof(*sd)); - for (uint32_t i = 0U; i < MAX_SEGS; ++i) { - info = &sd->dma.infos[i]; + for (uint32_t j = 0U; j < MAX_SEGS; ++j) { + info = &sd->dma.infos[j]; info->compl_param.transfer_id = ODP_DMA_TRANSFER_ID_INVALID; info->compl_param.event = ODP_EVENT_INVALID; info->compl_param.queue = ODP_QUEUE_INVALID; odp_ticketlock_init(&info->lock); - sd->seg.src_pkt[i] = ODP_PACKET_INVALID; - sd->seg.dst_pkt[i] = ODP_PACKET_INVALID; + sd->seg.src_pkt[j] = ODP_PACKET_INVALID; + sd->seg.dst_pkt[j] = ODP_PACKET_INVALID; } sd->dma.handle = ODP_DMA_INVALID; From 2709e8d94f47e4ef8f8d313f52af565b26b2873e Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 17:51:06 +0200 Subject: [PATCH 087/147] test: ipsec: avoid variable name shadowing Fixes -Wshadow compiler warnings. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/performance/odp_ipsec.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/performance/odp_ipsec.c b/test/performance/odp_ipsec.c index 77437a918..3ea93ec96 100644 --- a/test/performance/odp_ipsec.c +++ b/test/performance/odp_ipsec.c @@ -1383,8 +1383,6 @@ int main(int argc, char *argv[]) run_measure_one_config(&cargs, cargs.alg_config); } } else { - unsigned int i; - for (i = 0; i < ODPH_ARRAY_SIZE(algs_config); i++) { if (cargs.ah && algs_config[i].crypto.cipher_alg != From c5ff2620b9acd219096c203ab4d2a3036b522103 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Thu, 1 Feb 2024 15:29:55 +0200 Subject: [PATCH 088/147] configure: enable -Wshadow warnings Enables compiler warnings (-Wshadow=local) when a local variable shadows another local variable or parameter. For example: for (int i = 0; i < N; ++i) { ... for (int i = 0; i < M; ++i) { ... } } Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 3184b85d2..da1dbc491 100644 --- a/configure.ac +++ b/configure.ac @@ -113,6 +113,7 @@ ODP_CHECK_CFLAG([-Wundef]) ODP_CHECK_CFLAG([-Wwrite-strings]) ODP_CHECK_CFLAG([-Wformat-truncation=0]) ODP_CHECK_CFLAG([-Wformat-overflow=0]) +ODP_CHECK_CFLAG([-Wshadow=local]) # GCC 10 sometimes gets confused about object sizes and gives bogus warnings. # Make the affected warnings generate only warnings, not errors. From e15a6ddae6fe889829c36f8bb0d17fd2f2b4aaf0 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Thu, 8 Feb 2024 07:09:50 +0000 Subject: [PATCH 089/147] test: pool_latency: fix initialization parameter usage Actually pass instance initialization parameters to `odp_init_global()`. Previously, they were defined but erroneously not utilized. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/odp_pool_latency.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/performance/odp_pool_latency.c b/test/performance/odp_pool_latency.c index a3eceb4f7..6b964e773 100644 --- a/test/performance/odp_pool_latency.c +++ b/test/performance/odp_pool_latency.c @@ -1312,7 +1312,7 @@ int main(int argc, char **argv) odp_init_param_init(&init_param); init_param.mem_model = odph_opts.mem_model; - if (odp_init_global(&odp_instance, NULL, NULL)) { + if (odp_init_global(&odp_instance, &init_param, NULL)) { ODPH_ERR("ODP global init failed, exiting\n"); exit(EXIT_FAILURE); } From ae5c10bf3f9470aea236e31b9387d6ffc7525f47 Mon Sep 17 00:00:00 2001 From: Tuomas Taipale Date: Thu, 8 Feb 2024 07:15:22 +0000 Subject: [PATCH 090/147] test: dmafwd: fix initialization parameter usage Actually pass instance initialization parameters to `odp_init_global()`. Previously, they were defined but erroneously not utilized. Signed-off-by: Tuomas Taipale Reviewed-by: Matias Elo --- test/performance/odp_dmafwd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/performance/odp_dmafwd.c b/test/performance/odp_dmafwd.c index 3f89bf707..694973ce0 100644 --- a/test/performance/odp_dmafwd.c +++ b/test/performance/odp_dmafwd.c @@ -1381,7 +1381,7 @@ int main(int argc, char **argv) odp_init_param_init(&init_param); init_param.mem_model = odph_opts.mem_model; - if (odp_init_global(&odp_instance, NULL, NULL)) { + if (odp_init_global(&odp_instance, &init_param, NULL)) { ODPH_ERR("ODP global init failed, exiting\n"); exit(EXIT_FAILURE); } From 3302f49c252d824ca2882823950413212e190006 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 5 Feb 2024 15:09:33 +0200 Subject: [PATCH 091/147] test: bench_packet: add tests for user flag functions Add missing tests for odp_packet_user_flag() and odp_packet_user_flag_set() APIs. Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- test/performance/odp_bench_packet.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c index 9d6072275..cb9e3ca03 100644 --- a/test/performance/odp_bench_packet.c +++ b/test/performance/odp_bench_packet.c @@ -1096,6 +1096,27 @@ static int packet_user_area_size(void) return ret; } +static int packet_user_flag(void) +{ + int i; + uint32_t ret = 0; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + ret += !odp_packet_user_flag(gbl_args->pkt_tbl[i]); + + return ret; +} + +static int packet_user_flag_set(void) +{ + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_packet_user_flag_set(gbl_args->pkt_tbl[i], 1); + + return i; +} + static int packet_l2_ptr(void) { int i; @@ -1514,6 +1535,8 @@ bench_info_t test_suite[] = { BENCH_INFO(packet_user_ptr_set, create_packets, free_packets, NULL), BENCH_INFO(packet_user_area, create_packets, free_packets, NULL), BENCH_INFO(packet_user_area_size, create_packets, free_packets, NULL), + BENCH_INFO(packet_user_flag, create_packets, free_packets, NULL), + BENCH_INFO(packet_user_flag_set, create_packets, free_packets, NULL), BENCH_INFO(packet_l2_ptr, create_packets, free_packets, NULL), BENCH_INFO(packet_l2_offset, create_packets, free_packets, NULL), BENCH_INFO(packet_l2_offset_set, create_packets, free_packets, NULL), From 04331e3cfbdaa64864871d89b958704f111b7b11 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Mon, 29 Jan 2024 17:33:10 +0200 Subject: [PATCH 092/147] test: l2fwd: pack fast path flags Move application fast path flags close together and use uint8_t as their data type. Pass flags to the extra processing function through a const pointer. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/performance/odp_l2fwd.c | 51 +++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/test/performance/odp_l2fwd.c b/test/performance/odp_l2fwd.c index 8efdec3de..dc46510c2 100644 --- a/test/performance/odp_l2fwd.c +++ b/test/performance/odp_l2fwd.c @@ -89,7 +89,23 @@ static inline int sched_mode(pktin_mode_t in_mode) */ typedef struct { /* Some extra features (e.g. error checks) have been enabled */ - int extra_feat; + uint8_t extra_feat; + + /* Change destination eth addresses */ + uint8_t dst_change; + + /* Change source eth addresses */ + uint8_t src_change; + + /* Check packet errors */ + uint8_t error_check; + + /* Packet copy */ + uint8_t packet_copy; + + /* Checksum offload */ + uint8_t chksum; + unsigned int cpu_count; int if_count; /* Number of interfaces to be used */ int addr_count; /* Number of dst addresses to be used */ @@ -101,11 +117,6 @@ typedef struct { int time; /* Time in seconds to run. */ int accuracy; /* Number of seconds to get and print stats */ char *if_str; /* Storage for interface names */ - int dst_change; /* Change destination eth addresses */ - int src_change; /* Change source eth addresses */ - int error_check; /* Check packet errors */ - int packet_copy; /* Packet copy */ - int chksum; /* Checksum offload */ int sched_mode; /* Scheduler mode */ int num_groups; /* Number of scheduling groups */ int group_mode; /* How threads join groups */ @@ -352,21 +363,21 @@ static inline int copy_packets(odp_packet_t *pkt_tbl, int pkts) /* * Return number of packets remaining in the pkt_tbl */ -static inline int process_extra_features(odp_packet_t *pkt_tbl, int pkts, - stats_t *stats) +static inline int process_extra_features(const appl_args_t *appl_args, odp_packet_t *pkt_tbl, + int pkts, stats_t *stats) { - if (odp_unlikely(gbl_args->appl.extra_feat)) { - if (gbl_args->appl.packet_copy) { + if (odp_unlikely(appl_args->extra_feat)) { + if (appl_args->packet_copy) { int fails; fails = copy_packets(pkt_tbl, pkts); stats->s.copy_fails += fails; } - if (gbl_args->appl.chksum) + if (appl_args->chksum) chksum_insert(pkt_tbl, pkts); - if (gbl_args->appl.error_check) { + if (appl_args->error_check) { int rx_drops; /* Drop packets with errors */ @@ -430,6 +441,7 @@ static int run_worker_sched_mode_vector(void *arg) odp_queue_t tx_queue[MAX_PKTIOS]; thread_args_t *thr_args = arg; stats_t *stats = &thr_args->stats; + const appl_args_t *appl_args = &gbl_args->appl; int use_event_queue = gbl_args->appl.out_mode; pktin_mode_t in_mode = gbl_args->appl.in_mode; @@ -493,7 +505,8 @@ static int run_worker_sched_mode_vector(void *arg) pkts = odp_packet_vector_tbl(pkt_vec, &pkt_tbl); } - pkts = process_extra_features(pkt_tbl, pkts, stats); + pkts = process_extra_features(appl_args, pkt_tbl, pkts, stats); + if (odp_unlikely(pkts) == 0) { if (pkt_vec != ODP_PACKET_VECTOR_INVALID) odp_packet_vector_free(pkt_vec); @@ -576,6 +589,7 @@ static int run_worker_sched_mode(void *arg) char extra_str[EXTRA_STR_LEN]; thread_args_t *thr_args = arg; stats_t *stats = &thr_args->stats; + const appl_args_t *appl_args = &gbl_args->appl; int use_event_queue = gbl_args->appl.out_mode; pktin_mode_t in_mode = gbl_args->appl.in_mode; @@ -639,7 +653,8 @@ static int run_worker_sched_mode(void *arg) odp_packet_from_event_multi(pkt_tbl, ev_tbl, pkts); - pkts = process_extra_features(pkt_tbl, pkts, stats); + pkts = process_extra_features(appl_args, pkt_tbl, pkts, stats); + if (odp_unlikely(pkts) == 0) continue; @@ -713,6 +728,7 @@ static int run_worker_plain_queue_mode(void *arg) int pktio = 0; thread_args_t *thr_args = arg; stats_t *stats = &thr_args->stats; + const appl_args_t *appl_args = &gbl_args->appl; int use_event_queue = gbl_args->appl.out_mode; int i; @@ -752,7 +768,8 @@ static int run_worker_plain_queue_mode(void *arg) odp_packet_from_event_multi(pkt_tbl, event, pkts); - pkts = process_extra_features(pkt_tbl, pkts, stats); + pkts = process_extra_features(appl_args, pkt_tbl, pkts, stats); + if (odp_unlikely(pkts) == 0) continue; @@ -810,6 +827,7 @@ static int run_worker_direct_mode(void *arg) int pktio = 0; thread_args_t *thr_args = arg; stats_t *stats = &thr_args->stats; + const appl_args_t *appl_args = &gbl_args->appl; int use_event_queue = gbl_args->appl.out_mode; thr = odp_thread_id(); @@ -844,7 +862,8 @@ static int run_worker_direct_mode(void *arg) if (odp_unlikely(pkts <= 0)) continue; - pkts = process_extra_features(pkt_tbl, pkts, stats); + pkts = process_extra_features(appl_args, pkt_tbl, pkts, stats); + if (odp_unlikely(pkts) == 0) continue; From 54e97bce70bce03165a06e334e9efa10a009ed7f Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Tue, 30 Jan 2024 11:49:53 +0200 Subject: [PATCH 093/147] test: l2fwd: add packet data prefetch option Add -F option to control packet data prefetch. This allows testing without prefetch, since too late/too many prefetch may be harmful to performance. Moved prefetch loop directly after reception of a packet burst. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/performance/odp_l2fwd.c | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/test/performance/odp_l2fwd.c b/test/performance/odp_l2fwd.c index dc46510c2..de7559818 100644 --- a/test/performance/odp_l2fwd.c +++ b/test/performance/odp_l2fwd.c @@ -91,6 +91,9 @@ typedef struct { /* Some extra features (e.g. error checks) have been enabled */ uint8_t extra_feat; + /* Prefetch packet data */ + uint8_t prefetch; + /* Change destination eth addresses */ uint8_t dst_change; @@ -271,6 +274,15 @@ static inline int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned num) return dropped; } +static inline void prefetch_data(uint8_t prefetch, odp_packet_t pkt_tbl[], uint32_t num) +{ + if (prefetch == 0) + return; + + for (uint32_t i = 0; i < num; i++) + odp_packet_prefetch(pkt_tbl[i], 0, prefetch * 64); +} + /* * Fill packets' eth addresses according to the destination port * @@ -290,9 +302,6 @@ static inline void fill_eth_addrs(odp_packet_t pkt_tbl[], for (i = 0; i < num; ++i) { pkt = pkt_tbl[i]; - - odp_packet_prefetch(pkt, 0, ODPH_ETHHDR_LEN); - eth = odp_packet_data(pkt); if (gbl_args->appl.src_change) @@ -505,6 +514,8 @@ static int run_worker_sched_mode_vector(void *arg) pkts = odp_packet_vector_tbl(pkt_vec, &pkt_tbl); } + prefetch_data(appl_args->prefetch, pkt_tbl, pkts); + pkts = process_extra_features(appl_args, pkt_tbl, pkts, stats); if (odp_unlikely(pkts) == 0) { @@ -653,6 +664,8 @@ static int run_worker_sched_mode(void *arg) odp_packet_from_event_multi(pkt_tbl, ev_tbl, pkts); + prefetch_data(appl_args->prefetch, pkt_tbl, pkts); + pkts = process_extra_features(appl_args, pkt_tbl, pkts, stats); if (odp_unlikely(pkts) == 0) @@ -768,6 +781,8 @@ static int run_worker_plain_queue_mode(void *arg) odp_packet_from_event_multi(pkt_tbl, event, pkts); + prefetch_data(appl_args->prefetch, pkt_tbl, pkts); + pkts = process_extra_features(appl_args, pkt_tbl, pkts, stats); if (odp_unlikely(pkts) == 0) @@ -862,6 +877,8 @@ static int run_worker_direct_mode(void *arg) if (odp_unlikely(pkts <= 0)) continue; + prefetch_data(appl_args->prefetch, pkt_tbl, pkts); + pkts = process_extra_features(appl_args, pkt_tbl, pkts, stats); if (odp_unlikely(pkts) == 0) @@ -1547,6 +1564,7 @@ static void usage(char *progname) " -l, --packet_len Maximum length of packets supported (default %d).\n" " -L, --seg_len Packet pool segment length\n" " (default equal to packet length).\n" + " -F, --prefetch Prefetch packet data in 64 byte multiples (default 1).\n" " -f, --flow_aware Enable flow aware scheduling.\n" " -T, --input_ts Enable packet input timestamping.\n" " -v, --verbose Verbose output.\n" @@ -1597,6 +1615,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) {"promisc_mode", no_argument, NULL, 'P'}, {"packet_len", required_argument, NULL, 'l'}, {"seg_len", required_argument, NULL, 'L'}, + {"prefetch", required_argument, NULL, 'F'}, {"flow_aware", no_argument, NULL, 'f'}, {"input_ts", no_argument, NULL, 'T'}, {"verbose", no_argument, NULL, 'v'}, @@ -1604,7 +1623,8 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:t:a:i:m:o:r:d:s:e:k:g:G:I:b:q:p:y:n:l:L:w:x:z:M:uPfTvh"; + static const char *shortopts = "+c:t:a:i:m:o:r:d:s:e:k:g:G:I:" + "b:q:p:y:n:l:L:w:x:z:M:F:uPfTvh"; appl_args->time = 0; /* loop forever if time to run is 0 */ appl_args->accuracy = 1; /* get and print pps stats second */ @@ -1632,6 +1652,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->flow_aware = 0; appl_args->input_ts = 0; appl_args->num_prio = 0; + appl_args->prefetch = 1; while (1) { opt = getopt_long(argc, argv, shortopts, longopts, &long_index); @@ -1833,6 +1854,9 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) case 'z': appl_args->vec_tmo_ns = atoi(optarg); break; + case 'F': + appl_args->prefetch = atoi(optarg); + break; case 'f': appl_args->flow_aware = 1; break; @@ -1952,6 +1976,7 @@ static void print_info(void) printf("Packets per pool: %u\n", gbl_args->num_pkt); printf("Packet length: %u\n", gbl_args->pkt_len); printf("Segment length: %u\n", gbl_args->seg_len); + printf("Prefetch data %u bytes\n", appl_args->prefetch * 64); printf("Vectors per pool: %u\n", gbl_args->vector_num); printf("Vector size: %u\n", gbl_args->vector_max_size); printf("Priority per IF: "); From 280f006fbd7b2e450c380e1b2a2276343f2436e6 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Wed, 31 Jan 2024 10:19:00 +0200 Subject: [PATCH 094/147] test: l2fwd: add packet data read option Add -R option which can be used to simulate application reading data from each received packet. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/performance/odp_l2fwd.c | 59 +++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/test/performance/odp_l2fwd.c b/test/performance/odp_l2fwd.c index de7559818..075c71695 100644 --- a/test/performance/odp_l2fwd.c +++ b/test/performance/odp_l2fwd.c @@ -100,6 +100,9 @@ typedef struct { /* Change source eth addresses */ uint8_t src_change; + /* Read packet data in uint64_t words */ + uint16_t data_rd; + /* Check packet errors */ uint8_t error_check; @@ -154,6 +157,8 @@ typedef union ODP_ALIGNED_CACHE { uint64_t tx_drops; /* Number of failed packet copies */ uint64_t copy_fails; + /* Dummy sum of packet data */ + uint64_t dummy_sum; } s; uint8_t padding[ODP_CACHE_LINE_SIZE]; @@ -347,6 +352,30 @@ static inline void chksum_insert(odp_packet_t *pkt_tbl, int pkts) } } +static inline void data_rd(odp_packet_t *pkt_tbl, int num, uint16_t rd_words, stats_t *stats) +{ + odp_packet_t pkt; + uint64_t *data; + int i; + uint32_t len, words, j; + uint64_t sum = 0; + + for (i = 0; i < num; i++) { + pkt = pkt_tbl[i]; + data = odp_packet_data(pkt); + len = odp_packet_seg_len(pkt); + + words = rd_words; + if (rd_words * 8 > len) + words = len / 8; + + for (j = 0; j < words; j++) + sum += data[j]; + } + + stats->s.dummy_sum += sum; +} + static inline int copy_packets(odp_packet_t *pkt_tbl, int pkts) { odp_packet_t old_pkt, new_pkt; @@ -376,6 +405,11 @@ static inline int process_extra_features(const appl_args_t *appl_args, odp_packe int pkts, stats_t *stats) { if (odp_unlikely(appl_args->extra_feat)) { + uint16_t rd_words = appl_args->data_rd; + + if (rd_words) + data_rd(pkt_tbl, pkts, rd_words, stats); + if (appl_args->packet_copy) { int fails; @@ -1548,6 +1582,10 @@ static void usage(char *progname) " -p, --packet_copy 0: Don't copy packet (default)\n" " 1: Create and send copy of the received packet.\n" " Free the original packet.\n" + " -R, --data_rd Number of packet data words (uint64_t) to read from\n" + " every received packet. Number of words is rounded down\n" + " to fit into the first segment of a packet. Default\n" + " is 0.\n" " -y, --pool_per_if Create a packet (and packet vector) pool per interface.\n" " 0: Share a single pool between all interfaces (default)\n" " 1: Create a pool per interface\n" @@ -1605,6 +1643,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) {"burst_rx", required_argument, NULL, 'b'}, {"rx_queues", required_argument, NULL, 'q'}, {"packet_copy", required_argument, NULL, 'p'}, + {"data_rd", required_argument, NULL, 'R'}, {"pool_per_if", required_argument, NULL, 'y'}, {"num_pkt", required_argument, NULL, 'n'}, {"num_vec", required_argument, NULL, 'w'}, @@ -1624,7 +1663,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) }; static const char *shortopts = "+c:t:a:i:m:o:r:d:s:e:k:g:G:I:" - "b:q:p:y:n:l:L:w:x:z:M:F:uPfTvh"; + "b:q:p:R:y:n:l:L:w:x:z:M:F:uPfTvh"; appl_args->time = 0; /* loop forever if time to run is 0 */ appl_args->accuracy = 1; /* get and print pps stats second */ @@ -1653,6 +1692,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->input_ts = 0; appl_args->num_prio = 0; appl_args->prefetch = 1; + appl_args->data_rd = 0; while (1) { opt = getopt_long(argc, argv, shortopts, longopts, &long_index); @@ -1824,6 +1864,9 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) case 'p': appl_args->packet_copy = atoi(optarg); break; + case 'R': + appl_args->data_rd = atoi(optarg); + break; case 'y': appl_args->pool_per_if = atoi(optarg); break; @@ -1901,7 +1944,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->extra_feat = 0; if (appl_args->error_check || appl_args->chksum || - appl_args->packet_copy) + appl_args->packet_copy || appl_args->data_rd) appl_args->extra_feat = 1; optind = 1; /* reset 'extern optind' from the getopt lib */ @@ -1957,10 +2000,11 @@ static void print_info(void) appl_args->if_count : 1); if (appl_args->extra_feat) { - printf("Extra features: %s%s%s\n", + printf("Extra features: %s%s%s%s\n", appl_args->error_check ? "error_check " : "", appl_args->chksum ? "chksum " : "", - appl_args->packet_copy ? "packet_copy" : ""); + appl_args->packet_copy ? "packet_copy " : "", + appl_args->data_rd ? "data_rd" : ""); } printf("Num worker threads: %i\n", appl_args->num_workers); @@ -1976,6 +2020,7 @@ static void print_info(void) printf("Packets per pool: %u\n", gbl_args->num_pkt); printf("Packet length: %u\n", gbl_args->pkt_len); printf("Segment length: %u\n", gbl_args->seg_len); + printf("Read data: %u bytes\n", appl_args->data_rd * 8); printf("Prefetch data %u bytes\n", appl_args->prefetch * 64); printf("Vectors per pool: %u\n", gbl_args->vector_num); printf("Vector size: %u\n", gbl_args->vector_max_size); @@ -2216,6 +2261,12 @@ int main(int argc, char *argv[]) printf("\nWarning: Segment length requested %d configured %d\n", gbl_args->appl.seg_len, seg_len); + if (seg_len < gbl_args->appl.data_rd * 8) { + ODPH_ERR("Requested data read length %u exceeds maximum segment length %u\n", + gbl_args->appl.data_rd * 8, seg_len); + return -1; + } + /* zero means default number of packets */ if (gbl_args->appl.num_pkt == 0) num_pkt = DEFAULT_NUM_PKT; From f63fc68f49d72030d41f7820635e303cc63098b6 Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Wed, 31 Jan 2024 12:37:22 +0200 Subject: [PATCH 095/147] test: l2fwd: print resulting pool parameters When user leaves pool options to default values, pool capabilities are used to select the values. Print first option values from the user, and then the resulting pool parameters values. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/performance/odp_l2fwd.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/test/performance/odp_l2fwd.c b/test/performance/odp_l2fwd.c index 075c71695..254fa83a2 100644 --- a/test/performance/odp_l2fwd.c +++ b/test/performance/odp_l2fwd.c @@ -1950,10 +1950,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) optind = 1; /* reset 'extern optind' from the getopt lib */ } -/* - * Print system and application info - */ -static void print_info(void) +static void print_options(void) { int i; appl_args_t *appl_args = &gbl_args->appl; @@ -2017,13 +2014,14 @@ static void print_info(void) else printf("group: ODP_SCHED_GROUP_WORKER\n"); - printf("Packets per pool: %u\n", gbl_args->num_pkt); - printf("Packet length: %u\n", gbl_args->pkt_len); - printf("Segment length: %u\n", gbl_args->seg_len); + printf("Packets per pool: %u\n", appl_args->num_pkt); + printf("Packet length: %u\n", appl_args->packet_len); + printf("Segment length: %u\n", appl_args->seg_len == UINT32_MAX ? 0 : + appl_args->seg_len); printf("Read data: %u bytes\n", appl_args->data_rd * 8); printf("Prefetch data %u bytes\n", appl_args->prefetch * 64); - printf("Vectors per pool: %u\n", gbl_args->vector_num); - printf("Vector size: %u\n", gbl_args->vector_max_size); + printf("Vectors per pool: %u\n", appl_args->num_vec); + printf("Vector size: %u\n", appl_args->vec_size); printf("Priority per IF: "); for (i = 0; i < appl_args->if_count; i++) @@ -2212,8 +2210,7 @@ int main(int argc, char *argv[]) gbl_args->appl.num_workers = num_workers; - /* Print application information */ - print_info(); + print_options(); for (i = 0; i < num_workers; i++) gbl_args->thread_args[i].thr_idx = i; @@ -2289,6 +2286,11 @@ int main(int argc, char *argv[]) gbl_args->pkt_len = pkt_len; gbl_args->seg_len = seg_len; + printf("Resulting pool parameter values:\n"); + printf("Packets per pool: %u\n", num_pkt); + printf("Packet length: %u\n", pkt_len); + printf("Segment length: %u\n", seg_len); + /* Create packet pool */ odp_pool_param_init(¶ms); params.pkt.seg_len = seg_len; @@ -2329,6 +2331,10 @@ int main(int argc, char *argv[]) gbl_args->vector_num = params.vector.num; gbl_args->vector_max_size = params.vector.max_size; + /* Print resulting values */ + printf("Vectors per pool: %u\n", gbl_args->vector_num); + printf("Vector size: %u\n", gbl_args->vector_max_size); + for (i = 0; i < num_vec_pools; i++) { vec_pool_tbl[i] = odp_pool_create("vector pool", ¶ms); @@ -2342,6 +2348,8 @@ int main(int argc, char *argv[]) } } + printf("\n"); + bind_workers(); odp_schedule_config_init(&sched_config); From df1a77a9518a223d3e5131a4261126d340382e4e Mon Sep 17 00:00:00 2001 From: Petri Savolainen Date: Mon, 5 Feb 2024 12:20:44 +0200 Subject: [PATCH 096/147] test: l2fwd: add packet print option Added -V option which can be used to print debug information on every received packet. Signed-off-by: Petri Savolainen Reviewed-by: Matias Elo --- test/performance/odp_l2fwd.c | 48 +++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/test/performance/odp_l2fwd.c b/test/performance/odp_l2fwd.c index 254fa83a2..7435c5769 100644 --- a/test/performance/odp_l2fwd.c +++ b/test/performance/odp_l2fwd.c @@ -112,6 +112,9 @@ typedef struct { /* Checksum offload */ uint8_t chksum; + /* Print debug info on every packet */ + uint8_t verbose_pkt; + unsigned int cpu_count; int if_count; /* Number of interfaces to be used */ int addr_count; /* Number of dst addresses to be used */ @@ -352,6 +355,33 @@ static inline void chksum_insert(odp_packet_t *pkt_tbl, int pkts) } } +static void print_packets(odp_packet_t *pkt_tbl, int num) +{ + odp_packet_t pkt; + uintptr_t data_ptr; + uint32_t bit, align; + + for (int i = 0; i < num; i++) { + pkt = pkt_tbl[i]; + data_ptr = (uintptr_t)odp_packet_data(pkt); + + for (bit = 0, align = 1; bit < 32; bit++, align *= 2) + if (data_ptr & (0x1 << bit)) + break; + + printf(" Packet data: 0x%" PRIxPTR "\n" + " Packet len: %u\n" + " Packet seg len: %u\n" + " Data align: %u\n" + " Num segments: %i\n" + " Headroom size: %u\n" + " User area size: %u\n\n", + data_ptr, odp_packet_len(pkt), odp_packet_seg_len(pkt), align, + odp_packet_num_segs(pkt), odp_packet_headroom(pkt), + odp_packet_user_area_size(pkt)); + } +} + static inline void data_rd(odp_packet_t *pkt_tbl, int num, uint16_t rd_words, stats_t *stats) { odp_packet_t pkt; @@ -407,6 +437,9 @@ static inline int process_extra_features(const appl_args_t *appl_args, odp_packe if (odp_unlikely(appl_args->extra_feat)) { uint16_t rd_words = appl_args->data_rd; + if (appl_args->verbose_pkt) + print_packets(pkt_tbl, pkts); + if (rd_words) data_rd(pkt_tbl, pkts, rd_words, stats); @@ -1606,6 +1639,7 @@ static void usage(char *progname) " -f, --flow_aware Enable flow aware scheduling.\n" " -T, --input_ts Enable packet input timestamping.\n" " -v, --verbose Verbose output.\n" + " -V, --verbose_pkt Print debug information on every received packet.\n" " -h, --help Display help and exit.\n\n" "\n", DEFAULT_VEC_SIZE, DEFAULT_VEC_TMO, POOL_PKT_LEN); } @@ -1658,12 +1692,13 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) {"flow_aware", no_argument, NULL, 'f'}, {"input_ts", no_argument, NULL, 'T'}, {"verbose", no_argument, NULL, 'v'}, + {"verbose_pkt", no_argument, NULL, 'V'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; static const char *shortopts = "+c:t:a:i:m:o:r:d:s:e:k:g:G:I:" - "b:q:p:R:y:n:l:L:w:x:z:M:F:uPfTvh"; + "b:q:p:R:y:n:l:L:w:x:z:M:F:uPfTvVh"; appl_args->time = 0; /* loop forever if time to run is 0 */ appl_args->accuracy = 1; /* get and print pps stats second */ @@ -1677,6 +1712,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->burst_rx = 0; appl_args->rx_queues = 0; appl_args->verbose = 0; + appl_args->verbose_pkt = 0; appl_args->chksum = 0; /* don't use checksum offload by default */ appl_args->pool_per_if = 0; appl_args->num_pkt = 0; @@ -1909,6 +1945,9 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) case 'v': appl_args->verbose = 1; break; + case 'V': + appl_args->verbose_pkt = 1; + break; case 'h': usage(argv[0]); exit(EXIT_SUCCESS); @@ -1944,7 +1983,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->extra_feat = 0; if (appl_args->error_check || appl_args->chksum || - appl_args->packet_copy || appl_args->data_rd) + appl_args->packet_copy || appl_args->data_rd || appl_args->verbose_pkt) appl_args->extra_feat = 1; optind = 1; /* reset 'extern optind' from the getopt lib */ @@ -1997,11 +2036,12 @@ static void print_options(void) appl_args->if_count : 1); if (appl_args->extra_feat) { - printf("Extra features: %s%s%s%s\n", + printf("Extra features: %s%s%s%s%s\n", appl_args->error_check ? "error_check " : "", appl_args->chksum ? "chksum " : "", appl_args->packet_copy ? "packet_copy " : "", - appl_args->data_rd ? "data_rd" : ""); + appl_args->data_rd ? "data_rd" : "", + appl_args->verbose_pkt ? "verbose_pkt" : ""); } printf("Num worker threads: %i\n", appl_args->num_workers); From 53d82cb76a7ad94fdbb506570372ee91ab12d058 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 13 Nov 2023 15:20:40 +0200 Subject: [PATCH 097/147] linux-gen: wait_until: add generic 32-bit wait until functions Add architecture independent 32-bit wait until functions. The ARM specific WFE loop is temporarily removed from scalable scheduler and queue implementations. WFE support will be added back in a following commit, which adds aarch64 specific wait until functions. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- platform/linux-generic/Makefile.am | 15 ++++--- .../arch/default/odp_wait_until.h | 41 +++++++++++++++++++ platform/linux-generic/odp_queue_scalable.c | 29 ++++--------- .../linux-generic/odp_schedule_scalable.c | 27 ++++-------- 4 files changed, 66 insertions(+), 46 deletions(-) create mode 100644 platform/linux-generic/arch/default/odp_wait_until.h diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index f3707ab3a..f4683f5f7 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -352,7 +352,8 @@ noinst_HEADERS += arch/arm/odp_atomic.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h + arch/default/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_AARCH64 __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ @@ -382,7 +383,8 @@ noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/cpu_flags.h \ arch/aarch64/odp_cpu_idling.h \ arch/aarch64/odp_llsc.h \ - arch/aarch64/odp_random.h + arch/aarch64/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_DEFAULT __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ @@ -406,7 +408,8 @@ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h + arch/default/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_POWERPC __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ @@ -430,7 +433,8 @@ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h + arch/default/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_X86 __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ @@ -460,7 +464,8 @@ noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_random.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h + arch/default/odp_cpu_idling.h \ + arch/default/odp_wait_until.h endif if ODP_PKTIO_PCAP diff --git a/platform/linux-generic/arch/default/odp_wait_until.h b/platform/linux-generic/arch/default/odp_wait_until.h new file mode 100644 index 000000000..e81031e2d --- /dev/null +++ b/platform/linux-generic/arch/default/odp_wait_until.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2024 Nokia + */ + +#ifndef ODP_DEFAULT_WAIT_UNTIL_H_ +#define ODP_DEFAULT_WAIT_UNTIL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include + +/** + * The _odp_wait_until_eq_*() functions defined in this header are intended to + * be used only with the scalable scheduler and queue implementations. Even + * though these functions use standard non-atomic parameter types, the + * parameters must only be operated using atomic operations. If new functions + * are added to this file, they should use _odp_wait_until_equal_*() prefix and + * atomic parameter types. + */ + +static inline void _odp_wait_until_eq_u32(uint32_t *val, uint32_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_RELAXED) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u32(uint32_t *val, uint32_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index c7040dd3c..e04d39283 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -16,9 +16,10 @@ #include #include +#include + #include #include - #include #include #include @@ -472,12 +473,8 @@ static int queue_destroy(odp_queue_t handle) */ while (__atomic_load_n(&q->qschst.numevts, __ATOMIC_RELAXED) != 0 || __atomic_load_n(&q->qschst.cur_ticket, __ATOMIC_RELAXED) != - __atomic_load_n(&q->qschst.nxt_ticket, __ATOMIC_RELAXED)) { - sevl(); - while (wfe() && monitor32((uint32_t *)&q->qschst.numevts, - __ATOMIC_RELAXED) != 0) - odp_cpu_pause(); - } + __atomic_load_n(&q->qschst.nxt_ticket, __ATOMIC_RELAXED)) + _odp_wait_until_eq_u32((uint32_t *)&q->qschst.numevts, 0); if (q->schedq != NULL) { _odp_sched_queue_rem(q->sched_grp, q->sched_prio); @@ -596,13 +593,8 @@ static inline int _odp_queue_enq(sched_elem_t *q, __builtin_prefetch(&q->node, 1, 0); #endif /* Wait for our turn to signal consumers */ - if (odp_unlikely(__atomic_load_n(&q->cons_write, - __ATOMIC_RELAXED) != old_write)) { - sevl(); - while (wfe() && monitor32(&q->cons_write, - __ATOMIC_RELAXED) != old_write) - odp_cpu_pause(); - } + if (odp_unlikely(__atomic_load_n(&q->cons_write, __ATOMIC_RELAXED) != old_write)) + _odp_wait_until_eq_u32(&q->cons_write, old_write); /* Signal consumers that events are available (release events) * Enable other producers to continue @@ -824,13 +816,8 @@ int _odp_queue_deq(sched_elem_t *q, _odp_event_hdr_t *event_hdr[], int num) __builtin_prefetch(&q->node, 1, 0); #endif /* Wait for our turn to signal producers */ - if (odp_unlikely(__atomic_load_n(&q->prod_read, __ATOMIC_RELAXED) != - old_read)) { - sevl(); - while (wfe() && monitor32(&q->prod_read, - __ATOMIC_RELAXED) != old_read) - odp_cpu_pause(); - } + if (odp_unlikely(__atomic_load_n(&q->prod_read, __ATOMIC_RELAXED) != old_read)) + _odp_wait_until_eq_u32(&q->prod_read, old_read); /* Signal producers that empty slots are available * (release ring slots) diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 6d60c048f..1da55c5c3 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -21,6 +21,8 @@ #include #include +#include + #include #include #include @@ -1054,15 +1056,8 @@ static int _schedule(odp_queue_t *from, odp_event_t ev[], int num_evts) continue; } /* Wait for our turn to dequeue */ - if (odp_unlikely(__atomic_load_n(&rwin->turn, - __ATOMIC_ACQUIRE) - != sn)) { - sevl(); - while (wfe() && - monitor32(&rwin->turn, __ATOMIC_ACQUIRE) - != sn) - odp_cpu_pause(); - } + if (odp_unlikely(__atomic_load_n(&rwin->turn, __ATOMIC_ACQUIRE) != sn)) + _odp_wait_until_eq_acq_u32(&rwin->turn, sn); #ifdef CONFIG_QSCHST_LOCK LOCK(&elem->qschlock); #endif @@ -1143,13 +1138,8 @@ static void schedule_order_lock(uint32_t lock_index) return; } if (odp_unlikely(__atomic_load_n(&rctx->rwin->olock[lock_index], - __ATOMIC_ACQUIRE) != rctx->sn)) { - sevl(); - while (wfe() && - monitor32(&rctx->rwin->olock[lock_index], - __ATOMIC_ACQUIRE) != rctx->sn) - odp_cpu_pause(); - } + __ATOMIC_ACQUIRE) != rctx->sn)) + _odp_wait_until_eq_acq_u32(&rctx->rwin->olock[lock_index], rctx->sn); } static void schedule_order_unlock(uint32_t lock_index) @@ -2127,13 +2117,10 @@ static void order_lock(void) _ODP_ASSERT(ts->rctx != NULL); rwin = ts->rctx->rwin; sn = ts->rctx->sn; - sevl(); /* Use acquire ordering to be on the safe side even if * this isn't an acquire/release situation (aka lock). */ - while (wfe() && - monitor32(&rwin->hc.head, __ATOMIC_ACQUIRE) != sn) - odp_cpu_pause(); + _odp_wait_until_eq_acq_u32(&rwin->hc.head, sn); } } From d41bedda01a845fb964f3401a02c1998bf3acbf3 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 13 Nov 2023 16:31:35 +0200 Subject: [PATCH 098/147] linux-gen: wait_until: add generic 8-bit wait until function Add architecture independent 8-bit wait until function. The ARM specific WFE loop is temporarily removed from scalable scheduler implementation. WFE support will be added back in a following commit, which adds aarch64 specific wait until functions. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- .../arch/default/odp_wait_until.h | 6 ++++ .../linux-generic/odp_schedule_scalable.c | 29 ++++++------------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/platform/linux-generic/arch/default/odp_wait_until.h b/platform/linux-generic/arch/default/odp_wait_until.h index e81031e2d..8c70ed535 100644 --- a/platform/linux-generic/arch/default/odp_wait_until.h +++ b/platform/linux-generic/arch/default/odp_wait_until.h @@ -28,6 +28,12 @@ static inline void _odp_wait_until_eq_u32(uint32_t *val, uint32_t expected) odp_cpu_pause(); } +static inline void _odp_wait_until_eq_acq_u8(uint8_t *val, uint8_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + static inline void _odp_wait_until_eq_acq_u32(uint32_t *val, uint32_t expected) { while (__atomic_load_n(val, __ATOMIC_ACQUIRE) != expected) diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 1da55c5c3..78c9cc08f 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -225,13 +225,9 @@ void _odp_sched_update_enq(sched_elem_t *q, uint32_t actual) if (odp_unlikely(ticket != TICKET_INVALID)) { /* Wait for our turn to update schedq. */ if (odp_unlikely(__atomic_load_n(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket)) { - sevl(); - while (wfe() && - monitor8(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket) - odp_cpu_pause(); - } + __ATOMIC_ACQUIRE) != ticket)) + _odp_wait_until_eq_acq_u8(&q->qschst.cur_ticket, ticket); + /* Enqueue at end of scheduler queue */ /* We are here because of empty-to-non-empty transition * This means queue must be pushed to schedq if possible @@ -368,13 +364,9 @@ sched_update_deq(sched_elem_t *q, _ODP_ASSERT(q->qschst_type != ODP_SCHED_SYNC_ATOMIC); /* Wait for our turn to update schedq. */ if (odp_unlikely(__atomic_load_n(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket)) { - sevl(); - while (wfe() && - monitor8(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket) - odp_cpu_pause(); - } + __ATOMIC_ACQUIRE) != ticket)) + _odp_wait_until_eq_acq_u8(&q->qschst.cur_ticket, ticket); + /* We are here because of non-empty-to-empty transition or * WRR budget exhausted * This means the queue must be popped from the schedq, now or @@ -496,12 +488,9 @@ static inline void sched_update_popd(sched_elem_t *elem) 1, __ATOMIC_RELAXED); if (odp_unlikely(__atomic_load_n(&elem->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket)) { - sevl(); - while (wfe() && monitor8(&elem->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket) - odp_cpu_pause(); - } + __ATOMIC_ACQUIRE) != ticket)) + _odp_wait_until_eq_acq_u8(&elem->qschst.cur_ticket, ticket); + sched_update_popd_sc(elem); atomic_store_release(&elem->qschst.cur_ticket, ticket + 1, /*readonly=*/false); From 46558d971d3095bf5eb4b172a04180c82f24b927 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 13 Nov 2023 17:02:46 +0200 Subject: [PATCH 099/147] linux-gen: wait_until: add generic bitset wait until function Add architecture independent bitset wait until function. The ARM specific WFE loop is temporarily removed from scalable scheduler implementation. WFE support will be added back in a following commit, which adds aarch64 specific wait until functions. This commit also fixes a bug in the 32-bit Arm implementation, where the bitset_monitor() function could previously operate on a 128-bit bitset using 64-bit lld() function. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- .../arch/default/odp_wait_until.h | 6 ++++++ platform/linux-generic/include/odp_bitset.h | 18 ------------------ platform/linux-generic/odp_schedule_scalable.c | 7 +------ 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/platform/linux-generic/arch/default/odp_wait_until.h b/platform/linux-generic/arch/default/odp_wait_until.h index 8c70ed535..c51f4355e 100644 --- a/platform/linux-generic/arch/default/odp_wait_until.h +++ b/platform/linux-generic/arch/default/odp_wait_until.h @@ -28,6 +28,12 @@ static inline void _odp_wait_until_eq_u32(uint32_t *val, uint32_t expected) odp_cpu_pause(); } +static inline void _odp_wait_until_eq_bitset(bitset_t *val, bitset_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_RELAXED) != expected) + odp_cpu_pause(); +} + static inline void _odp_wait_until_eq_acq_u8(uint8_t *val, uint8_t expected) { while (__atomic_load_n(val, __ATOMIC_ACQUIRE) != expected) diff --git a/platform/linux-generic/include/odp_bitset.h b/platform/linux-generic/include/odp_bitset.h index 0931fb337..e55b9ef1a 100644 --- a/platform/linux-generic/include/odp_bitset.h +++ b/platform/linux-generic/include/odp_bitset.h @@ -32,12 +32,6 @@ static inline uint32_t bitset_ffs(bitset_t b) return __builtin_ffsl(b); } -/* Load-exclusive with memory ordering */ -static inline bitset_t bitset_monitor(bitset_t *bs, int mo) -{ - return monitor32(bs, mo); -} - #elif ATOM_BITSET_SIZE <= 64 /* Return first-bit-set with StdC ffs() semantics */ @@ -46,12 +40,6 @@ static inline uint32_t bitset_ffs(bitset_t b) return __builtin_ffsll(b); } -/* Load-exclusive with memory ordering */ -static inline bitset_t bitset_monitor(bitset_t *bs, int mo) -{ - return monitor64(bs, mo); -} - #elif ATOM_BITSET_SIZE <= 128 /* Return first-bit-set with StdC ffs() semantics */ @@ -65,12 +53,6 @@ static inline uint32_t bitset_ffs(bitset_t b) return 0; } -/* Load-exclusive with memory ordering */ -static inline bitset_t bitset_monitor(bitset_t *bs, int mo) -{ - return monitor128(bs, mo); -} - #else #error Unsupported size of bit sets (ATOM_BITSET_SIZE) #endif diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 78c9cc08f..118a4441b 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -1534,12 +1534,7 @@ static int schedule_group_destroy(odp_schedule_group_t group) if (sg->xcount[p] != 0) { bitset_t wanted = atom_bitset_load(&sg->thr_wanted, __ATOMIC_RELAXED); - sevl(); - while (wfe() && - !bitset_is_eql(wanted, - bitset_monitor(&sg->thr_actual[p], - __ATOMIC_RELAXED))) - odp_cpu_pause(); + _odp_wait_until_eq_bitset(&sg->thr_actual[p], wanted); } /* Else ignore because no ODP queues on this prio */ } From 7f3328c03a1ab1241669776a4559874e0fb3e3da Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Mon, 13 Nov 2023 17:05:37 +0200 Subject: [PATCH 100/147] linux-gen: cpu: remove now unused generic idling header The architecture independent CPU idling header has been made redundant by the generic wait until functions. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- platform/linux-generic/Makefile.am | 4 --- .../arch/aarch64/odp_cpu_idling.h | 2 +- .../linux-generic/arch/arm/odp_cpu_idling.h | 2 +- platform/linux-generic/arch/default/odp_cpu.h | 2 +- .../arch/default/odp_cpu_idling.h | 31 ------------------- platform/linux-generic/odp_queue_scalable.c | 2 -- .../linux-generic/odp_schedule_scalable.c | 2 -- 7 files changed, 3 insertions(+), 42 deletions(-) delete mode 100644 platform/linux-generic/arch/default/odp_cpu_idling.h diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index f4683f5f7..e55627031 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -351,7 +351,6 @@ noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_llsc.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ arch/default/odp_random.h \ arch/default/odp_wait_until.h endif @@ -407,7 +406,6 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ arch/default/odp_random.h \ arch/default/odp_wait_until.h endif @@ -432,7 +430,6 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ arch/default/odp_random.h \ arch/default/odp_wait_until.h endif @@ -464,7 +461,6 @@ noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_random.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ arch/default/odp_wait_until.h endif diff --git a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h b/platform/linux-generic/arch/aarch64/odp_cpu_idling.h index a6cea8c63..4a4e9817e 100644 --- a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h +++ b/platform/linux-generic/arch/aarch64/odp_cpu_idling.h @@ -15,7 +15,7 @@ #ifndef CONFIG_WFE -#include "../default/odp_cpu_idling.h" +#include "../default/odp_wait_until.h" #else /* CONFIG_WFE */ diff --git a/platform/linux-generic/arch/arm/odp_cpu_idling.h b/platform/linux-generic/arch/arm/odp_cpu_idling.h index a6cea8c63..4a4e9817e 100644 --- a/platform/linux-generic/arch/arm/odp_cpu_idling.h +++ b/platform/linux-generic/arch/arm/odp_cpu_idling.h @@ -15,7 +15,7 @@ #ifndef CONFIG_WFE -#include "../default/odp_cpu_idling.h" +#include "../default/odp_wait_until.h" #else /* CONFIG_WFE */ diff --git a/platform/linux-generic/arch/default/odp_cpu.h b/platform/linux-generic/arch/default/odp_cpu.h index 821956819..6b10966c6 100644 --- a/platform/linux-generic/arch/default/odp_cpu.h +++ b/platform/linux-generic/arch/default/odp_cpu.h @@ -21,6 +21,6 @@ __atomic_store_n(loc, val, __ATOMIC_RELEASE) #include "odp_atomic.h" -#include "odp_cpu_idling.h" +#include "odp_wait_until.h" #endif diff --git a/platform/linux-generic/arch/default/odp_cpu_idling.h b/platform/linux-generic/arch/default/odp_cpu_idling.h deleted file mode 100644 index 9d23ad20d..000000000 --- a/platform/linux-generic/arch/default/odp_cpu_idling.h +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_DEFAULT_CPU_IDLING_H_ -#define ODP_DEFAULT_CPU_IDLING_H_ - -/****************************************************************************** - * Idle mgmt - *****************************************************************************/ - -static inline void sevl(void) -{ - /* empty */ -} - -static inline int wfe(void) -{ - return 1; -} - -#define monitor128(addr, mo) __atomic_load_n((addr), (mo)) -#define monitor64(addr, mo) __atomic_load_n((addr), (mo)) -#define monitor32(addr, mo) __atomic_load_n((addr), (mo)) -#define monitor8(addr, mo) __atomic_load_n((addr), (mo)) - -#endif diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index e04d39283..bddaa532d 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -16,8 +16,6 @@ #include #include -#include - #include #include #include diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 118a4441b..5166fb6d0 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -21,8 +21,6 @@ #include #include -#include - #include #include #include From d4dd30fe451729e2d704077527ca401b754437c2 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 14 Nov 2023 09:53:28 +0200 Subject: [PATCH 101/147] linux-gen: wait_until: add aarch64 specific wait until functions Add aarch64 specific implementations of internal wait until functions. The old aarch64 specific headers have been merged to simplify the file structure. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- platform/linux-generic/Makefile.am | 4 +- platform/linux-generic/arch/aarch64/odp_cpu.h | 166 +++++++++++++++-- .../arch/aarch64/odp_cpu_idling.h | 39 ---- .../linux-generic/arch/aarch64/odp_llsc.h | 170 ------------------ .../arch/aarch64/odp_wait_until.h | 100 +++++++++++ 5 files changed, 251 insertions(+), 228 deletions(-) delete mode 100644 platform/linux-generic/arch/aarch64/odp_cpu_idling.h delete mode 100644 platform/linux-generic/arch/aarch64/odp_llsc.h create mode 100644 platform/linux-generic/arch/aarch64/odp_wait_until.h diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index e55627031..ab7faf75f 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -380,10 +380,8 @@ endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ arch/aarch64/cpu_flags.h \ - arch/aarch64/odp_cpu_idling.h \ - arch/aarch64/odp_llsc.h \ arch/aarch64/odp_random.h \ - arch/default/odp_wait_until.h + arch/aarch64/odp_wait_until.h endif if ARCH_IS_DEFAULT __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ diff --git a/platform/linux-generic/arch/aarch64/odp_cpu.h b/platform/linux-generic/arch/aarch64/odp_cpu.h index 84bc4dffd..bc2842ad9 100644 --- a/platform/linux-generic/arch/aarch64/odp_cpu.h +++ b/platform/linux-generic/arch/aarch64/odp_cpu.h @@ -14,6 +14,7 @@ #endif #include +#include /* * Use LLD/SCD atomic primitives instead of lock-based code path in llqueue @@ -31,20 +32,6 @@ */ #define CONFIG_DMBSTR -/* - * Use ARM event signalling mechanism - * Event signalling minimises spinning (busy waiting) which decreases - * cache coherency traffic when spinning on shared locations (thus faster and - * more scalable) and enables the CPU to enter a sleep state (lower power - * consumption). - */ -#define CONFIG_WFE - -static inline void _odp_dmb(void) -{ - __asm__ volatile("dmb" : : : "memory"); -} - /* Only ARMv8 supports DMB ISHLD */ /* A load only barrier is much cheaper than full barrier */ #define _odp_release_barrier(ro) \ @@ -55,9 +42,156 @@ do { \ __asm__ volatile("dmb ish" ::: "memory"); \ } while (0) -#include "odp_llsc.h" +static inline uint16_t ll8(uint8_t *var, int mm) +{ + uint16_t old; + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxrb %w0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("ldxrb %w0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + else + _ODP_ABORT(); + return old; +} + +static inline uint32_t ll32(uint32_t *var, int mm) +{ + uint32_t old; + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("ldxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + else + _ODP_ABORT(); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm) +{ + uint32_t ret; + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxr %w0, %w1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("stxr %w0, %w1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + else + _ODP_ABORT(); + return ret; +} + +static inline uint64_t ll64(uint64_t *var, int mm) +{ + uint64_t old; + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxr %0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("ldxr %0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + else + _ODP_ABORT(); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc64(uint64_t *var, uint64_t neu, int mm) +{ + uint32_t ret; + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxr %w0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("stxr %w0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + else + _ODP_ABORT(); + return ret; +} + +union i128 { + _odp_u128_t i128; + int64_t i64[2]; +}; + +static inline _odp_u128_t lld(_odp_u128_t *var, int mm) +{ + union i128 old; + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxp %0, %1, [%2]" + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) + : "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("ldxp %0, %1, [%2]" + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) + : "r" (var) + : ); + else + _ODP_ABORT(); + return old.i128; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm) +{ + uint32_t ret; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxp %w0, %1, %2, [%3]" + : "=&r" (ret) + : "r" (((*(union i128 *)&neu)).i64[0]), + "r" (((*(union i128 *)&neu)).i64[1]), + "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("stxp %w0, %1, %2, [%3]" + : "=&r" (ret) + : "r" (((*(union i128 *)&neu)).i64[0]), + "r" (((*(union i128 *)&neu)).i64[1]), + "r" (var) + : ); + else + _ODP_ABORT(); +#pragma GCC diagnostic pop + return ret; +} + #include "odp_atomic.h" -#include "odp_cpu_idling.h" +#include "odp_wait_until.h" #ifdef __ARM_FEATURE_UNALIGNED #define _ODP_UNALIGNED 1 diff --git a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h b/platform/linux-generic/arch/aarch64/odp_cpu_idling.h deleted file mode 100644 index 4a4e9817e..000000000 --- a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -#ifndef CONFIG_WFE - -#include "../default/odp_wait_until.h" - -#else /* CONFIG_WFE */ - -static inline void sevl(void) -{ - __asm__ volatile("sevl" : : : ); -} - -static inline int wfe(void) -{ - __asm__ volatile("wfe" : : : "memory"); - return 1; -} - -#define monitor128(addr, mo) lld((addr), (mo)) -#define monitor64(addr, mo) ll64((addr), (mo)) -#define monitor32(addr, mo) ll32((addr), (mo)) -#define monitor8(addr, mo) ll8((addr), (mo)) -#endif /* CONFIG_WFE */ - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_llsc.h b/platform/linux-generic/arch/aarch64/odp_llsc.h deleted file mode 100644 index 498785bd4..000000000 --- a/platform/linux-generic/arch/aarch64/odp_llsc.h +++ /dev/null @@ -1,170 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -#include - -static inline uint16_t ll8(uint8_t *var, int mm) -{ - uint16_t old; - - if (mm == __ATOMIC_ACQUIRE) - __asm__ volatile("ldaxrb %w0, [%1]" - : "=&r" (old) - : "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("ldxrb %w0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - else - _ODP_ABORT(); - return old; -} - -static inline uint32_t ll32(uint32_t *var, int mm) -{ - uint32_t old; - - if (mm == __ATOMIC_ACQUIRE) - __asm__ volatile("ldaxr %w0, [%1]" - : "=&r" (old) - : "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("ldxr %w0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - else - _ODP_ABORT(); - return old; -} - -/* Return 0 on success, 1 on failure */ -static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm) -{ - uint32_t ret; - - if (mm == __ATOMIC_RELEASE) - __asm__ volatile("stlxr %w0, %w1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("stxr %w0, %w1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : ); - else - _ODP_ABORT(); - return ret; -} - -static inline uint64_t ll(uint64_t *var, int mm) -{ - uint64_t old; - - if (mm == __ATOMIC_ACQUIRE) - __asm__ volatile("ldaxr %0, [%1]" - : "=&r" (old) - : "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("ldxr %0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - else - _ODP_ABORT(); - return old; -} - -#define ll64(a, b) ll((a), (b)) - -/* Return 0 on success, 1 on failure */ -static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm) -{ - uint32_t ret; - - if (mm == __ATOMIC_RELEASE) - __asm__ volatile("stlxr %w0, %1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("stxr %w0, %1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : ); - else - _ODP_ABORT(); - return ret; -} - -#define sc64(a, b, c) sc((a), (b), (c)) - -union i128 { - _odp_u128_t i128; - int64_t i64[2]; -}; - -static inline _odp_u128_t lld(_odp_u128_t *var, int mm) -{ - union i128 old; - - if (mm == __ATOMIC_ACQUIRE) - __asm__ volatile("ldaxp %0, %1, [%2]" - : "=&r" (old.i64[0]), "=&r" (old.i64[1]) - : "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("ldxp %0, %1, [%2]" - : "=&r" (old.i64[0]), "=&r" (old.i64[1]) - : "r" (var) - : ); - else - _ODP_ABORT(); - return old.i128; -} - -/* Return 0 on success, 1 on failure */ -static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm) -{ - uint32_t ret; - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" - if (mm == __ATOMIC_RELEASE) - __asm__ volatile("stlxp %w0, %1, %2, [%3]" - : "=&r" (ret) - : "r" (((*(union i128 *)&neu)).i64[0]), - "r" (((*(union i128 *)&neu)).i64[1]), - "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("stxp %w0, %1, %2, [%3]" - : "=&r" (ret) - : "r" (((*(union i128 *)&neu)).i64[0]), - "r" (((*(union i128 *)&neu)).i64[1]), - "r" (var) - : ); - else - _ODP_ABORT(); -#pragma GCC diagnostic pop - return ret; -} - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_wait_until.h b/platform/linux-generic/arch/aarch64/odp_wait_until.h new file mode 100644 index 000000000..eca3f9ce5 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_wait_until.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017 ARM Limited + * Copyright (c) 2017-2018 Linaro Limited + * Copyright (c) 2024 Nokia + */ + +#ifndef ODP_AARCH64_WAIT_UNTIL_H_ +#define ODP_AARCH64_WAIT_UNTIL_H_ + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H +#error This file should not be included directly, please include odp_cpu.h +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include + +#include + +static inline void _odp_sevl(void) +{ + __asm__ volatile("sevl" : : : ); +} + +static inline int _odp_wfe(void) +{ + __asm__ volatile("wfe" : : : "memory"); + return 1; +} + +#define _odp_monitor_u8(addr, mo) ll8((addr), (mo)) +#define _odp_monitor_u32(addr, mo) ll32((addr), (mo)) +#define _odp_monitor_u64(addr, mo) ll64((addr), (mo)) +#define _odp_monitor_u128(addr, mo) lld((addr), (mo)) + +#if ATOM_BITSET_SIZE <= 32 +static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) +{ + return _odp_monitor_u32(bs, mo); +} +#elif ATOM_BITSET_SIZE <= 64 +static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) +{ + return _odp_monitor_u64(bs, mo); +} +#elif ATOM_BITSET_SIZE <= 128 +static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) +{ + return _odp_monitor_u128(bs, mo); +} +#else +#error Unsupported size of bit sets (ATOM_BITSET_SIZE) +#endif + +/** + * The _odp_wait_until_eq_*() functions defined in this header are intended to + * be used only with the scalable scheduler and queue implementations. Even + * though these functions use standard non-atomic parameter types, the + * parameters must only be operated using atomic operations. If new functions + * are added to this file, they should use _odp_wait_until_equal_*() prefix and + * atomic parameter types. + */ + +static inline void _odp_wait_until_eq_u32(uint32_t *val, uint32_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_monitor_u32(val, __ATOMIC_RELAXED) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_bitset(bitset_t *val, bitset_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_bitset_monitor(val, __ATOMIC_RELAXED != expected)) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u8(uint8_t *val, uint8_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_monitor_u8(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u32(uint32_t *val, uint32_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_monitor_u32(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +#ifdef __cplusplus +} +#endif + +#endif From f2c3a8785de8ba4e05827958202d04c8e299db65 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 14 Nov 2023 12:38:59 +0200 Subject: [PATCH 102/147] linux-gen: wait_until: clean 32-bit arm headers Drop 32-bit Arm WFE support (already disabled by default), which enables simplifying the Arm CPU specific header files and removing unused code. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- platform/linux-generic/Makefile.am | 2 - platform/linux-generic/arch/arm/odp_cpu.h | 41 +++++--- .../linux-generic/arch/arm/odp_cpu_idling.h | 39 -------- platform/linux-generic/arch/arm/odp_llsc.h | 96 ------------------- 4 files changed, 30 insertions(+), 148 deletions(-) delete mode 100644 platform/linux-generic/arch/arm/odp_cpu_idling.h delete mode 100644 platform/linux-generic/arch/arm/odp_llsc.h diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index ab7faf75f..759153c4c 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -347,8 +347,6 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ - arch/arm/odp_cpu_idling.h \ - arch/arm/odp_llsc.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_random.h \ diff --git a/platform/linux-generic/arch/arm/odp_cpu.h b/platform/linux-generic/arch/arm/odp_cpu.h index 82d47325f..b1a9eac70 100644 --- a/platform/linux-generic/arch/arm/odp_cpu.h +++ b/platform/linux-generic/arch/arm/odp_cpu.h @@ -31,15 +31,6 @@ */ #define CONFIG_DMBSTR -/* - * Use ARM event signalling mechanism - * Event signalling minimises spinning (busy waiting) which decreases - * cache coherency traffic when spinning on shared locations (thus faster and - * more scalable) and enables the CPU to enter a sleep state (lower power - * consumption). - */ -/* #define CONFIG_WFE */ - static inline void _odp_dmb(void) { __asm__ volatile("dmb" : : : "memory"); @@ -48,9 +39,37 @@ static inline void _odp_dmb(void) #define _odp_release_barrier(ro) \ __atomic_thread_fence(__ATOMIC_RELEASE) -#include "odp_llsc.h" +static inline uint64_t lld(uint64_t *var, int mm) +{ + uint64_t old; + + __asm__ volatile("ldrexd %0, %H0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + /* Barrier after an acquiring load */ + if (mm == __ATOMIC_ACQUIRE) + _odp_dmb(); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm) +{ + uint32_t ret; + + /* Barrier before a releasing store */ + if (mm == __ATOMIC_RELEASE) + _odp_dmb(); + __asm__ volatile("strexd %0, %1, %H1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; +} + #include "odp_atomic.h" -#include "odp_cpu_idling.h" +#include "../default/odp_wait_until.h" #ifdef __ARM_FEATURE_UNALIGNED #define _ODP_UNALIGNED 1 diff --git a/platform/linux-generic/arch/arm/odp_cpu_idling.h b/platform/linux-generic/arch/arm/odp_cpu_idling.h deleted file mode 100644 index 4a4e9817e..000000000 --- a/platform/linux-generic/arch/arm/odp_cpu_idling.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -#ifndef CONFIG_WFE - -#include "../default/odp_wait_until.h" - -#else /* CONFIG_WFE */ - -static inline void sevl(void) -{ - __asm__ volatile("sevl" : : : ); -} - -static inline int wfe(void) -{ - __asm__ volatile("wfe" : : : "memory"); - return 1; -} - -#define monitor128(addr, mo) lld((addr), (mo)) -#define monitor64(addr, mo) ll64((addr), (mo)) -#define monitor32(addr, mo) ll32((addr), (mo)) -#define monitor8(addr, mo) ll8((addr), (mo)) -#endif /* CONFIG_WFE */ - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H */ diff --git a/platform/linux-generic/arch/arm/odp_llsc.h b/platform/linux-generic/arch/arm/odp_llsc.h deleted file mode 100644 index 2fea6a0dc..000000000 --- a/platform/linux-generic/arch/arm/odp_llsc.h +++ /dev/null @@ -1,96 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -static inline uint32_t ll8(uint8_t *var, int mm) -{ - uint8_t old; - - __asm__ volatile("ldrexb %0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - /* Barrier after an acquiring load */ - if (mm == __ATOMIC_ACQUIRE) - _odp_dmb(); - return old; -} - -static inline uint32_t ll(uint32_t *var, int mm) -{ - uint32_t old; - - __asm__ volatile("ldrex %0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - /* Barrier after an acquiring load */ - if (mm == __ATOMIC_ACQUIRE) - _odp_dmb(); - return old; -} - -#define ll32(a, b) ll((a), (b)) - -/* Return 0 on success, 1 on failure */ -static inline uint32_t sc(uint32_t *var, uint32_t neu, int mm) -{ - uint32_t ret; - - /* Barrier before a releasing store */ - if (mm == __ATOMIC_RELEASE) - _odp_dmb(); - __asm__ volatile("strex %0, %1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : ); - return ret; -} - -#define sc32(a, b, c) sc((a), (b), (c)) - -static inline uint64_t lld(uint64_t *var, int mm) -{ - uint64_t old; - - __asm__ volatile("ldrexd %0, %H0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - /* Barrier after an acquiring load */ - if (mm == __ATOMIC_ACQUIRE) - _odp_dmb(); - return old; -} - -#define ll64(a, b) lld((a), (b)) - -/* Return 0 on success, 1 on failure */ -static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm) -{ - uint32_t ret; - - /* Barrier before a releasing store */ - if (mm == __ATOMIC_RELEASE) - _odp_dmb(); - __asm__ volatile("strexd %0, %1, %H1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : ); - return ret; -} - -#define sc64(a, b, c) scd((a), (b), (c)) - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H */ From 5f88e8d1e423d360d9d6e467ffcbf9437be0e6b5 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 14 Nov 2023 16:00:04 +0200 Subject: [PATCH 103/147] linux-gen: cpu: optimize aarch64 error branches Optimize aarch64 specific CPU functions' error branches. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- platform/linux-generic/arch/aarch64/odp_cpu.h | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/platform/linux-generic/arch/aarch64/odp_cpu.h b/platform/linux-generic/arch/aarch64/odp_cpu.h index bc2842ad9..ad8b36d87 100644 --- a/platform/linux-generic/arch/aarch64/odp_cpu.h +++ b/platform/linux-generic/arch/aarch64/odp_cpu.h @@ -46,18 +46,18 @@ static inline uint16_t ll8(uint8_t *var, int mm) { uint16_t old; + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + if (mm == __ATOMIC_ACQUIRE) __asm__ volatile("ldaxrb %w0, [%1]" : "=&r" (old) : "r" (var) : "memory"); - else if (mm == __ATOMIC_RELAXED) + else __asm__ volatile("ldxrb %w0, [%1]" : "=&r" (old) : "r" (var) : ); - else - _ODP_ABORT(); return old; } @@ -65,18 +65,18 @@ static inline uint32_t ll32(uint32_t *var, int mm) { uint32_t old; + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + if (mm == __ATOMIC_ACQUIRE) __asm__ volatile("ldaxr %w0, [%1]" : "=&r" (old) : "r" (var) : "memory"); - else if (mm == __ATOMIC_RELAXED) + else __asm__ volatile("ldxr %w0, [%1]" : "=&r" (old) : "r" (var) : ); - else - _ODP_ABORT(); return old; } @@ -85,18 +85,18 @@ static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm) { uint32_t ret; + _ODP_ASSERT(mm == __ATOMIC_RELEASE || mm == __ATOMIC_RELAXED); + if (mm == __ATOMIC_RELEASE) __asm__ volatile("stlxr %w0, %w1, [%2]" : "=&r" (ret) : "r" (neu), "r" (var) : "memory"); - else if (mm == __ATOMIC_RELAXED) + else __asm__ volatile("stxr %w0, %w1, [%2]" : "=&r" (ret) : "r" (neu), "r" (var) : ); - else - _ODP_ABORT(); return ret; } @@ -104,18 +104,18 @@ static inline uint64_t ll64(uint64_t *var, int mm) { uint64_t old; + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + if (mm == __ATOMIC_ACQUIRE) __asm__ volatile("ldaxr %0, [%1]" : "=&r" (old) : "r" (var) : "memory"); - else if (mm == __ATOMIC_RELAXED) + else __asm__ volatile("ldxr %0, [%1]" : "=&r" (old) : "r" (var) : ); - else - _ODP_ABORT(); return old; } @@ -124,18 +124,18 @@ static inline uint32_t sc64(uint64_t *var, uint64_t neu, int mm) { uint32_t ret; + _ODP_ASSERT(mm == __ATOMIC_RELEASE || mm == __ATOMIC_RELAXED); + if (mm == __ATOMIC_RELEASE) __asm__ volatile("stlxr %w0, %1, [%2]" : "=&r" (ret) : "r" (neu), "r" (var) : "memory"); - else if (mm == __ATOMIC_RELAXED) + else __asm__ volatile("stxr %w0, %1, [%2]" : "=&r" (ret) : "r" (neu), "r" (var) : ); - else - _ODP_ABORT(); return ret; } @@ -148,18 +148,18 @@ static inline _odp_u128_t lld(_odp_u128_t *var, int mm) { union i128 old; + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + if (mm == __ATOMIC_ACQUIRE) __asm__ volatile("ldaxp %0, %1, [%2]" : "=&r" (old.i64[0]), "=&r" (old.i64[1]) : "r" (var) : "memory"); - else if (mm == __ATOMIC_RELAXED) + else __asm__ volatile("ldxp %0, %1, [%2]" : "=&r" (old.i64[0]), "=&r" (old.i64[1]) : "r" (var) : ); - else - _ODP_ABORT(); return old.i128; } @@ -168,6 +168,8 @@ static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm) { uint32_t ret; + _ODP_ASSERT(mm == __ATOMIC_RELEASE || mm == __ATOMIC_RELAXED); + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wpedantic" if (mm == __ATOMIC_RELEASE) @@ -177,15 +179,13 @@ static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm) "r" (((*(union i128 *)&neu)).i64[1]), "r" (var) : "memory"); - else if (mm == __ATOMIC_RELAXED) + else __asm__ volatile("stxp %w0, %1, %2, [%3]" : "=&r" (ret) : "r" (((*(union i128 *)&neu)).i64[0]), "r" (((*(union i128 *)&neu)).i64[1]), "r" (var) : ); - else - _ODP_ABORT(); #pragma GCC diagnostic pop return ret; } From 92282e857f4f6d531afffeff9b7d6c177f8f93fe Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Wed, 15 Nov 2023 10:01:34 +0200 Subject: [PATCH 104/147] linux-gen: remove 32-bit arm specific atomics header Remove now unnecessary 32-bit arm odp_atomic.h header. Move remaining arm specific atomic functions to odp_cpu.h and use generic implementations for the rest. Signed-off-by: Matias Elo Reviewed-by: Janne Peltonen --- platform/linux-generic/Makefile.am | 3 +- platform/linux-generic/arch/arm/odp_atomic.h | 109 ------------------- platform/linux-generic/arch/arm/odp_cpu.h | 29 +++-- 3 files changed, 19 insertions(+), 122 deletions(-) delete mode 100644 platform/linux-generic/arch/arm/odp_atomic.h diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 759153c4c..ab4f98720 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -345,8 +345,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/wait_until_generic.h \ arch/default/odp/api/abi/wait_until.h endif -noinst_HEADERS += arch/arm/odp_atomic.h \ - arch/arm/odp_cpu.h \ +noinst_HEADERS += arch/arm/odp_cpu.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_random.h \ diff --git a/platform/linux-generic/arch/arm/odp_atomic.h b/platform/linux-generic/arch/arm/odp_atomic.h deleted file mode 100644 index e400f52d4..000000000 --- a/platform/linux-generic/arch/arm/odp_atomic.h +++ /dev/null @@ -1,109 +0,0 @@ -/* Copyright (c) 2017-2021, ARM Limited. All rights reserved. - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -#include -#include - -#ifdef CONFIG_DMBSTR - -#define atomic_store_release(loc, val, ro) \ -do { \ - _odp_release_barrier(ro); \ - __atomic_store_n(loc, val, __ATOMIC_RELAXED); \ -} while (0) - -#else - -#define atomic_store_release(loc, val, ro) \ - __atomic_store_n(loc, val, __ATOMIC_RELEASE) - -#endif /* CONFIG_DMBSTR */ - -/** Atomic bit set operations with memory ordering */ -#if __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ - __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ -typedef unsigned long long bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) - -#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ -typedef unsigned long bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) - -#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 -typedef unsigned int bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) - -#else -/* Target does not support lock-free atomic operations */ -typedef unsigned int bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) -#endif - -#if ATOM_BITSET_SIZE <= 32 - -static inline bitset_t bitset_mask(uint32_t bit) -{ - return 1UL << bit; -} - -#elif ATOM_BITSET_SIZE <= 64 - -static inline bitset_t bitset_mask(uint32_t bit) -{ - return 1ULL << bit; -} - -#elif ATOM_BITSET_SIZE <= 128 - -static inline bitset_t bitset_mask(uint32_t bit) -{ - if (bit < 64) - return 1ULL << bit; - else - return (_odp_u128_t)(1ULL << (bit - 64)) << 64; -} - -#else -#error Unsupported size of bit sets (ATOM_BITSET_SIZE) -#endif - -static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) -{ - return __atomic_load_n(bs, mo); -} - -static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) -{ - (void)__atomic_fetch_or(bs, bitset_mask(bit), mo); -} - -static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) -{ - (void)__atomic_fetch_and(bs, ~bitset_mask(bit), mo); -} - -static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) -{ - return __atomic_exchange_n(bs, neu, mo); -} - -static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, - bitset_t neu, bool weak, - int mo_success, int mo_failure) -{ - return __atomic_compare_exchange_n(bs, old, neu, weak, mo_success, - mo_failure); -} - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */ diff --git a/platform/linux-generic/arch/arm/odp_cpu.h b/platform/linux-generic/arch/arm/odp_cpu.h index b1a9eac70..6b2674736 100644 --- a/platform/linux-generic/arch/arm/odp_cpu.h +++ b/platform/linux-generic/arch/arm/odp_cpu.h @@ -31,14 +31,6 @@ */ #define CONFIG_DMBSTR -static inline void _odp_dmb(void) -{ - __asm__ volatile("dmb" : : : "memory"); -} - -#define _odp_release_barrier(ro) \ - __atomic_thread_fence(__ATOMIC_RELEASE) - static inline uint64_t lld(uint64_t *var, int mm) { uint64_t old; @@ -49,7 +41,7 @@ static inline uint64_t lld(uint64_t *var, int mm) : ); /* Barrier after an acquiring load */ if (mm == __ATOMIC_ACQUIRE) - _odp_dmb(); + __asm__ volatile("dmb" : : : "memory"); return old; } @@ -60,7 +52,7 @@ static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm) /* Barrier before a releasing store */ if (mm == __ATOMIC_RELEASE) - _odp_dmb(); + __asm__ volatile("dmb" : : : "memory"); __asm__ volatile("strexd %0, %1, %H1, [%2]" : "=&r" (ret) : "r" (neu), "r" (var) @@ -68,7 +60,22 @@ static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm) return ret; } -#include "odp_atomic.h" +#ifdef CONFIG_DMBSTR + +#define atomic_store_release(loc, val, ro) \ +do { \ + __atomic_thread_fence(__ATOMIC_RELEASE); \ + __atomic_store_n(loc, val, __ATOMIC_RELAXED); \ +} while (0) + +#else + +#define atomic_store_release(loc, val, ro) \ + __atomic_store_n(loc, val, __ATOMIC_RELEASE) + +#endif /* CONFIG_DMBSTR */ + +#include "../default/odp_atomic.h" #include "../default/odp_wait_until.h" #ifdef __ARM_FEATURE_UNALIGNED From 35366b3ce83bc9d729e3c679196d208fd9f62087 Mon Sep 17 00:00:00 2001 From: Matias Elo Date: Tue, 13 Feb 2024 13:24:40 +0200 Subject: [PATCH 105/147] checkpatch: update script version Update checkpatch.pl script to the latest version from https://github.com/torvalds/linux repo (commit c664e16bb1ba1c8cf1d7ecf3df5fd83bbb8ac15a). Signed-off-by: Matias Elo Reviewed-by: Petri Savolainen --- scripts/checkpatch.pl | 657 +++++++++++++++++++++++++++++++++--------- 1 file changed, 528 insertions(+), 129 deletions(-) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 83bdf378c..9e6d8d750 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -23,6 +23,9 @@ use Getopt::Long qw(:config no_auto_abbrev); my $quiet = 0; +my $verbose = 0; +my %verbose_messages = (); +my %verbose_emitted = (); my $tree = 1; my $chk_signoff = 1; my $chk_patch = 1; @@ -60,7 +63,9 @@ my $spelling_file = "$D/spelling.txt"; my $codespell = 0; my $codespellfile = "/usr/share/codespell/dictionary.txt"; +my $user_codespellfile = ""; my $conststructsfile = "$D/const_structs.checkpatch"; +my $docsfile = "$D/../Documentation/dev-tools/checkpatch.rst"; my $typedefsfile; my $color = "auto"; my $allow_c99_comments = 1; # Can be overridden by --ignore C99_COMMENT_TOLERANCE @@ -69,6 +74,8 @@ my $tabsize = 8; my ${CONFIG_} = "CONFIG_"; +my %maybe_linker_symbol; # for externs in c exceptions, when seen in *vmlinux.lds.h + sub help { my ($exitcode) = @_; @@ -78,6 +85,7 @@ sub help { Options: -q, --quiet quiet + -v, --verbose verbose mode --no-tree run without a kernel tree --no-signoff do not check for 'Signed-off-by' line --patch treat FILE as patchfile (default) @@ -125,7 +133,7 @@ sub help { --ignore-perl-version override checking of perl version. expect runtime errors. --codespell Use the codespell dictionary for spelling/typos - (default:/usr/share/codespell/dictionary.txt) + (default:$codespellfile) --codespellfile Use this codespell dictionary --typedefsfile Read additional types from this file --color[=WHEN] Use colors 'always', 'never', or only when output @@ -158,15 +166,51 @@ sub list_types { my $text = <$script>; close($script); - my @types = (); + my %types = (); # Also catch when type or level is passed through a variable - for ($text =~ /(?:(?:\bCHK|\bWARN|\bERROR|&\{\$msg_level})\s*\(|\$msg_type\s*=)\s*"([^"]+)"/g) { - push (@types, $_); + while ($text =~ /(?:(\bCHK|\bWARN|\bERROR|&\{\$msg_level})\s*\(|\$msg_type\s*=)\s*"([^"]+)"/g) { + if (defined($1)) { + if (exists($types{$2})) { + $types{$2} .= ",$1" if ($types{$2} ne $1); + } else { + $types{$2} = $1; + } + } else { + $types{$2} = "UNDETERMINED"; + } } - @types = sort(uniq(@types)); + print("#\tMessage type\n\n"); - foreach my $type (@types) { + if ($color) { + print(" ( Color coding: "); + print(RED . "ERROR" . RESET); + print(" | "); + print(YELLOW . "WARNING" . RESET); + print(" | "); + print(GREEN . "CHECK" . RESET); + print(" | "); + print("Multiple levels / Undetermined"); + print(" )\n\n"); + } + + foreach my $type (sort keys %types) { + my $orig_type = $type; + if ($color) { + my $level = $types{$type}; + if ($level eq "ERROR") { + $type = RED . $type . RESET; + } elsif ($level eq "WARN") { + $type = YELLOW . $type . RESET; + } elsif ($level eq "CHK") { + $type = GREEN . $type . RESET; + } + } print(++$count . "\t" . $type . "\n"); + if ($verbose && exists($verbose_messages{$orig_type})) { + my $message = $verbose_messages{$orig_type}; + $message =~ s/\n/\n\t/g; + print("\t" . $message . "\n\n"); + } } exit($exitcode); @@ -198,6 +242,46 @@ sub list_types { unshift(@ARGV, @conf_args) if @conf_args; } +sub load_docs { + open(my $docs, '<', "$docsfile") + or warn "$P: Can't read the documentation file $docsfile $!\n"; + + my $type = ''; + my $desc = ''; + my $in_desc = 0; + + while (<$docs>) { + chomp; + my $line = $_; + $line =~ s/\s+$//; + + if ($line =~ /^\s*\*\*(.+)\*\*$/) { + if ($desc ne '') { + $verbose_messages{$type} = trim($desc); + } + $type = $1; + $desc = ''; + $in_desc = 1; + } elsif ($in_desc) { + if ($line =~ /^(?:\s{4,}|$)/) { + $line =~ s/^\s{4}//; + $desc .= $line; + $desc .= "\n"; + } else { + $verbose_messages{$type} = trim($desc); + $type = ''; + $desc = ''; + $in_desc = 0; + } + } + } + + if ($desc ne '') { + $verbose_messages{$type} = trim($desc); + } + close($docs); +} + # Perl's Getopt::Long allows options to take optional arguments after a space. # Prevent --color by itself from consuming other arguments foreach (@ARGV) { @@ -208,6 +292,7 @@ sub list_types { GetOptions( 'q|quiet+' => \$quiet, + 'v|verbose!' => \$verbose, 'tree!' => \$tree, 'signoff!' => \$chk_signoff, 'patch!' => \$chk_patch, @@ -235,7 +320,7 @@ sub list_types { 'debug=s' => \%debug, 'test-only=s' => \$tst_only, 'codespell!' => \$codespell, - 'codespellfile=s' => \$codespellfile, + 'codespellfile=s' => \$user_codespellfile, 'typedefsfile=s' => \$typedefsfile, 'color=s' => \$color, 'no-color' => \$color, #keep old behaviors of -nocolor @@ -243,17 +328,54 @@ sub list_types { 'kconfig-prefix=s' => \${CONFIG_}, 'h|help' => \$help, 'version' => \$help -) or help(1); +) or $help = 2; + +if ($user_codespellfile) { + # Use the user provided codespell file unconditionally + $codespellfile = $user_codespellfile; +} elsif (!(-f $codespellfile)) { + # If /usr/share/codespell/dictionary.txt is not present, try to find it + # under codespell's install directory: /data/dictionary.txt + if (($codespell || $help) && which("python3") ne "") { + my $python_codespell_dict = << "EOF"; + +import os.path as op +import codespell_lib +codespell_dir = op.dirname(codespell_lib.__file__) +codespell_file = op.join(codespell_dir, 'data', 'dictionary.txt') +print(codespell_file, end='') +EOF + + my $codespell_dict = `python3 -c "$python_codespell_dict" 2> /dev/null`; + $codespellfile = $codespell_dict if (-f $codespell_dict); + } +} + +# $help is 1 if either -h, --help or --version is passed as option - exitcode: 0 +# $help is 2 if invalid option is passed - exitcode: 1 +help($help - 1) if ($help); -help(0) if ($help); +die "$P: --git cannot be used with --file or --fix\n" if ($git && ($file || $fix)); +die "$P: --verbose cannot be used with --terse\n" if ($verbose && $terse); +if ($color =~ /^[01]$/) { + $color = !$color; +} elsif ($color =~ /^always$/i) { + $color = 1; +} elsif ($color =~ /^never$/i) { + $color = 0; +} elsif ($color =~ /^auto$/i) { + $color = (-t STDOUT); +} else { + die "$P: Invalid color mode: $color\n"; +} + +load_docs() if ($verbose); list_types(0) if ($list_types); $fix = 1 if ($fix_inplace); $check_orig = $check; -die "$P: --git cannot be used with --file or --fix\n" if ($git && ($file || $fix)); - my $exit = 0; my $perl_version_ok = 1; @@ -268,18 +390,6 @@ sub list_types { push(@ARGV, '-'); } -if ($color =~ /^[01]$/) { - $color = !$color; -} elsif ($color =~ /^always$/i) { - $color = 1; -} elsif ($color =~ /^never$/i) { - $color = 0; -} elsif ($color =~ /^auto$/i) { - $color = (-t STDOUT); -} else { - die "$P: Invalid color mode: $color\n"; -} - # skip TAB size 1 to avoid additional checks on $tabsize - 1 die "$P: Invalid TAB size: $tabsize\n" if ($tabsize < 2); @@ -402,10 +512,12 @@ sub hash_show_words { __ro_after_init| __kprobes| $InitAttribute| + __aligned\s*\(.*\)| ____cacheline_aligned| ____cacheline_aligned_in_smp| ____cacheline_internodealigned_in_smp| - __weak + __weak| + __alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) }x; our $Modifier; our $Inline = qr{inline|__always_inline|noinline|__inline|__inline__}; @@ -417,7 +529,7 @@ sub hash_show_words { our $Hex = qr{(?i)0x[0-9a-f]+$Int_type?}; our $Int = qr{[0-9]+$Int_type?}; our $Octal = qr{0[0-7]+$Int_type?}; -our $String = qr{"[X\t]*"}; +our $String = qr{(?:\b[Lu])?"[X\t]*"}; our $Float_hex = qr{(?i)0x[0-9a-f]+p-?[0-9]+[fl]?}; our $Float_dec = qr{(?i)(?:[0-9]+\.[0-9]*|[0-9]*\.[0-9]+)(?:e-?[0-9]+)?[fl]?}; our $Float_int = qr{(?i)[0-9]+e-?[0-9]+[fl]?}; @@ -467,10 +579,14 @@ sub hash_show_words { (?:__)?(?:u|s|be|le)(?:8|16|32|64)| atomic_t )}; +our $typeStdioTypedefs = qr{(?x: + FILE +)}; our $typeTypedefs = qr{(?x: $typeC99Typedefs\b| $typeOtherOSTypedefs\b| - $typeKernelTypedefs\b + $typeKernelTypedefs\b| + $typeStdioTypedefs\b )}; our $zero_initializer = qr{(?:(?:0[xX])?0+$Int_type?|NULL|false)\b}; @@ -507,6 +623,22 @@ sub hash_show_words { Cc: )}; +our @link_tags = qw(Link Closes); + +#Create a search and print patterns for all these strings to be used directly below +our $link_tags_search = ""; +our $link_tags_print = ""; +foreach my $entry (@link_tags) { + if ($link_tags_search ne "") { + $link_tags_search .= '|'; + $link_tags_print .= ' or '; + } + $entry .= ':'; + $link_tags_search .= $entry; + $link_tags_print .= "'$entry'"; +} +$link_tags_search = "(?:${link_tags_search})"; + our $tracing_logging_tags = qr{(?xi: [=-]*> | <[=-]* | @@ -589,6 +721,17 @@ sub find_standard_signature { return ""; } +our $obsolete_archives = qr{(?xi: + \Qfreedesktop.org/archives/dri-devel\E | + \Qlists.infradead.org\E | + \Qlkml.org\E | + \Qmail-archive.com\E | + \Qmailman.alsa-project.org/pipermail\E | + \Qmarc.info\E | + \Qozlabs.org/pipermail\E | + \Qspinics.net\E +)}; + our @typeListMisordered = ( qr{char\s+(?:un)?signed}, qr{int\s+(?:(?:un)?signed\s+)?short\s}, @@ -698,6 +841,10 @@ sub find_standard_signature { "rcu_barrier_sched" => "rcu_barrier", "get_state_synchronize_sched" => "get_state_synchronize_rcu", "cond_synchronize_sched" => "cond_synchronize_rcu", + "kmap" => "kmap_local_page", + "kunmap" => "kunmap_local", + "kmap_atomic" => "kmap_local_page", + "kunmap_atomic" => "kunmap_local", ); #Create a search pattern for all these strings to speed up a loop below @@ -933,7 +1080,8 @@ sub build_types { our $declaration_macros = qr{(?x: (?:$Storage\s+)?(?:[A-Z_][A-Z0-9]*_){0,2}(?:DEFINE|DECLARE)(?:_[A-Z0-9]+){1,6}\s*\(| (?:$Storage\s+)?[HLP]?LIST_HEAD\s*\(| - (?:SKCIPHER_REQUEST|SHASH_DESC|AHASH_REQUEST)_ON_STACK\s*\( + (?:SKCIPHER_REQUEST|SHASH_DESC|AHASH_REQUEST)_ON_STACK\s*\(| + (?:$Storage\s+)?(?:XA_STATE|XA_STATE_ORDER)\s*\( )}; our %allow_repeated_words = ( @@ -1000,10 +1148,10 @@ sub is_maintained_obsolete { sub is_SPDX_License_valid { my ($license) = @_; - return 1 if (!$tree || which("python") eq "" || !(-e "$root/scripts/spdxcheck.py") || !(-e "$gitroot")); + return 1 if (!$tree || which("python3") eq "" || !(-x "$root/scripts/spdxcheck.py") || !(-e "$gitroot")); my $root_path = abs_path($root); - my $status = `cd "$root_path"; echo "$license" | python scripts/spdxcheck.py -`; + my $status = `cd "$root_path"; echo "$license" | scripts/spdxcheck.py -`; return 0 if ($status ne ""); return 1; } @@ -1097,7 +1245,8 @@ sub git_commit_info { # git log --format='%H %s' -1 $line | # echo "commit $(cut -c 1-12,41-)" # done - } elsif ($lines[0] =~ /^fatal: ambiguous argument '$commit': unknown revision or path not in the working tree\./) { + } elsif ($lines[0] =~ /^fatal: ambiguous argument '$commit': unknown revision or path not in the working tree\./ || + $lines[0] =~ /^fatal: bad object $commit/) { $id = undef; } else { $id = substr($lines[0], 0, 12); @@ -2209,7 +2358,16 @@ sub report { splice(@lines, 1, 1); $output = join("\n", @lines); } - $output = (split('\n', $output))[0] . "\n" if ($terse); + + if ($terse) { + $output = (split('\n', $output))[0] . "\n"; + } + + if ($verbose && exists($verbose_messages{$type}) && + !exists($verbose_emitted{$type})) { + $output .= $verbose_messages{$type} . "\n\n"; + $verbose_emitted{$type} = 1; + } push(our @report, $output); @@ -2494,6 +2652,8 @@ sub process { my $reported_maintainer_file = 1; # ODP has no MAINTAINERS file my $non_utf8_charset = 0; + my $last_git_commit_id_linenr = -1; + my $last_blank_line = 0; my $last_coalesced_string_linenr = -1; @@ -2816,10 +2976,10 @@ sub process { my ($email_name, $email_comment, $email_address, $comment1) = parse_email($ctx); my ($author_name, $author_comment, $author_address, $comment2) = parse_email($author); - if ($email_address eq $author_address && $email_name eq $author_name) { + if (lc $email_address eq lc $author_address && $email_name eq $author_name) { $author_sob = $ctx; $authorsignoff = 2; - } elsif ($email_address eq $author_address) { + } elsif (lc $email_address eq lc $author_address) { $author_sob = $ctx; $authorsignoff = 3; } elsif ($email_name eq $author_name) { @@ -3003,17 +3163,73 @@ sub process { if ($sign_off =~ /^co-developed-by:$/i) { if ($email eq $author) { WARN("BAD_SIGN_OFF", - "Co-developed-by: should not be used to attribute nominal patch author '$author'\n" . "$here\n" . $rawline); + "Co-developed-by: should not be used to attribute nominal patch author '$author'\n" . $herecurr); } if (!defined $lines[$linenr]) { WARN("BAD_SIGN_OFF", - "Co-developed-by: must be immediately followed by Signed-off-by:\n" . "$here\n" . $rawline); - } elsif ($rawlines[$linenr] !~ /^\s*signed-off-by:\s*(.*)/i) { + "Co-developed-by: must be immediately followed by Signed-off-by:\n" . $herecurr); + } elsif ($rawlines[$linenr] !~ /^signed-off-by:\s*(.*)/i) { WARN("BAD_SIGN_OFF", - "Co-developed-by: must be immediately followed by Signed-off-by:\n" . "$here\n" . $rawline . "\n" .$rawlines[$linenr]); + "Co-developed-by: must be immediately followed by Signed-off-by:\n" . $herecurr . $rawlines[$linenr] . "\n"); } elsif ($1 ne $email) { WARN("BAD_SIGN_OFF", - "Co-developed-by and Signed-off-by: name/email do not match \n" . "$here\n" . $rawline . "\n" .$rawlines[$linenr]); + "Co-developed-by and Signed-off-by: name/email do not match\n" . $herecurr . $rawlines[$linenr] . "\n"); + } + } + +# check if Reported-by: is followed by a Closes: tag + if ($sign_off =~ /^reported(?:|-and-tested)-by:$/i) { + if (!defined $lines[$linenr]) { + WARN("BAD_REPORTED_BY_LINK", + "Reported-by: should be immediately followed by Closes: with a URL to the report\n" . $herecurr . "\n"); + } elsif ($rawlines[$linenr] !~ /^closes:\s*/i) { + WARN("BAD_REPORTED_BY_LINK", + "Reported-by: should be immediately followed by Closes: with a URL to the report\n" . $herecurr . $rawlines[$linenr] . "\n"); + } + } + } + + +# Check Fixes: styles is correct + if (!$in_header_lines && + $line =~ /^\s*fixes:?\s*(?:commit\s*)?[0-9a-f]{5,}\b/i) { + my $orig_commit = ""; + my $id = "0123456789ab"; + my $title = "commit title"; + my $tag_case = 1; + my $tag_space = 1; + my $id_length = 1; + my $id_case = 1; + my $title_has_quotes = 0; + + if ($line =~ /(\s*fixes:?)\s+([0-9a-f]{5,})\s+($balanced_parens)/i) { + my $tag = $1; + $orig_commit = $2; + $title = $3; + + $tag_case = 0 if $tag eq "Fixes:"; + $tag_space = 0 if ($line =~ /^fixes:? [0-9a-f]{5,} ($balanced_parens)/i); + + $id_length = 0 if ($orig_commit =~ /^[0-9a-f]{12}$/i); + $id_case = 0 if ($orig_commit !~ /[A-F]/); + + # Always strip leading/trailing parens then double quotes if existing + $title = substr($title, 1, -1); + if ($title =~ /^".*"$/) { + $title = substr($title, 1, -1); + $title_has_quotes = 1; + } + } + + my ($cid, $ctitle) = git_commit_info($orig_commit, $id, + $title); + + if ($ctitle ne $title || $tag_case || $tag_space || + $id_length || $id_case || !$title_has_quotes) { + if (WARN("BAD_FIXES_TAG", + "Please use correct Fixes: style 'Fixes: <12 chars of sha1> (\"\")' - ie: 'Fixes: $cid (\"$ctitle\")'\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] = "Fixes: $cid (\"$ctitle\")"; } } } @@ -3051,13 +3267,13 @@ sub process { length($line) > 75 && !($line =~ /^\s*[a-zA-Z0-9_\/\.]+\s+\|\s+\d+/ || # file delta changes - $line =~ /^\s*(?:[\w\.\-]+\/)++[\w\.\-]+:/ || + $line =~ /^\s*(?:[\w\.\-\+]*\/)++[\w\.\-\+]+:/ || # filename then : - $line =~ /^\s*(?:Fixes:|Link:|$signature_tags)/i || - # A Fixes: or Link: line or signature tag line + $line =~ /^\s*(?:Fixes:|$link_tags_search|$signature_tags)/i || + # A Fixes:, link or signature tag line $commit_log_possible_stack_dump)) { WARN("COMMIT_LOG_LONG_LINE", - "Possible unwrapped commit description (prefer a maximum 75 chars per line)\n" . $herecurr); + "Prefer a maximum 75 chars per line (possible unwrapped commit description?)\n" . $herecurr); $commit_log_long_line = 1; } @@ -3067,6 +3283,29 @@ sub process { $commit_log_possible_stack_dump = 0; } +# Check for odd tags before a URI/URL + if ($in_commit_log && + $line =~ /^\s*(\w+:)\s*http/ && $1 !~ /^$link_tags_search$/) { + if ($1 =~ /^v(?:ersion)?\d+/i) { + WARN("COMMIT_LOG_VERSIONING", + "Patch version information should be after the --- line\n" . $herecurr); + } else { + WARN("COMMIT_LOG_USE_LINK", + "Unknown link reference '$1', use $link_tags_print instead\n" . $herecurr); + } + } + +# Check for misuse of the link tags + if ($in_commit_log && + $line =~ /^\s*(\w+:)\s*(\S+)/) { + my $tag = $1; + my $value = $2; + if ($tag =~ /^$link_tags_search$/ && $value !~ m{^https?://}) { + WARN("COMMIT_LOG_WRONG_LINK", + "'$tag' should be followed by a public http(s) link\n" . $herecurr); + } + } + # Check for lines starting with a # if ($in_commit_log && $line =~ /^#/) { if (WARN("COMMIT_COMMENT_SYMBOL", @@ -3077,10 +3316,20 @@ sub process { } # Check for git id commit length and improperly formed commit descriptions - if ($in_commit_log && !$commit_log_possible_stack_dump && +# A correctly formed commit description is: +# commit <SHA-1 hash length 12+ chars> ("Complete commit subject") +# with the commit subject '("' prefix and '")' suffix +# This is a fairly compilicated block as it tests for what appears to be +# bare SHA-1 hash with minimum length of 5. It also avoids several types of +# possible SHA-1 matches. +# A commit match can span multiple lines so this block attempts to find a +# complete typical commit on a maximum of 3 lines + if ($perl_version_ok && + $in_commit_log && !$commit_log_possible_stack_dump && $line !~ /^\s*(?:Link|Patchwork|http|https|BugLink|base-commit):/i && $line !~ /^This reverts commit [0-9a-f]{7,40}/ && - ($line =~ /\bcommit\s+[0-9a-f]{5,}\b/i || + (($line =~ /\bcommit\s+[0-9a-f]{5,}\b/i || + ($line =~ /\bcommit\s*$/i && defined($rawlines[$linenr]) && $rawlines[$linenr] =~ /^\s*[0-9a-f]{5,}\b/i)) || ($line =~ /(?:\s|^)[0-9a-f]{12,40}(?:[\s"'\(\[]|$)/i && $line !~ /[\<\[][0-9a-f]{12,40}[\>\]]/i && $line !~ /\bfixes:\s*[0-9a-f]{12,40}/i))) { @@ -3090,49 +3339,62 @@ sub process { my $long = 0; my $case = 1; my $space = 1; - my $hasdesc = 0; - my $hasparens = 0; my $id = '0123456789ab'; my $orig_desc = "commit description"; my $description = ""; + my $herectx = $herecurr; + my $has_parens = 0; + my $has_quotes = 0; + + my $input = $line; + if ($line =~ /(?:\bcommit\s+[0-9a-f]{5,}|\bcommit\s*$)/i) { + for (my $n = 0; $n < 2; $n++) { + if ($input =~ /\bcommit\s+[0-9a-f]{5,}\s*($balanced_parens)/i) { + $orig_desc = $1; + $has_parens = 1; + # Always strip leading/trailing parens then double quotes if existing + $orig_desc = substr($orig_desc, 1, -1); + if ($orig_desc =~ /^".*"$/) { + $orig_desc = substr($orig_desc, 1, -1); + $has_quotes = 1; + } + last; + } + last if ($#lines < $linenr + $n); + $input .= " " . trim($rawlines[$linenr + $n]); + $herectx .= "$rawlines[$linenr + $n]\n"; + } + $herectx = $herecurr if (!$has_parens); + } - if ($line =~ /\b(c)ommit\s+([0-9a-f]{5,})\b/i) { + if ($input =~ /\b(c)ommit\s+([0-9a-f]{5,})\b/i) { $init_char = $1; $orig_commit = lc($2); - } elsif ($line =~ /\b([0-9a-f]{12,40})\b/i) { + $short = 0 if ($input =~ /\bcommit\s+[0-9a-f]{12,40}/i); + $long = 1 if ($input =~ /\bcommit\s+[0-9a-f]{41,}/i); + $space = 0 if ($input =~ /\bcommit [0-9a-f]/i); + $case = 0 if ($input =~ /\b[Cc]ommit\s+[0-9a-f]{5,40}[^A-F]/); + } elsif ($input =~ /\b([0-9a-f]{12,40})\b/i) { $orig_commit = lc($1); } - $short = 0 if ($line =~ /\bcommit\s+[0-9a-f]{12,40}/i); - $long = 1 if ($line =~ /\bcommit\s+[0-9a-f]{41,}/i); - $space = 0 if ($line =~ /\bcommit [0-9a-f]/i); - $case = 0 if ($line =~ /\b[Cc]ommit\s+[0-9a-f]{5,40}[^A-F]/); - if ($line =~ /\bcommit\s+[0-9a-f]{5,}\s+\("([^"]+)"\)/i) { - $orig_desc = $1; - $hasparens = 1; - } elsif ($line =~ /\bcommit\s+[0-9a-f]{5,}\s*$/i && - defined $rawlines[$linenr] && - $rawlines[$linenr] =~ /^\s*\("([^"]+)"\)/) { - $orig_desc = $1; - $hasparens = 1; - } elsif ($line =~ /\bcommit\s+[0-9a-f]{5,}\s+\("[^"]+$/i && - defined $rawlines[$linenr] && - $rawlines[$linenr] =~ /^\s*[^"]+"\)/) { - $line =~ /\bcommit\s+[0-9a-f]{5,}\s+\("([^"]+)$/i; - $orig_desc = $1; - $rawlines[$linenr] =~ /^\s*([^"]+)"\)/; - $orig_desc .= " " . $1; - $hasparens = 1; - } - ($id, $description) = git_commit_info($orig_commit, $id, $orig_desc); if (defined($id) && - ($short || $long || $space || $case || ($orig_desc ne $description) || !$hasparens)) { + ($short || $long || $space || $case || ($orig_desc ne $description) || !$has_quotes) && + $last_git_commit_id_linenr != $linenr - 1) { ERROR("GIT_COMMIT_ID", - "Please use git commit description style 'commit <12+ chars of sha1> (\"<title line>\")' - ie: '${init_char}ommit $id (\"$description\")'\n" . $herecurr); + "Please use git commit description style 'commit <12+ chars of sha1> (\"<title line>\")' - ie: '${init_char}ommit $id (\"$description\")'\n" . $herectx); } + #don't report the next line if this line ends in commit and the sha1 hash is the next line + $last_git_commit_id_linenr = $linenr if ($line =~ /\bcommit\s*$/i); + } + +# Check for mailing list archives other than lore.kernel.org + if ($rawline =~ m{http.*\b$obsolete_archives}) { + WARN("PREFER_LORE_ARCHIVE", + "Use lore.kernel.org archive links when possible - see https://lore.kernel.org/lists.html\n" . $herecurr); } # Check for added, moved or deleted files @@ -3152,7 +3414,7 @@ sub process { ($line =~ /^new file mode\s*\d+\s*$/) && ($realfile =~ m@^Documentation/devicetree/bindings/.*\.txt$@)) { WARN("DT_SCHEMA_BINDING_PATCH", - "DT bindings should be in DT schema format. See: Documentation/devicetree/writing-schema.rst\n"); + "DT bindings should be in DT schema format. See: Documentation/devicetree/bindings/writing-schema.rst\n"); } # Check for wrappage within a valid hunk of the file @@ -3341,47 +3603,47 @@ sub process { # Kconfig supports named choices), so use a word boundary # (\b) rather than a whitespace character (\s) $line =~ /^\+\s*(?:config|menuconfig|choice)\b/) { - my $length = 0; - my $cnt = $realcnt; - my $ln = $linenr + 1; - my $f; - my $is_start = 0; - my $is_end = 0; - for (; $cnt > 0 && defined $lines[$ln - 1]; $ln++) { - $f = $lines[$ln - 1]; - $cnt-- if ($lines[$ln - 1] !~ /^-/); - $is_end = $lines[$ln - 1] =~ /^\+/; + my $ln = $linenr; + my $needs_help = 0; + my $has_help = 0; + my $help_length = 0; + while (defined $lines[$ln]) { + my $f = $lines[$ln++]; next if ($f =~ /^-/); - last if (!$file && $f =~ /^\@\@/); + last if ($f !~ /^[\+ ]/); # !patch context - if ($lines[$ln - 1] =~ /^\+\s*(?:bool|tristate|prompt)\s*["']/) { - $is_start = 1; - } elsif ($lines[$ln - 1] =~ /^\+\s*(?:---)?help(?:---)?$/) { - $length = -1; + if ($f =~ /^\+\s*(?:bool|tristate|prompt)\s*["']/) { + $needs_help = 1; + next; + } + if ($f =~ /^\+\s*help\s*$/) { + $has_help = 1; + next; } - $f =~ s/^.//; - $f =~ s/#.*//; - $f =~ s/^\s+//; - next if ($f =~ /^$/); + $f =~ s/^.//; # strip patch context [+ ] + $f =~ s/#.*//; # strip # directives + $f =~ s/^\s+//; # strip leading blanks + next if ($f =~ /^$/); # skip blank lines + # At the end of this Kconfig block: # This only checks context lines in the patch # and so hopefully shouldn't trigger false # positives, even though some of these are # common words in help texts - if ($f =~ /^\s*(?:config|menuconfig|choice|endchoice| - if|endif|menu|endmenu|source)\b/x) { - $is_end = 1; + if ($f =~ /^(?:config|menuconfig|choice|endchoice| + if|endif|menu|endmenu|source)\b/x) { last; } - $length++; + $help_length++ if ($has_help); } - if ($is_start && $is_end && $length < $min_conf_desc_length) { + if ($needs_help && + $help_length < $min_conf_desc_length) { + my $stat_real = get_stat_real($linenr, $ln - 1); WARN("CONFIG_DESCRIPTION", - "please write a paragraph that describes the config symbol fully\n" . $herecurr); + "please write a help paragraph that fully describes the config symbol\n" . "$here\n$stat_real\n"); } - #print "is_start<$is_start> is_end<$is_end> length<$length>\n"; } # check MAINTAINERS entries @@ -3477,7 +3739,7 @@ sub process { my $comment = ""; if ($realfile =~ /\.(h|s|S)$/) { $comment = '/*'; - } elsif ($realfile =~ /\.(c|dts|dtsi)$/) { + } elsif ($realfile =~ /\.(c|rs|dts|dtsi)$/) { $comment = '//'; } elsif (($checklicenseline == 2) || $realfile =~ /\.(sh|pl|py|awk|tc|yaml)$/) { $comment = '#'; @@ -3504,7 +3766,7 @@ sub process { "'$spdx_license' is not supported in LICENSES/...\n" . $herecurr); } if ($realfile =~ m@^Documentation/devicetree/bindings/@ && - not $spdx_license =~ /GPL-2\.0.*BSD-2-Clause/) { + $spdx_license !~ /GPL-2\.0(?:-only)? OR BSD-2-Clause/) { my $msg_level = \&WARN; $msg_level = \&CHK if ($file); if (&{$msg_level}("SPDX_LICENSE_TAG", @@ -3514,18 +3776,23 @@ sub process { $fixed[$fixlinenr] =~ s/SPDX-License-Identifier: .*/SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)/; } } + if ($realfile =~ m@^include/dt-bindings/@ && + $spdx_license !~ /GPL-2\.0(?:-only)? OR \S+/) { + WARN("SPDX_LICENSE_TAG", + "DT binding headers should be licensed (GPL-2.0-only OR .*)\n" . $herecurr); + } } } } # check for embedded filenames - if ($rawline =~ /^\+.*\Q$realfile\E/) { + if ($rawline =~ /^\+.*\b\Q$realfile\E\b/) { WARN("EMBEDDED_FILENAME", "It's generally not useful to have the filename in the file\n" . $herecurr); } # check we are in a valid source file if not then ignore this hunk - next if ($realfile !~ /\.(h|c|s|S|sh|dtsi|dts)$/); + next if ($realfile !~ /\.(h|c|rs|s|S|sh|dtsi|dts)$/); # check for using SPDX-License-Identifier on the wrong line number if ($realline != $checklicenseline && @@ -3612,7 +3879,7 @@ sub process { if ($realfile =~ /\.S$/ && $line =~ /^\+\s*(?:[A-Z]+_)?SYM_[A-Z]+_(?:START|END)(?:_[A-Z_]+)?\s*\(\s*\.L/) { WARN("AVOID_L_PREFIX", - "Avoid using '.L' prefixed local symbol names for denoting a range of code via 'SYM_*_START/END' annotations; see Documentation/asm-annotations.rst\n" . $herecurr); + "Avoid using '.L' prefixed local symbol names for denoting a range of code via 'SYM_*_START/END' annotations; see Documentation/core-api/asm-annotations.rst\n" . $herecurr); } # check we are in a valid source file C or perl if not then ignore this hunk @@ -3788,7 +4055,7 @@ sub process { if ($prevline =~ /^[\+ ]};?\s*$/ && $line =~ /^\+/ && !($line =~ /^\+\s*$/ || - $line =~ /^\+\s*EXPORT_SYMBOL/ || + $line =~ /^\+\s*(?:EXPORT_SYMBOL|early_param|ALLOW_ERROR_INJECTION)/ || $line =~ /^\+\s*MODULE_/i || $line =~ /^\+\s*\#\s*(?:end|elif|else)/ || $line =~ /^\+[a-z_]*init/ || @@ -4335,6 +4602,7 @@ sub process { # XXX(foo); # EXPORT_SYMBOL(something_foo); my $name = $1; + $name =~ s/^\s*($Ident).*/$1/; if ($stat =~ /^(?:.\s*}\s*\n)?.([A-Z_]+)\s*\(\s*($Ident)/ && $name =~ /^${Ident}_$2/) { #print "FOO C name<$name>\n"; @@ -4555,12 +4823,12 @@ sub process { } } -# avoid BUG() or BUG_ON() - if ($line =~ /\b(?:BUG|BUG_ON)\b/) { +# do not use BUG() or variants + if ($line =~ /\b(?!AA_|BUILD_|DCCP_|IDA_|KVM_|RWLOCK_|snd_|SPIN_)(?:[a-zA-Z_]*_)?BUG(?:_ON)?(?:_[A-Z_]+)?\s*\(/) { my $msg_level = \&WARN; $msg_level = \&CHK if ($file); &{$msg_level}("AVOID_BUG", - "Avoid crashing the kernel - try using WARN_ON & recovery code rather than BUG() or BUG_ON()\n" . $herecurr); + "Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants\n" . $herecurr); } # avoid LINUX_VERSION_CODE @@ -4781,7 +5049,7 @@ sub process { if|for|while|switch|return|case| volatile|__volatile__| __attribute__|format|__extension__| - asm|__asm__)$/x) + asm|__asm__|scoped_guard)$/x) { # cpp #define statements have non-optional spaces, ie # if there is a space between the name and the open @@ -5268,9 +5536,13 @@ sub process { } } -#goto labels aren't indented, allow a single space however - if ($line=~/^.\s+[A-Za-z\d_]+:(?![0-9]+)/ and - !($line=~/^. [A-Za-z\d_]+:/) and !($line=~/^.\s+default:/)) { +# check that goto labels aren't indented (allow a single space indentation) +# and ignore bitfield definitions like foo:1 +# Strictly, labels can have whitespace after the identifier and before the : +# but this is not allowed here as many ?: uses would appear to be labels + if ($sline =~ /^.\s+[A-Za-z_][A-Za-z\d_]*:(?!\s*\d+)/ && + $sline !~ /^. [A-Za-z\d_][A-Za-z\d_]*:/ && + $sline !~ /^.\s+default:/) { if (WARN("INDENTED_LABEL", "labels should not be indented\n" . $herecurr) && $fix) { @@ -5365,7 +5637,7 @@ sub process { # Return of what appears to be an errno should normally be negative if ($sline =~ /\breturn(?:\s*\(+\s*|\s+)(E[A-Z]+)(?:\s*\)+\s*|\s*)[;:,]/) { my $name = $1; - if ($name ne 'EOF' && $name ne 'ERROR') { + if ($name ne 'EOF' && $name ne 'ERROR' && $name !~ /^EPOLL/) { WARN("USE_NEGATIVE_ERRNO", "return of an errno should typically be negative (ie: return -$1)\n" . $herecurr); } @@ -5408,6 +5680,7 @@ sub process { defined($stat) && defined($cond) && $line =~ /\b(?:if|while|for)\s*\(/ && $line !~ /^.\s*#/) { my ($s, $c) = ($stat, $cond); + my $fixed_assign_in_if = 0; if ($c =~ /\bif\s*\(.*[^<>!=]=[^=].*/s) { if (ERROR("ASSIGN_IN_IF", @@ -5432,6 +5705,7 @@ sub process { $newline .= ')'; $newline .= " {" if (defined($brace)); fix_insert_line($fixlinenr + 1, $newline); + $fixed_assign_in_if = 1; } } } @@ -5455,8 +5729,20 @@ sub process { $stat_real = "[...]\n$stat_real"; } - ERROR("TRAILING_STATEMENTS", - "trailing statements should be on next line\n" . $herecurr . $stat_real); + if (ERROR("TRAILING_STATEMENTS", + "trailing statements should be on next line\n" . $herecurr . $stat_real) && + !$fixed_assign_in_if && + $cond_lines == 0 && + $fix && $perl_version_ok && + $fixed[$fixlinenr] =~ /^\+(\s*)((?:if|while|for)\s*$balanced_parens)\s*(.*)$/) { + my $indent = $1; + my $test = $2; + my $rest = rtrim($4); + if ($rest =~ /;$/) { + $fixed[$fixlinenr] = "\+$indent$test"; + fix_insert_line($fixlinenr + 1, "$indent\t$rest"); + } + } } } @@ -5569,12 +5855,14 @@ sub process { $var !~ /\bSCN[diux]16/ && $var !~ /\bSCN[diux]32/ && $var !~ /\bSCN[diux]64/ && +#Ignore ETHTOOL_LINK_MODE_<foo> variants + $var !~ /^ETHTOOL_LINK_MODE_/ && #Ignore SI style variants like nS, mV and dB #(ie: max_uV, regulator_min_uA_show, RANGE_mA_VALUE) $var !~ /^(?:[a-z0-9_]*|[A-Z0-9_]*)?_?[a-z][A-Z](?:_[a-z0-9_]+|_[A-Z0-9_]+)?$/ && #Ignore some three character SI units explicitly, like MiB and KHz $var !~ /^(?:[a-z_]*?)_?(?:[KMGT]iB|[KMGT]?Hz)(?:_[a-z_]+)?$/) { - while ($var =~ m{($Ident)}g) { + while ($var =~ m{\b($Ident)}g) { my $word = $1; next if ($word !~ /[A-Z][a-z]|[a-z][A-Z]/); if ($check) { @@ -5704,6 +5992,7 @@ sub process { $dstat !~ /$exceptions/ && $dstat !~ /^\.$Ident\s*=/ && # .foo = $dstat !~ /^(?:\#\s*$Ident|\#\s*$Constant)\s*$/ && # stringification #foo + $dstat !~ /^case\b/ && # case ... $dstat !~ /^do\s*$Constant\s*while\s*$Constant;?$/ && # do {...} while (...); // do {...} while (...) $dstat !~ /^while\s*$Constant\s*$Constant\s*$/ && # while (...) {...} $dstat !~ /^for\s*$Constant$/ && # for (...) @@ -5747,7 +6036,7 @@ sub process { next if ($arg =~ /\.\.\./); next if ($arg =~ /^type$/i); my $tmp_stmt = $define_stmt; - $tmp_stmt =~ s/\b(sizeof|typeof|__typeof__|__builtin\w+|typecheck\s*\(\s*$Type\s*,|\#+)\s*\(*\s*$arg\s*\)*\b//g; + $tmp_stmt =~ s/\b(__must_be_array|offsetof|sizeof|sizeof_field|__stringify|typeof|__typeof__|__builtin\w+|typecheck\s*\(\s*$Type\s*,|\#+)\s*\(*\s*$arg\s*\)*\b//g; $tmp_stmt =~ s/\#+\s*$arg\b//g; $tmp_stmt =~ s/\b$arg\s*\#\#//g; my $use_cnt = () = $tmp_stmt =~ /\b$arg\b/g; @@ -5776,6 +6065,9 @@ sub process { # check for line continuations outside of #defines, preprocessor #, and asm + } elsif ($realfile =~ m@/vmlinux.lds.h$@) { + $line =~ s/(\w+)/$maybe_linker_symbol{$1}++/ge; + #print "REAL: $realfile\nln: $line\nkeys:", sort keys %maybe_linker_symbol; } else { if ($prevline !~ /^..*\\$/ && $line !~ /^\+\s*\#.*\\$/ && # preprocessor @@ -6046,7 +6338,8 @@ sub process { } # concatenated string without spaces between elements - if ($line =~ /$String[A-Za-z0-9_]/ || $line =~ /[A-Za-z0-9_]$String/) { + if ($line =~ /$String[A-Z_]/ || + ($line =~ /([A-Za-z0-9_]+)$String/ && $1 !~ /^[Lu]$/)) { if (CHK("CONCATENATED_STRING", "Concatenated strings should use spaces between elements\n" . $herecurr) && $fix) { @@ -6059,7 +6352,7 @@ sub process { } # uncoalesced string fragments - if ($line =~ /$String\s*"/) { + if ($line =~ /$String\s*[Lu]?"/) { if (WARN("STRING_FRAGMENTS", "Consecutive strings are generally better as a single string\n" . $herecurr) && $fix) { @@ -6617,9 +6910,11 @@ sub process { $specifier = $1; $extension = $2; $qualifier = $3; - if ($extension !~ /[SsBKRraEehMmIiUDdgVCbGNOxtf]/ || + if ($extension !~ /[4SsBKRraEehMmIiUDdgVCbGNOxtf]/ || ($extension eq "f" && - defined $qualifier && $qualifier !~ /^w/)) { + defined $qualifier && $qualifier !~ /^w/) || + ($extension eq "4" && + defined $qualifier && $qualifier !~ /^cc/)) { $bad_specifier = $specifier; last; } @@ -6633,15 +6928,19 @@ sub process { } if ($bad_specifier ne "") { my $stat_real = get_stat_real($linenr, $lc); + my $msg_level = \&WARN; my $ext_type = "Invalid"; my $use = ""; if ($bad_specifier =~ /p[Ff]/) { $use = " - use %pS instead"; $use =~ s/pS/ps/ if ($bad_specifier =~ /pf/); + } elsif ($bad_specifier =~ /pA/) { + $use = " - '%pA' is only intended to be used from Rust code"; + $msg_level = \&ERROR; } - WARN("VSPRINTF_POINTER_EXTENSION", - "$ext_type vsprintf pointer extension '$bad_specifier'$use\n" . "$here\n$stat_real\n"); + &{$msg_level}("VSPRINTF_POINTER_EXTENSION", + "$ext_type vsprintf pointer extension '$bad_specifier'$use\n" . "$here\n$stat_real\n"); } } } @@ -6706,12 +7005,43 @@ sub process { # } # } +# strcpy uses that should likely be strscpy + if ($line =~ /\bstrcpy\s*\(/) { + WARN("STRCPY", + "Prefer strscpy over strcpy - see: https://github.com/KSPP/linux/issues/88\n" . $herecurr); + } + # strlcpy uses that should likely be strscpy if ($line =~ /\bstrlcpy\s*\(/) { WARN("STRLCPY", - "Prefer strscpy over strlcpy - see: https://lore.kernel.org/r/CAHk-=wgfRnXz0W3D37d01q3JFkr_i_uTL=V6A6G1oUZcprmknw\@mail.gmail.com/\n" . $herecurr); + "Prefer strscpy over strlcpy - see: https://github.com/KSPP/linux/issues/89\n" . $herecurr); + } + +# strncpy uses that should likely be strscpy or strscpy_pad + if ($line =~ /\bstrncpy\s*\(/) { + WARN("STRNCPY", + "Prefer strscpy, strscpy_pad, or __nonstring over strncpy - see: https://github.com/KSPP/linux/issues/90\n" . $herecurr); + } + +# ethtool_sprintf uses that should likely be ethtool_puts + if ($line =~ /\bethtool_sprintf\s*\(\s*$FuncArg\s*,\s*$FuncArg\s*\)/) { + if (WARN("PREFER_ETHTOOL_PUTS", + "Prefer ethtool_puts over ethtool_sprintf with only two arguments\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/\bethtool_sprintf\s*\(\s*($FuncArg)\s*,\s*($FuncArg)/ethtool_puts($1, $7)/; + } } + # use $rawline because $line loses %s via sanitization and thus we can't match against it. + if ($rawline =~ /\bethtool_sprintf\s*\(\s*$FuncArg\s*,\s*\"\%s\"\s*,\s*$FuncArg\s*\)/) { + if (WARN("PREFER_ETHTOOL_PUTS", + "Prefer ethtool_puts over ethtool_sprintf with standalone \"%s\" specifier\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/\bethtool_sprintf\s*\(\s*($FuncArg)\s*,\s*"\%s"\s*,\s*($FuncArg)/ethtool_puts($1, $7)/; + } + } + + # typecasts on min/max could be min_t/max_t if ($perl_version_ok && defined $stat && @@ -6816,6 +7146,21 @@ sub process { "arguments for function declarations should follow identifier\n" . $herecurr); } + } elsif ($realfile =~ /\.c$/ && defined $stat && + $stat =~ /^\+extern struct\s+(\w+)\s+(\w+)\[\];/) + { + my ($st_type, $st_name) = ($1, $2); + + for my $s (keys %maybe_linker_symbol) { + #print "Linker symbol? $st_name : $s\n"; + goto LIKELY_LINKER_SYMBOL + if $st_name =~ /$s/; + } + WARN("AVOID_EXTERNS", + "found a file-scoped extern type:$st_type name:$st_name in .c file\n" + . "is this a linker symbol ?\n" . $herecurr); + LIKELY_LINKER_SYMBOL: + } elsif ($realfile =~ /\.c$/ && defined $stat && $stat =~ /^.\s*extern\s+/) { @@ -6884,14 +7229,16 @@ sub process { "Prefer $3(sizeof(*$1)...) over $3($4...)\n" . $herecurr); } -# check for k[mz]alloc with multiplies that could be kmalloc_array/kcalloc +# check for (kv|k)[mz]alloc with multiplies that could be kmalloc_array/kvmalloc_array/kvcalloc/kcalloc if ($perl_version_ok && defined $stat && - $stat =~ /^\+\s*($Lval)\s*\=\s*(?:$balanced_parens)?\s*(k[mz]alloc)\s*\(\s*($FuncArg)\s*\*\s*($FuncArg)\s*,/) { + $stat =~ /^\+\s*($Lval)\s*\=\s*(?:$balanced_parens)?\s*((?:kv|k)[mz]alloc)\s*\(\s*($FuncArg)\s*\*\s*($FuncArg)\s*,/) { my $oldfunc = $3; my $a1 = $4; my $a2 = $10; my $newfunc = "kmalloc_array"; + $newfunc = "kvmalloc_array" if ($oldfunc eq "kvmalloc"); + $newfunc = "kvcalloc" if ($oldfunc eq "kvzalloc"); $newfunc = "kcalloc" if ($oldfunc eq "kzalloc"); my $r1 = $a1; my $r2 = $a2; @@ -6908,7 +7255,7 @@ sub process { "Prefer $newfunc over $oldfunc with multiply\n" . $herectx) && $cnt == 1 && $fix) { - $fixed[$fixlinenr] =~ s/\b($Lval)\s*\=\s*(?:$balanced_parens)?\s*(k[mz]alloc)\s*\(\s*($FuncArg)\s*\*\s*($FuncArg)/$1 . ' = ' . "$newfunc(" . trim($r1) . ', ' . trim($r2)/e; + $fixed[$fixlinenr] =~ s/\b($Lval)\s*\=\s*(?:$balanced_parens)?\s*((?:kv|k)[mz]alloc)\s*\(\s*($FuncArg)\s*\*\s*($FuncArg)/$1 . ' = ' . "$newfunc(" . trim($r1) . ', ' . trim($r2)/e; } } } @@ -6922,7 +7269,7 @@ sub process { } # check for alloc argument mismatch - if ($line =~ /\b(kcalloc|kmalloc_array)\s*\(\s*sizeof\b/) { + if ($line =~ /\b((?:devm_)?((?:k|kv)?(calloc|malloc_array)(?:_node)?))\s*\(\s*sizeof\b/) { WARN("ALLOC_ARRAY_ARGS", "$1 uses number as first arg, sizeof is generally wrong\n" . $herecurr); } @@ -7114,6 +7461,27 @@ sub process { "Using $1 should generally have parentheses around the comparison\n" . $herecurr); } +# return sysfs_emit(foo, fmt, ...) fmt without newline + if ($line =~ /\breturn\s+sysfs_emit\s*\(\s*$FuncArg\s*,\s*($String)/ && + substr($rawline, $-[6], $+[6] - $-[6]) !~ /\\n"$/) { + my $offset = $+[6] - 1; + if (WARN("SYSFS_EMIT", + "return sysfs_emit(...) formats should include a terminating newline\n" . $herecurr) && + $fix) { + substr($fixed[$fixlinenr], $offset, 0) = '\\n'; + } + } + +# check for array definition/declarations that should use flexible arrays instead + if ($sline =~ /^[\+ ]\s*\}(?:\s*__packed)?\s*;\s*$/ && + $prevline =~ /^\+\s*(?:\}(?:\s*__packed\s*)?|$Type)\s*$Ident\s*\[\s*(0|1)\s*\]\s*;\s*$/) { + if (ERROR("FLEXIBLE_ARRAY", + "Use C99 flexible arrays - see https://docs.kernel.org/process/deprecated.html#zero-length-and-one-element-arrays\n" . $hereprev) && + $1 == '0' && $fix) { + $fixed[$fixlinenr - 1] =~ s/\[\s*0\s*\]/[]/; + } + } + # nested likely/unlikely calls if ($line =~ /\b(?:(?:un)?likely)\s*\(\s*!?\s*(IS_ERR(?:_OR_NULL|_VALUE)?|WARN)/) { WARN("LIKELY_MISUSE", @@ -7131,6 +7499,30 @@ sub process { } } +# Complain about RCU Tasks Trace used outside of BPF (and of course, RCU). + our $rcu_trace_funcs = qr{(?x: + rcu_read_lock_trace | + rcu_read_lock_trace_held | + rcu_read_unlock_trace | + call_rcu_tasks_trace | + synchronize_rcu_tasks_trace | + rcu_barrier_tasks_trace | + rcu_request_urgent_qs_task + )}; + our $rcu_trace_paths = qr{(?x: + kernel/bpf/ | + include/linux/bpf | + net/bpf/ | + kernel/rcu/ | + include/linux/rcu + )}; + if ($line =~ /\b($rcu_trace_funcs)\s*\(/) { + if ($realfile !~ m{^$rcu_trace_paths}) { + WARN("RCU_TASKS_TRACE", + "use of RCU tasks trace is incorrect outside BPF or core RCU code\n" . $herecurr); + } + } + # check for lockdep_set_novalidate_class if ($line =~ /^.\s*lockdep_set_novalidate_class\s*\(/ || $line =~ /__lockdep_no_validate__\s*\)/ ) { @@ -7272,6 +7664,13 @@ sub process { WARN("MODULE_LICENSE", "unknown module license " . $extracted_string . "\n" . $herecurr); } + if (!$file && $extracted_string eq '"GPL v2"') { + if (WARN("MODULE_LICENSE", + "Prefer \"GPL\" over \"GPL v2\" - see commit bf7fbeeae6db (\"module: Cure the MODULE_LICENSE \"GPL\" vs. \"GPL v2\" bogosity\")\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/\bMODULE_LICENSE\s*\(\s*"GPL v2"\s*\)/MODULE_LICENSE("GPL")/; + } + } } # check for sysctl duplicate constants @@ -7414,4 +7813,4 @@ sub process { } } return $clean; -} +} \ No newline at end of file From f2cd3083ed12719a8a20d148f14440ab80ed06bb Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Tue, 13 Feb 2024 13:25:55 +0200 Subject: [PATCH 106/147] checkpatch: update spelling file Update spelling.txt to the latest version from https://github.com/torvalds/linux repo (commit c664e16bb1ba1c8cf1d7ecf3df5fd83bbb8ac15a). Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> --- scripts/spelling.txt | 121 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 118 insertions(+), 3 deletions(-) diff --git a/scripts/spelling.txt b/scripts/spelling.txt index 69b5537a7..a22e955c6 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -23,8 +23,10 @@ absoulte||absolute acccess||access acceess||access accelaration||acceleration +accelearion||acceleration acceleratoin||acceleration accelleration||acceleration +accelrometer||accelerometer accesing||accessing accesnt||accent accessable||accessible @@ -58,11 +60,13 @@ acording||according activete||activate actived||activated actualy||actually +actvie||active acumulating||accumulating acumulative||accumulative acumulator||accumulator acutally||actually adapater||adapter +adderted||asserted addional||additional additionaly||additionally additonal||additional @@ -120,6 +124,7 @@ alue||value ambigious||ambiguous ambigous||ambiguous amoung||among +amount of times||number of times amout||amount amplifer||amplifier amplifyer||amplifier @@ -133,6 +138,7 @@ anniversery||anniversary annoucement||announcement anomolies||anomalies anomoly||anomaly +anonynous||anonymous anway||anyway aplication||application appearence||appearance @@ -151,6 +157,7 @@ aquired||acquired aquisition||acquisition arbitary||arbitrary architechture||architecture +archtecture||architecture arguement||argument arguements||arguments arithmatic||arithmetic @@ -178,7 +185,9 @@ assum||assume assumtpion||assumption asuming||assuming asycronous||asynchronous +asychronous||asynchronous asynchnous||asynchronous +asynchronus||asynchronous asynchromous||asynchronous asymetric||asymmetric asymmeric||asymmetric @@ -230,6 +239,7 @@ baloons||balloons bandwith||bandwidth banlance||balance batery||battery +battey||battery beacuse||because becasue||because becomming||becoming @@ -241,6 +251,7 @@ beter||better betweeen||between bianries||binaries bitmast||bitmask +bitwiedh||bitwidth boardcast||broadcast borad||board boundry||boundary @@ -249,6 +260,7 @@ brigde||bridge broadcase||broadcast broadcat||broadcast bufer||buffer +bufferred||buffered bufufer||buffer cacluated||calculated caculate||calculate @@ -257,6 +269,7 @@ cadidate||candidate cahces||caches calender||calendar calescing||coalescing +calibraiton||calibration calle||called callibration||calibration callled||called @@ -265,7 +278,12 @@ calucate||calculate calulate||calculate cancelation||cancellation cancle||cancel +cant||can't +cant'||can't canot||cannot +cann't||can't +cannnot||cannot +capabiity||capability capabilites||capabilities capabilties||capabilities capabilty||capability @@ -273,9 +291,11 @@ capabitilies||capabilities capablity||capability capatibilities||capabilities capapbilities||capabilities +captuer||capture caputure||capture carefuly||carefully cariage||carriage +casued||caused catagory||category cehck||check challange||challenge @@ -302,6 +322,7 @@ chiled||child chked||checked chnage||change chnages||changes +chnange||change chnnel||channel choosen||chosen chouse||chose @@ -323,11 +344,13 @@ comminucation||communication commited||committed commiting||committing committ||commit +commmand||command commnunication||communication commoditiy||commodity comsume||consume comsumer||consumer comsuming||consuming +comaptible||compatible compability||compatibility compaibility||compatibility comparsion||comparison @@ -348,13 +371,16 @@ compoment||component comppatible||compatible compres||compress compresion||compression +compresser||compressor comression||compression +comsumed||consumed comunicate||communicate comunication||communication conbination||combination conditionaly||conditionally conditon||condition condtion||condition +condtional||conditional conected||connected conector||connector configration||configuration @@ -385,6 +411,7 @@ continious||continuous continous||continuous continously||continuously continueing||continuing +contiuous||continuous contraints||constraints contruct||construct contol||control @@ -407,7 +434,9 @@ cotrol||control cound||could couter||counter coutner||counter +creationg||creating cryptocraphic||cryptographic +cummulative||cumulative cunter||counter curently||currently cylic||cyclic @@ -429,6 +458,7 @@ defferred||deferred definate||definite definately||definitely definiation||definition +definiton||definition defintion||definition defintions||definitions defualt||default @@ -442,6 +472,7 @@ delare||declare delares||declares delaring||declaring delemiter||delimiter +delibrately||deliberately delievered||delivered demodualtor||demodulator demension||dimension @@ -470,7 +501,9 @@ destorys||destroys destroied||destroyed detabase||database deteced||detected +detecion||detection detectt||detect +detroyed||destroyed develope||develop developement||development developped||developed @@ -490,6 +523,7 @@ diferent||different differrence||difference diffrent||different differenciate||differentiate +diffreential||differential diffrentiate||differentiate difinition||definition digial||digital @@ -497,10 +531,12 @@ dimention||dimension dimesions||dimensions diconnected||disconnected disabed||disabled +disasembler||disassembler disble||disable disgest||digest disired||desired dispalying||displaying +dissable||disable diplay||display directon||direction direcly||directly @@ -524,6 +560,7 @@ dissconect||disconnect distiction||distinction divisable||divisible divsiors||divisors +dsiabled||disabled docuentation||documentation documantation||documentation documentaion||documentation @@ -591,10 +628,12 @@ evalute||evaluate evalutes||evaluates evalution||evaluation excecutable||executable +excceed||exceed exceded||exceeded exceds||exceeds exceeed||exceed excellant||excellent +exchnage||exchange execeeded||exceeded execeeds||exceeds exeed||exceed @@ -603,8 +642,11 @@ exeuction||execution existance||existence existant||existent exixt||exist +exsits||exists exlcude||exclude +exlcuding||excluding exlcusive||exclusive +exlusive||exclusive exmaple||example expecially||especially experies||expires @@ -642,17 +684,20 @@ feauture||feature feautures||features fetaure||feature fetaures||features +fetcing||fetching fileystem||filesystem fimrware||firmware fimware||firmware firmare||firmware firmaware||firmware +firtly||firstly firware||firmware firwmare||firmware finanize||finalize findn||find finilizes||finalizes finsih||finish +fliter||filter flusing||flushing folloing||following followign||following @@ -670,6 +715,7 @@ frequence||frequency frequncy||frequency frequancy||frequency frome||from +fronend||frontend fucntion||function fuction||function fuctions||functions @@ -693,6 +739,8 @@ generiously||generously genereate||generate genereted||generated genric||generic +gerenal||general +geting||getting globel||global grabing||grabbing grahical||graphical @@ -711,9 +759,11 @@ hanled||handled happend||happened hardare||hardware harware||hardware +hardward||hardware havind||having heirarchically||hierarchically heirarchy||hierarchy +heirachy||hierarchy helpfull||helpful hearbeat||heartbeat heterogenous||heterogeneous @@ -726,6 +776,7 @@ howver||however hsould||should hypervior||hypervisor hypter||hyper +idel||idle identidier||identifier iligal||illegal illigal||illegal @@ -754,6 +805,7 @@ implmentation||implementation implmenting||implementing incative||inactive incomming||incoming +incompaitiblity||incompatibility incompatabilities||incompatibilities incompatable||incompatible incompatble||incompatible @@ -811,6 +863,7 @@ integrety||integrity integrey||integrity intendet||intended intented||intended +interal||internal interanl||internal interchangable||interchangeable interferring||interfering @@ -862,6 +915,7 @@ iteraions||iterations iternations||iterations itertation||iteration itslef||itself +ivalid||invalid jave||java jeffies||jiffies jumpimng||jumping @@ -916,9 +970,11 @@ matchs||matches mathimatical||mathematical mathimatic||mathematic mathimatics||mathematics +maxmium||maximum maximium||maximum maxium||maximum mechamism||mechanism +mechanim||mechanism meetign||meeting memeory||memory memmber||member @@ -927,6 +983,7 @@ memroy||memory ment||meant mergable||mergeable mesage||message +mesages||messages messags||messages messgaes||messages messsage||message @@ -935,9 +992,12 @@ metdata||metadata micropone||microphone microprocesspr||microprocessor migrateable||migratable +millenium||millennium milliseonds||milliseconds +minimim||minimum minium||minimum minimam||minimum +minimun||minimum miniumum||minimum minumum||minimum misalinged||misaligned @@ -956,6 +1016,7 @@ mmnemonic||mnemonic mnay||many modfiy||modify modifer||modifier +modul||module modulues||modules momery||memory memomry||memory @@ -965,8 +1026,9 @@ monochromo||monochrome monocrome||monochrome mopdule||module mroe||more -multipler||multiplier mulitplied||multiplied +muliple||multiple +multipler||multiplier multidimensionnal||multidimensional multipe||multiple multple||multiple @@ -989,6 +1051,7 @@ negotation||negotiation nerver||never nescessary||necessary nessessary||necessary +none existent||non-existent noticable||noticeable notication||notification notications||notifications @@ -997,7 +1060,9 @@ notifed||notified notity||notify nubmer||number numebr||number +numer||number numner||number +nunber||number obtaion||obtain obusing||abusing occassionally||occasionally @@ -1009,17 +1074,20 @@ occured||occurred occurence||occurrence occure||occurred occuring||occurring +ocurrence||occurrence offser||offset offet||offset offlaod||offload offloded||offloaded offseting||offsetting +oflload||offload omited||omitted omiting||omitting omitt||omit ommiting||omitting ommitted||omitted onself||oneself +onthe||on the ony||only openning||opening operatione||operation @@ -1037,6 +1105,7 @@ oustanding||outstanding overaall||overall overhread||overhead overlaping||overlapping +oveflow||overflow overflw||overflow overlfow||overflow overide||override @@ -1056,6 +1125,7 @@ pakage||package paket||packet pallette||palette paln||plan +palne||plane paramameters||parameters paramaters||parameters paramater||parameter @@ -1085,10 +1155,12 @@ perfomring||performing periperal||peripheral peripherial||peripheral permissons||permissions +permited||permitted peroid||period persistance||persistence persistant||persistent phoneticly||phonetically +plaform||platform plalform||platform platfoem||platform platfrom||platform @@ -1126,15 +1198,19 @@ preprare||prepare pressre||pressure presuambly||presumably previosuly||previously +previsously||previously primative||primitive princliple||principle priorty||priority +priting||printing privilaged||privileged privilage||privilege priviledge||privilege +priviledged||privileged priviledges||privileges privleges||privileges probaly||probably +probabalistic||probabilistic procceed||proceed proccesors||processors procesed||processed @@ -1154,6 +1230,7 @@ programable||programmable programers||programmers programm||program programms||programs +progres||progress progresss||progress prohibitted||prohibited prohibitting||prohibiting @@ -1182,6 +1259,7 @@ purgable||purgeable pwoer||power queing||queuing quering||querying +querrying||querying queus||queues randomally||randomly raoming||roaming @@ -1214,6 +1292,7 @@ refering||referring refernces||references refernnce||reference refrence||reference +regiser||register registed||registered registerd||registered registeration||registration @@ -1254,6 +1333,8 @@ reseting||resetting reseved||reserved reseverd||reserved resizeable||resizable +resonable||reasonable +resotre||restore resouce||resource resouces||resources resoures||resources @@ -1286,11 +1367,14 @@ routins||routines rquest||request runing||running runned||ran +runnnig||running runnning||running runtine||runtime sacrifying||sacrificing safly||safely safty||safety +satify||satisfy +satisifed||satisfied savable||saveable scaleing||scaling scaned||scanned @@ -1328,6 +1412,7 @@ servive||service setts||sets settting||setting shapshot||snapshot +shoft||shift shotdown||shutdown shoud||should shouldnt||shouldn't @@ -1341,14 +1426,18 @@ similiar||similar simlar||similar simliar||similar simpified||simplified +simultaneusly||simultaneously +simultanous||simultaneous singaled||signaled singal||signal singed||signed +slect||select sleeped||slept sliped||slipped softwade||software softwares||software soley||solely +soluation||solution souce||source speach||speech specfic||specific @@ -1380,6 +1469,7 @@ standart||standard standy||standby stardard||standard staticly||statically +statisitcs||statistics statuss||status stoped||stopped stoping||stopping @@ -1398,6 +1488,7 @@ submited||submitted submition||submission succeded||succeeded suceed||succeed +succesfuly||successfully succesfully||successfully succesful||successful successed||succeeded @@ -1413,6 +1504,7 @@ suported||supported suport||support supportet||supported suppored||supported +supporing||supporting supportin||supporting suppoted||supported suppported||supported @@ -1439,6 +1531,8 @@ syfs||sysfs symetric||symmetric synax||syntax synchonized||synchronized +sychronization||synchronization +sychronously||synchronously synchronuously||synchronously syncronize||synchronize syncronized||synchronized @@ -1448,22 +1542,27 @@ syste||system sytem||system sythesis||synthesis taht||that +tained||tainted +tarffic||traffic tansmit||transmit targetted||targeted targetting||targeting taskelt||tasklet teh||the +temeprature||temperature temorary||temporary temproarily||temporarily temperture||temperature -thead||thread +theads||threads therfore||therefore thier||their threds||threads threee||three threshhold||threshold thresold||threshold +throtting||throttling throught||through +tansition||transition trackling||tracking troughput||throughput trys||tries @@ -1476,6 +1575,7 @@ timout||timeout tmis||this toogle||toggle torerable||tolerable +torlence||tolerance traget||target traking||tracking tramsmitted||transmitted @@ -1484,12 +1584,14 @@ tranasction||transaction tranceiver||transceiver tranfer||transfer tranmission||transmission +tranport||transport transcevier||transceiver transciever||transceiver transferd||transferred transfered||transferred transfering||transferring transision||transition +transistioned||transitioned transmittd||transmitted transormed||transformed trasfer||transfer @@ -1503,13 +1605,16 @@ tunning||tuning ture||true tyep||type udpate||update +updtes||updates uesd||used +unknwon||unknown uknown||unknown usccess||success uncommited||uncommitted uncompatible||incompatible unconditionaly||unconditionally undeflow||underflow +undelying||underlying underun||underrun unecessary||unnecessary unexecpted||unexpected @@ -1521,6 +1626,7 @@ unexpexted||unexpected unfortunatelly||unfortunately unifiy||unify uniterrupted||uninterrupted +uninterruptable||uninterruptible unintialized||uninitialized unitialized||uninitialized unkmown||unknown @@ -1537,13 +1643,17 @@ unneccessary||unnecessary unnecesary||unnecessary unneedingly||unnecessarily unnsupported||unsupported +unuspported||unsupported unmached||unmatched unprecise||imprecise +unpriviledged||unprivileged +unpriviliged||unprivileged unregester||unregister unresgister||unregister unrgesiter||unregister unsinged||unsigned unstabel||unstable +unsolicted||unsolicited unsolicitied||unsolicited unsuccessfull||unsuccessful unsuported||unsupported @@ -1553,6 +1663,8 @@ unuseful||useless unvalid||invalid upate||update upsupported||unsupported +upto||up to +useable||usable usefule||useful usefull||useful usege||usage @@ -1574,9 +1686,11 @@ varient||variant vaule||value verbse||verbose veify||verify +verfication||verification veriosn||version verisons||versions verison||version +veritical||vertical verson||version vicefersa||vice-versa virtal||virtual @@ -1586,6 +1700,7 @@ visiters||visitors vitual||virtual vunerable||vulnerable wakeus||wakeups +was't||wasn't wathdog||watchdog wating||waiting wiat||wait @@ -1596,6 +1711,7 @@ whenver||whenever wheter||whether whe||when wierd||weird +wihout||without wiil||will wirte||write withing||within @@ -1607,7 +1723,6 @@ writting||writing wtih||with zombe||zombie zomebie||zombie - # # ODP additions # From 30be76579945dc53aabd97f9fdedc38cc9e3f030 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Tue, 13 Feb 2024 14:59:50 +0200 Subject: [PATCH 107/147] checkpatch: look for codespell dictionary automatically Current checkpatch script version will find codespell dictionary file automatically if user doesn't set the file path manually. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> --- .checkpatch.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/.checkpatch.conf b/.checkpatch.conf index 16235c39b..abb822996 100644 --- a/.checkpatch.conf +++ b/.checkpatch.conf @@ -19,4 +19,3 @@ --ignore=EMAIL_SUBJECT --ignore=C99_COMMENT_TOLERANCE --codespell ---codespellfile=/usr/share/codespell/dictionary.txt From f09b6a59156e55115620a9ea514f4159d80c3731 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Fri, 7 Jan 2022 14:14:02 +0200 Subject: [PATCH 108/147] test: sched_perf: add signal handler Add signal handler for SIGINT for closing the application. Also, run the application indefinitely if the number of events to schedule is set to zero. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Janne Peltonen <janne.peltonen@nokia.com> --- test/performance/odp_sched_perf.c | 39 +++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c index 70c91092e..47f703338 100644 --- a/test/performance/odp_sched_perf.c +++ b/test/performance/odp_sched_perf.c @@ -13,6 +13,11 @@ * @cond _ODP_HIDE_FROM_DOXYGEN_ */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE /* Needed for sigaction */ +#endif + +#include <signal.h> #include <stdio.h> #include <string.h> #include <stdint.h> @@ -99,6 +104,7 @@ typedef struct test_global_t { test_stat_t stat[ODP_THREAD_COUNT_MAX]; thread_arg_t thread_arg[ODP_THREAD_COUNT_MAX]; odp_atomic_u32_t num_worker; + odp_atomic_u32_t exit_threads; } test_global_t; @@ -107,6 +113,23 @@ typedef struct { odp_atomic_u64_t count; } queue_context_t; +static test_global_t *test_globals; + +static void sig_handler(int signum ODP_UNUSED) +{ + odp_atomic_store_u32(&test_globals->exit_threads, 1); +} + +static int setup_sig_handler(void) +{ + struct sigaction action = { .sa_handler = sig_handler }; + + if (sigemptyset(&action.sa_mask) || sigaction(SIGINT, &action, NULL)) + return -1; + + return 0; +} + static void print_usage(void) { printf("\n" @@ -122,7 +145,8 @@ static void print_usage(void) " the queues are default (or lowest) priority. Default: 0.\n" " -d, --num_dummy Number of empty queues. Default: 0.\n" " -e, --num_event Number of events per queue. Default: 100.\n" - " -s, --num_sched Number of events to schedule per thread. Default: 100 000.\n" + " -s, --num_sched Number of events to schedule per thread. If zero, the application runs\n" + " until SIGINT is received. Default: 100 000.\n" " -g, --num_group Number of schedule groups. Round robins threads and queues into groups.\n" " -1: SCHED_GROUP_WORKER\n" " 0: SCHED_GROUP_ALL (default)\n" @@ -981,6 +1005,7 @@ static int test_sched(void *arg) const uint32_t uarea_rw = test_options->uarea_rw; odp_pool_type_t pool_type = test_options->pool_type; int touch_ctx = ctx_rd_words || ctx_rw_words; + odp_atomic_u32_t *exit_threads = &global->exit_threads; uint32_t ctx_offset = 0; uint32_t sched_retries = 0; uint64_t data_sum = 0; @@ -1040,7 +1065,10 @@ static int test_sched(void *arg) c1 = odp_cpu_cycles(); last_retry_ts = t1; - for (rounds = 0; events < num_sched; rounds++) { + for (rounds = 0; odp_likely(!odp_atomic_load_u32(exit_threads)); rounds++) { + if (odp_unlikely(num_sched && events >= num_sched)) + break; + num = odp_schedule_multi(&queue, ODP_SCHED_NO_WAIT, ev, max_burst); @@ -1399,10 +1427,17 @@ int main(int argc, char **argv) ODPH_ERR("Error: SHM alloc failed\n"); exit(EXIT_FAILURE); } + test_globals = global; memset(global, 0, sizeof(test_global_t)); global->pool = ODP_POOL_INVALID; global->ctx_shm = ODP_SHM_INVALID; + odp_atomic_init_u32(&global->exit_threads, 0); + + if (setup_sig_handler()) { + ODPH_ERR("Error: signal handler setup failed\n"); + exit(EXIT_FAILURE); + } if (parse_options(argc, argv, &global->test_options)) return -1; From 218d405feade246be95ae0a7aa4ab628821b4afb Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Mon, 19 Feb 2024 15:26:22 +0200 Subject: [PATCH 109/147] build: re-enable parallel build Enabling parallel build in all directories outweighs the benefits from being able to run distcheck in parallel. Running 'make check' with -j option is not expected to work anymore after this commit. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> --- example/packet/Makefile.am | 2 -- example/timer/Makefile.am | 2 -- helper/test/Makefile.am | 2 -- platform/linux-generic/test/example/ipsec_api/Makefile.am | 2 -- platform/linux-generic/test/example/ipsec_crypto/Makefile.am | 2 -- scripts/ci/distcheck.sh | 2 +- test/performance/Makefile.am | 2 -- 7 files changed, 1 insertion(+), 13 deletions(-) diff --git a/example/packet/Makefile.am b/example/packet/Makefile.am index cf33b2ef7..5e4d9f5ea 100644 --- a/example/packet/Makefile.am +++ b/example/packet/Makefile.am @@ -35,5 +35,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: diff --git a/example/timer/Makefile.am b/example/timer/Makefile.am index da85be233..66cf0edfb 100644 --- a/example/timer/Makefile.am +++ b/example/timer/Makefile.am @@ -16,5 +16,3 @@ TESTS = odp_timer_accuracy_run.sh \ endif EXTRA_DIST = odp_timer_accuracy_run.sh - -.NOTPARALLEL: diff --git a/helper/test/Makefile.am b/helper/test/Makefile.am index 80aac1083..9cf48d7d9 100644 --- a/helper/test/Makefile.am +++ b/helper/test/Makefile.am @@ -67,5 +67,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: diff --git a/platform/linux-generic/test/example/ipsec_api/Makefile.am b/platform/linux-generic/test/example/ipsec_api/Makefile.am index 101c97cdf..2535ad466 100644 --- a/platform/linux-generic/test/example/ipsec_api/Makefile.am +++ b/platform/linux-generic/test/example/ipsec_api/Makefile.am @@ -19,5 +19,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: diff --git a/platform/linux-generic/test/example/ipsec_crypto/Makefile.am b/platform/linux-generic/test/example/ipsec_crypto/Makefile.am index 101c97cdf..2535ad466 100644 --- a/platform/linux-generic/test/example/ipsec_crypto/Makefile.am +++ b/platform/linux-generic/test/example/ipsec_crypto/Makefile.am @@ -19,5 +19,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: diff --git a/scripts/ci/distcheck.sh b/scripts/ci/distcheck.sh index 6cf49f8a7..9d45536f4 100755 --- a/scripts/ci/distcheck.sh +++ b/scripts/ci/distcheck.sh @@ -16,4 +16,4 @@ export CI="true" # Additional configure flags for distcheck export DISTCHECK_CONFIGURE_FLAGS="${CONF}" -make -j $(nproc) distcheck +make distcheck diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am index bc239980d..356e98a2d 100644 --- a/test/performance/Makefile.am +++ b/test/performance/Makefile.am @@ -123,5 +123,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: From a44b31aa643794b33f6e650def8f4c37c5db50e9 Mon Sep 17 00:00:00 2001 From: Janne Peltonen <janne.peltonen@nokia.com> Date: Tue, 20 Feb 2024 10:57:25 +0200 Subject: [PATCH 110/147] test: performance: scheduling: fix calloc argument order Fix the order of arguments to calloc() to avoid a compiler warning. Signed-off-by: Janne Peltonen <janne.peltonen@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> --- test/performance/odp_scheduling.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/performance/odp_scheduling.c b/test/performance/odp_scheduling.c index 15a1506a5..c9f3eb89f 100644 --- a/test/performance/odp_scheduling.c +++ b/test/performance/odp_scheduling.c @@ -864,7 +864,7 @@ int main(int argc, char *argv[]) printf("first CPU: %i\n", odp_cpumask_first(&cpumask)); printf("cpu mask: %s\n", cpumaskstr); - thread_tbl = calloc(sizeof(odph_thread_t), num_workers); + thread_tbl = calloc(num_workers, sizeof(odph_thread_t)); if (!thread_tbl) { ODPH_ERR("no memory for thread_tbl\n"); return -1; From ccc0142f39e1add563afa2e5ccee67b19f39d19f Mon Sep 17 00:00:00 2001 From: Janne Peltonen <janne.peltonen@nokia.com> Date: Tue, 20 Feb 2024 10:57:26 +0200 Subject: [PATCH 111/147] linux-gen: shm: fix maybe-uninitialized warning Silence a false warning about a possibly uninitialized variable. Signed-off-by: Janne Peltonen <janne.peltonen@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> --- platform/linux-generic/odp_ishmpool.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform/linux-generic/odp_ishmpool.c b/platform/linux-generic/odp_ishmpool.c index 9b6340d7e..89ec10695 100644 --- a/platform/linux-generic/odp_ishmpool.c +++ b/platform/linux-generic/odp_ishmpool.c @@ -135,8 +135,8 @@ static inline uintptr_t get_bblock_nr(pool_t *bpool, void *addr) static inline void remove_from_list(pool_t *bpool, uint8_t order, bblock_t *bblock) { - bblock_t *curr; /* current bblock (when parsing list) */ - bblock_t *prev; /* previous bblock (when parsing list) */ + bblock_t *curr; + bblock_t *prev = NULL; curr = bpool->ctrl.free_heads[order]; if (!curr) From 45f617f440a0b99c71859f29b14d21fb7a1df1ba Mon Sep 17 00:00:00 2001 From: Petri Savolainen <petri.savolainen@nokia.com> Date: Fri, 27 Oct 2023 14:44:13 +0300 Subject: [PATCH 112/147] api: ml: add machine learning API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add new API module for Machine Learning offload. Signed-off-by: Petri Savolainen <petri.savolainen@nokia.com> Reviewed-by: Jere Leppänen <jere.leppanen@nokia.com> Reviewed-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> Reviewed-by: Srikanth Yalavarthi <syalavarthi@marvell.com> Reviewed-by: Shivah Shankar S <sshankarnara@marvell.com> Reviewed-by: Jerin Jacob <jerinj@marvell.com> Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com> --- include/Makefile.am | 13 + include/odp/api/abi-default/ml_types.h | 48 + include/odp/api/ml.h | 24 + include/odp/api/ml_quantize.h | 24 + include/odp/api/ml_types.h | 26 + include/odp/api/spec/ml.h | 699 ++++++++++++++ include/odp/api/spec/ml_quantize.h | 124 +++ include/odp/api/spec/ml_types.h | 873 ++++++++++++++++++ .../arch/arm32-linux/odp/api/abi/ml_types.h | 5 + .../arch/arm64-linux/odp/api/abi/ml_types.h | 5 + .../arch/default-linux/odp/api/abi/ml_types.h | 5 + .../arch/power64-linux/odp/api/abi/ml_types.h | 5 + .../arch/x86_32-linux/odp/api/abi/ml_types.h | 5 + .../arch/x86_64-linux/odp/api/abi/ml_types.h | 5 + include/odp_api.h | 2 + platform/linux-generic/Makefile.am | 1 + .../include-abi/odp/api/abi/ml_types.h | 45 + 17 files changed, 1909 insertions(+) create mode 100644 include/odp/api/abi-default/ml_types.h create mode 100644 include/odp/api/ml.h create mode 100644 include/odp/api/ml_quantize.h create mode 100644 include/odp/api/ml_types.h create mode 100644 include/odp/api/spec/ml.h create mode 100644 include/odp/api/spec/ml_quantize.h create mode 100644 include/odp/api/spec/ml_types.h create mode 100644 include/odp/arch/arm32-linux/odp/api/abi/ml_types.h create mode 100644 include/odp/arch/arm64-linux/odp/api/abi/ml_types.h create mode 100644 include/odp/arch/default-linux/odp/api/abi/ml_types.h create mode 100644 include/odp/arch/power64-linux/odp/api/abi/ml_types.h create mode 100644 include/odp/arch/x86_32-linux/odp/api/abi/ml_types.h create mode 100644 include/odp/arch/x86_64-linux/odp/api/abi/ml_types.h create mode 100644 platform/linux-generic/include-abi/odp/api/abi/ml_types.h diff --git a/include/Makefile.am b/include/Makefile.am index bc28a5cba..9e23c4fa3 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -34,6 +34,9 @@ odpapiinclude_HEADERS = \ odp/api/init.h \ odp/api/ipsec.h \ odp/api/ipsec_types.h \ + odp/api/ml.h \ + odp/api/ml_quantize.h \ + odp/api/ml_types.h \ odp/api/packet.h \ odp/api/packet_types.h \ odp/api/packet_flags.h \ @@ -101,6 +104,9 @@ odpapispecinclude_HEADERS = \ odp/api/spec/init.h \ odp/api/spec/ipsec.h \ odp/api/spec/ipsec_types.h \ + odp/api/spec/ml.h \ + odp/api/spec/ml_quantize.h \ + odp/api/spec/ml_types.h \ odp/api/spec/packet.h \ odp/api/spec/packet_types.h \ odp/api/spec/packet_flags.h \ @@ -169,6 +175,7 @@ odpapiabidefaultinclude_HEADERS = \ odp/api/abi-default/init.h \ odp/api/abi-default/ipsec.h \ odp/api/abi-default/ipsec_types.h \ + odp/api/abi-default/ml_types.h \ odp/api/abi-default/packet.h \ odp/api/abi-default/packet_types.h \ odp/api/abi-default/packet_flags.h \ @@ -232,6 +239,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm32-linux/odp/api/abi/init.h \ odp/arch/arm32-linux/odp/api/abi/ipsec.h \ odp/arch/arm32-linux/odp/api/abi/ipsec_types.h \ + odp/arch/arm32-linux/odp/api/abi/ml_types.h \ odp/arch/arm32-linux/odp/api/abi/packet.h \ odp/arch/arm32-linux/odp/api/abi/packet_types.h \ odp/arch/arm32-linux/odp/api/abi/packet_flags.h \ @@ -291,6 +299,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm64-linux/odp/api/abi/init.h \ odp/arch/arm64-linux/odp/api/abi/ipsec.h \ odp/arch/arm64-linux/odp/api/abi/ipsec_types.h \ + odp/arch/arm64-linux/odp/api/abi/ml_types.h \ odp/arch/arm64-linux/odp/api/abi/packet.h \ odp/arch/arm64-linux/odp/api/abi/packet_types.h \ odp/arch/arm64-linux/odp/api/abi/packet_flags.h \ @@ -350,6 +359,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/default-linux/odp/api/abi/init.h \ odp/arch/default-linux/odp/api/abi/ipsec.h \ odp/arch/default-linux/odp/api/abi/ipsec_types.h \ + odp/arch/default-linux/odp/api/abi/ml_types.h \ odp/arch/default-linux/odp/api/abi/packet.h \ odp/arch/default-linux/odp/api/abi/packet_types.h \ odp/arch/default-linux/odp/api/abi/packet_flags.h \ @@ -409,6 +419,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/power64-linux/odp/api/abi/init.h \ odp/arch/power64-linux/odp/api/abi/ipsec.h \ odp/arch/power64-linux/odp/api/abi/ipsec_types.h \ + odp/arch/power64-linux/odp/api/abi/ml_types.h \ odp/arch/power64-linux/odp/api/abi/packet.h \ odp/arch/power64-linux/odp/api/abi/packet_types.h \ odp/arch/power64-linux/odp/api/abi/packet_flags.h \ @@ -468,6 +479,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_32-linux/odp/api/abi/init.h \ odp/arch/x86_32-linux/odp/api/abi/ipsec.h \ odp/arch/x86_32-linux/odp/api/abi/ipsec_types.h \ + odp/arch/x86_32-linux/odp/api/abi/ml_types.h \ odp/arch/x86_32-linux/odp/api/abi/packet.h \ odp/arch/x86_32-linux/odp/api/abi/packet_types.h \ odp/arch/x86_32-linux/odp/api/abi/packet_flags.h \ @@ -527,6 +539,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_64-linux/odp/api/abi/init.h \ odp/arch/x86_64-linux/odp/api/abi/ipsec.h \ odp/arch/x86_64-linux/odp/api/abi/ipsec_types.h \ + odp/arch/x86_64-linux/odp/api/abi/ml_types.h \ odp/arch/x86_64-linux/odp/api/abi/packet.h \ odp/arch/x86_64-linux/odp/api/abi/packet_types.h \ odp/arch/x86_64-linux/odp/api/abi/packet_flags.h \ diff --git a/include/odp/api/abi-default/ml_types.h b/include/odp/api/abi-default/ml_types.h new file mode 100644 index 000000000..723beb1bc --- /dev/null +++ b/include/odp/api/abi-default/ml_types.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021-2023 Nokia + */ + +#ifndef ODP_ABI_ML_TYPES_H_ +#define ODP_ABI_ML_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/** @internal Dummy type for strong typing */ +typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_ml_model_t; + +/** @internal Dummy type for strong typing */ +typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_ml_compl_t; + +/** @internal Implementation specific ML parameters */ +struct _odp_ml_model_extra_param_t { + /** @internal Dummy field to avoid empty struct */ + char dummy; +}; + +/** @addtogroup odp_ml + * @{ + */ + +typedef _odp_abi_ml_model_t *odp_ml_model_t; +typedef _odp_abi_ml_compl_t *odp_ml_compl_t; +typedef struct _odp_ml_model_extra_param_t odp_ml_model_extra_param_t; + +#define ODP_ML_MODEL_INVALID ((odp_ml_model_t)0) +#define ODP_ML_COMPL_INVALID ((odp_ml_compl_t)0) + +#define ODP_ML_MODEL_NAME_LEN 64 +#define ODP_ML_MODEL_IO_NAME_LEN 64 +#define ODP_ML_SHAPE_NAME_LEN 16 +#define ODP_ML_EXTRA_STAT_NAME_LEN 64 + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/ml.h b/include/odp/api/ml.h new file mode 100644 index 000000000..55213dd52 --- /dev/null +++ b/include/odp/api/ml.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +/** + * @file + * + * ODP Machine Learning (ML) offload + */ + +#ifndef ODP_API_ML_H_ +#define ODP_API_ML_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/spec/ml.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/ml_quantize.h b/include/odp/api/ml_quantize.h new file mode 100644 index 000000000..cab2a3f22 --- /dev/null +++ b/include/odp/api/ml_quantize.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +/** + * @file + * + * ODP Machine Learning (ML) offload + */ + +#ifndef ODP_API_ML_QUANTIZE_H_ +#define ODP_API_ML_QUANTIZE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/spec/ml_quantize.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/ml_types.h b/include/odp/api/ml_types.h new file mode 100644 index 000000000..3c3f8a416 --- /dev/null +++ b/include/odp/api/ml_types.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +/** + * @file + * + * ODP Machine Learning (ML) types + */ + +#ifndef ODP_API_ML_TYPES_H_ +#define ODP_API_ML_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/abi/ml_types.h> + +#include <odp/api/spec/ml_types.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/spec/ml.h b/include/odp/api/spec/ml.h new file mode 100644 index 000000000..1a7710ab3 --- /dev/null +++ b/include/odp/api/spec/ml.h @@ -0,0 +1,699 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021-2023 Nokia + * Copyright (c) 2021 Marvell + */ + +/** + * @file + * + * ODP Machine Learning (ML) offload + */ + +#ifndef ODP_API_SPEC_ML_H_ +#define ODP_API_SPEC_ML_H_ +#include <odp/visibility_begin.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/event_types.h> +#include <odp/api/ml_types.h> +#include <odp/api/pool_types.h> +#include <odp/api/std_types.h> + +/** + * @addtogroup odp_ml + * Machine Learning (ML) offload + * @{ + * + * <b> ML API call sequence </b> + * + * Before ML offload can be used, it must be configured with an odp_ml_config() call. An application + * fills in configuration parameters to describe its intended ML offload usage. The parameter + * values may help ODP implementation to optimize memory and other HW resource usage. The + * application may use odp_ml_capability() to check ML capabilities both before and after the + * configuration step. + * + * After configuration, an ML model binary is passed with other parameters to odp_ml_model_create() + * call which checks and prepares the model for usage. The application may use odp_ml_model_info(), + * odp_ml_model_input_info() and odp_ml_model_output_info() calls to check model input and output + * data formats. Before the application can use the model for inference, it loads the model with + * an odp_ml_model_load() or odp_ml_model_load_start() call. After a successful load, the + * application may use e.g. odp_ml_run() or odp_ml_run_start() to perform inferences. + * + * When all previously started inference operations are complete, application uses + * odp_ml_model_unload() or odp_ml_model_unload_start() to unload the model. After a successful + * unload, the model may be destroyed with an odp_ml_model_destroy() call, or loaded again. + * + * <b> Completion identifiers </b> + * + * Completion identifiers are used with ML operations in asynchronous poll mode + * (#ODP_ML_COMPL_MODE_POLL). Application declares the maximum identifier value it will + * use per model with odp_ml_model_param_t.max_compl_id parameter. It cannot exceed + * the implementation capability of odp_ml_capability_t.max_compl_id. Completion identifier + * values are model specific. The same value can be used simultaneously with two different + * models, but cannot be used simultaneously in two ML operations on the same model. A value may be + * reused for the next ML operation (on the same model) only after the previous operation is + * complete. Within those limitations, application may use/reuse completion identifier + * values from 0 to max_compl_id range freely. + */ + +/** + * Query ML capabilities + * + * Outputs ML capabilities on success. Use this capability call to check ML offload implementation + * limits and its support of various ML API features. When ML offload is not available, + * odp_ml_capability_t.max_models is zero. + * + * @param[out] capa Pointer to a capability structure for output + * + * @retval 0 on success + * @retval <0 on failure + */ +int odp_ml_capability(odp_ml_capability_t *capa); + +/** + * Initialize ML configuration parameters + * + * Initialize an odp_ml_config_t to its default values. + * + * @param[out] config Configuration structure to be initialized + */ +void odp_ml_config_init(odp_ml_config_t *config); + +/** + * Configure ML offload + * + * Initializes and configures ML offload according to the configuration parameters. This function + * must be called only once and before any ML resources are created. Use odp_ml_capability() to + * query configuration capabilities and odp_ml_config_init() to initialize configuration + * parameters into their default values. + * + * @param config ML configuration parameters + * + * @retval 0 on success + * @retval <0 on failure + */ +int odp_ml_config(const odp_ml_config_t *config); + +/** + * Initialize ML model parameters + * + * Initialize an odp_ml_model_param_t to its default values. + * + * @param[out] param Parameters structure to be initialized + */ +void odp_ml_model_param_init(odp_ml_model_param_t *param); + +/** + * Create an ML model + * + * Creates an ML model according to the parameters. Use odp_ml_model_param_init() to initialize + * parameters into their default values. The use of model name is optional. Unique names are not + * required. However, odp_ml_model_lookup() returns only a single matching model. Maximum name + * string length is #ODP_ML_MODEL_NAME_LEN. + * + * The call copies the model binary and prepares it for loading. Application may free memory + * buffers pointed by the parameters when the call returns. Use odp_ml_model_load() + * or odp_ml_model_load_start() to load the model. A model is ready for inference runs + * (see e.g. odp_ml_run()) after it has been loaded successfully. + * + * When model metadata misses some details of model input / output data format, user can + * pass those with odp_ml_model_param_t.extra_info. Some ODP implementations may define + * implementation specific extra parameters (e.g. hints about HW resource usage), user can pass + * those with odp_ml_model_param_t.extra_param when applicable. + * + * @param name Name of the model, or NULL + * @param param ML model parameters + * + * @return ML model handle on success + * @retval ODP_ML_MODEL_INVALID on failure + */ +odp_ml_model_t odp_ml_model_create(const char *name, const odp_ml_model_param_t *param); + +/** + * Destroy an ML model + * + * Destroys a model and releases the resources reserved for it. If the model has been loaded, it + * must be unloaded (see odp_ml_model_unload() or odp_ml_model_unload_start()) prior to calling + * this function. + * + * @param model ML model to be destroyed + * + * @retval 0 on success + * @retval <0 on failure + */ +int odp_ml_model_destroy(odp_ml_model_t model); + +/** + * Find a model by name + * + * @param name Name of the model + * + * @return Handle of the first matching ML model + * @retval ODP_ML_MODEL_INVALID Model could not be found + */ +odp_ml_model_t odp_ml_model_lookup(const char *name); + +/** + * Load ML model + * + * Loads ML model in synchronous mode. When the call returns, load is complete and the model is + * ready for inference requests. A loaded model must be unloaded before it can be destroyed. + * The same model can be loaded and unloaded multiple times before being destroyed. + * + * The call optionally outputs results. Use NULL as 'result' pointer if results are not required. + * + * Application should not try to keep loaded more than configured number of models + * (odp_ml_config_t.max_models_loaded). Check ML capabilities for maximum number of loaded + * models (odp_ml_capability_t.max_models_loaded) and support of load completion modes + * (odp_ml_capability_t.load). + * + * @param model ML model to be loaded + * @param[out] result Pointer to load result structure for output, or NULL + * + * @retval 0 Model load was successful + * @retval <0 on failure + */ +int odp_ml_model_load(odp_ml_model_t model, odp_ml_load_result_t *result); + +/** + * Start asynchronous model load + * + * Otherwise like odp_ml_model_load(), but loads the model asynchronously. A successful call + * requests the model to be loaded, but does not wait for load completion. Completion parameters + * are used to select if load completion is reported in poll (#ODP_ML_COMPL_MODE_POLL) or event + * (#ODP_ML_COMPL_MODE_EVENT) mode. For poll mode, odp_ml_model_load_status() is called to check + * for completion. For event mode, ML offload sends the completion event into the completion + * queue when the load is complete. Use odp_ml_compl_param_init() to initialize completion + * parameters into their default values. + * + * @param model ML model to be loaded + * @param compl_param Completion parameters for load + * + * @retval 0 Model load started successfully + * @retval <0 on failure + */ +int odp_ml_model_load_start(odp_ml_model_t model, const odp_ml_compl_param_t *compl_param); + +/** + * Check model load completion + * + * Checks if a previously started model load (in #ODP_ML_COMPL_MODE_POLL mode) has completed. + * The completion identifier value from load operation completion parameters + * (odp_ml_compl_param_t.compl_id) is passed as a parameter. It specifies the load operation to be + * checked. Initially 0 is returned for all configured (but unused) completion identifier values. + * An odp_ml_model_load_start() call clears the previous completion status of an identifier, and + * this function returns 0 while the load is in progress. When the load is successfully + * complete, >0 is returned. If the load completed with a failure, -1 is returned. The same + * value is returned until the next start operation that reuses the identifier (with the same + * model). The completion identifier may be reused only after >0 or -1 is returned. + * + * Optionally, outputs more detailed operation results into odp_ml_load_result_t structure. + * Use NULL as 'result' pointer if these results are not required. + * + * @param model ML model being loaded + * @param compl_id Completion identifier that was used in load start + * @param[out] result Pointer to load result structure for output, or NULL + * + * @retval >0 Model load was successful + * @retval 0 Model load has not finished + * @retval -1 Model load failed + * @retval <-1 Failed to read completion status (e.g. bad handle) + */ +int odp_ml_model_load_status(odp_ml_model_t model, uint32_t compl_id, odp_ml_load_result_t *result); + +/** + * Unload ML model + * + * Unloads ML model in synchronous mode. All previously started inference operations must have been + * completed before model unload is attempted. When the call returns, unload is complete and the + * model is ready to be destroyed or loaded again. + * + * The call optionally outputs results. Use NULL as 'result' pointer if results are not required. + * + * @param model ML model to be unloaded + * @param[out] result Pointer to load result structure for output, or NULL + * + * @retval 0 Model unload was successful + * @retval <0 on failure + */ +int odp_ml_model_unload(odp_ml_model_t model, odp_ml_load_result_t *result); + +/** + * Start asynchronous model unload + * + * Otherwise like odp_ml_model_unload(), but unloads the model asynchronously. A successful call + * requests the model to be unloaded, but does not wait for unload completion. Completion + * parameters are used to select if unload completion is reported in poll (#ODP_ML_COMPL_MODE_POLL) + * or event (#ODP_ML_COMPL_MODE_EVENT) mode. For poll mode, odp_ml_model_unload_status() is called + * to check for completion. For event mode, ML offload sends the completion event into the + * completion queue when the unload is complete. Use odp_ml_compl_param_init() to initialize + * completion parameters into their default values. + * + * @param model ML model to be unloaded + * @param compl_param Completion parameters for unload + * + * @retval 0 Model unload started successfully + * @retval <0 on failure + */ +int odp_ml_model_unload_start(odp_ml_model_t model, const odp_ml_compl_param_t *compl_param); + +/** + * Check model unload completion + * + * Checks if a previously started model unload (in #ODP_ML_COMPL_MODE_POLL mode) has completed. + * The completion identifier value from unload operation completion parameters + * (odp_ml_compl_param_t.compl_id) is passed as a parameter. It specifies the unload operation to be + * checked. Initially 0 is returned for all configured (but unused) completion identifier values. + * An odp_ml_model_unload_start() call clears the previous completion status of an identifier, and + * this function returns 0 while the unload is in progress. When the unload is successfully + * complete, >0 is returned. If the unload completed with a failure, -1 is returned. The same + * value is returned until the next start operation that reuses the identifier (with the same + * model). The completion identifier may be reused only after >0 or -1 is returned. + * + * Optionally, outputs more detailed operation results into odp_ml_load_result_t structure. + * Use NULL as 'result' pointer if these results are not required. + * + * @param model ML model being unloaded + * @param compl_id Completion identifier that was used in unload start + * @param[out] result Pointer to load result structure for output, or NULL + * + * @retval >0 Model unload was successful + * @retval 0 Model unload has not finished + * @retval -1 Model unload failed + * @retval <-1 Failed to read completion status (e.g. bad handle) + */ +int odp_ml_model_unload_status(odp_ml_model_t model, uint32_t compl_id, + odp_ml_load_result_t *result); + +/** + * Initialize model run parameters + * + * Initialize an odp_ml_run_param_t to its default values. + * + * @param[out] param Model run parameters structure to be initialized + */ +void odp_ml_run_param_init(odp_ml_run_param_t *param); + +/** + * Run the model in synchronous mode + * + * Performs an ML inference operation using the model and input data pointed by the data descriptor. + * A successful operation writes inference output data into memory buffers pointed by the data + * descriptor. Input/output data buffers are described as an array of segment descriptors. Each + * segment descriptor specifies a memory buffer used with only one model input/output. Multiple + * subsequent descriptors may be used to specify segmented data for the same input/output. + * When the model has multiple inputs/outputs, descriptor order in the array follows the model + * input/output order reported by odp_ml_model_input_info() and odp_ml_model_output_info() calls. + * All memory buffers for the first input/output are specified before any buffers for the second + * input/output, and so on. + * + * When some model inputs/outputs have #ODP_ML_SHAPE_BATCH shape type, the batch size is specified + * in run parameters (odp_ml_run_param_t.batch_size). The same batch size is used for all such + * inputs/outputs. Application may request additional operation results by setting 'result' pointer + * in run parameters. Use odp_ml_run_param_init() to initialize run parameters into their default + * values. Default run parameter values are used when 'param' is NULL. + * + * Returns 1 when model run completed successfully. Returns 0 when the operation was not performed + * due to ML offload resources being temporarily busy. Returns <0 on failure. + * + * @param model ML model to be run + * @param data Model input/output data descriptor + * @param param Model run parameters, or NULL + * + * @retval 1 Model run completed successfully + * @retval 0 Resources are busy and model was not run + * @retval <0 on failure + */ +int odp_ml_run(odp_ml_model_t model, const odp_ml_data_t *data, const odp_ml_run_param_t *param); + +/** + * Run the model multiple times in synchronous mode + * + * Otherwise like odp_ml_run(), but runs the model 'num' times with different input/output + * data buffers. Output data buffers of one ML inference operation must not overlap with + * input/output data buffers of another one. + * + * Returns number of model runs successfully completed. When return value is less than 'num', + * the remaining runs were not performed due to ML offload resources being temporarily busy. + * Returns <0 on failure. + * + * @param model ML model to be run + * @param data Array of model input/output data descriptors. The array has 'num' elements. + * @param param Array of model run parameters, or NULL. The array has 'num' elements + * when used. + * @param num Number of model runs to perform + * + * @return Number of model runs completed successfully (1 ... num) + * @retval 0 Resources are busy and model was not run + * @retval <0 on failure + */ +int odp_ml_run_multi(odp_ml_model_t model, const odp_ml_data_t data[], + const odp_ml_run_param_t param[], int num); + +/** + * Start model run in asynchronous mode + * + * Otherwise like odp_ml_run(), but runs the model asynchronously. A successful call + * requests the model to be run, but does not wait for run completion. Completion parameters + * select if run completion is reported in poll (#ODP_ML_COMPL_MODE_POLL) or event + * (#ODP_ML_COMPL_MODE_EVENT) mode. For poll mode, odp_ml_run_status() is called to check + * for completion. For event mode, ML offload sends the completion event into the completion queue + * when the run is complete. Use odp_ml_compl_param_init() to initialize completion parameters + * into their default values. + * + * Additional operation results (odp_ml_run_result_t) are available through the status call + * (odp_ml_run_status()) or completion event (odp_ml_compl_run_result()). Results are + * not output through the run parameters structure (i.e. odp_ml_run_param_t.result is ignored). + * + * Returns 1 when model run was started successfully. Returns 0 when model run was not started + * due to ML offload resources being temporarily busy. Returns <0 on failure. + * + * @param model ML model to be run + * @param data Model input/output data descriptor + * @param compl_param Completion parameters + * @param run_param Model run parameters, or NULL + * + * @retval 1 Model run started successfully + * @retval 0 Resources are busy and model run was not started + * @retval <0 on failure + */ +int odp_ml_run_start(odp_ml_model_t model, const odp_ml_data_t *data, + const odp_ml_compl_param_t *compl_param, const odp_ml_run_param_t *run_param); + +/** + * Start multiple model runs in asynchronous mode + * + * Otherwise like odp_ml_run_start(), but starts 'num' model runs with different input/output + * data buffers. Output data buffers of one ML inference operation must not overlap with + * input/output data buffers of another one. + * + * Returns number of model runs started successfully. When return value is less than 'num', + * the remaining runs were not started due to ML offload resources being temporarily busy. + * Returns <0 on failure. + * + * @param model ML model to be run + * @param data Array of model input/output data descriptors. The array has 'num' elements. + * @param compl_param Array of completion parameters. The array has 'num' elements. + * @param run_param Array of model run parameters, or NULL. The array has 'num' elements + * when used. + * @param num Number of model runs to start + * + * @return Number of model runs started successfully (1 ... num) + * @retval 0 Resources are busy and model runs were not started + * @retval <0 on failure + */ +int odp_ml_run_start_multi(odp_ml_model_t model, const odp_ml_data_t data[], + const odp_ml_compl_param_t compl_param[], + const odp_ml_run_param_t run_param[], int num); + +/** + * Check model run completion + * + * Checks if a previously started model run (in #ODP_ML_COMPL_MODE_POLL mode) has completed. + * The completion identifier value from run operation completion parameters + * (odp_ml_compl_param_t.compl_id) is passed as a parameter. It specifies the run operation to be + * checked. Initially 0 is returned for all configured (but unused) completion identifier values. + * An odp_ml_run_start() call clears the previous completion status of an identifier, and + * this function returns 0 while the run is in progress. When the run is successfully + * complete, >0 is returned. If the run completed with a failure, -1 is returned. The same + * value is returned until the next start operation that reuses the identifier (with the same + * model). The completion identifier may be reused only after >0 or -1 is returned. + * + * Optionally, outputs more detailed operation results into odp_ml_run_result_t structure. + * Use NULL as 'result' pointer if these results are not required. + * + * @param model ML model running + * @param compl_id Completion identifier that was used in run start + * @param[out] result Pointer to run result structure for output, or NULL + * + * @retval >0 Model run was successful + * @retval 0 Model run has not finished + * @retval -1 Model run failed + * @retval <-1 Failed to read completion status (e.g. bad handle) + */ +int odp_ml_run_status(odp_ml_model_t model, uint32_t compl_id, odp_ml_run_result_t *result); + +/** + * Initialize ML completion event pool parameters + * + * Initialize an odp_ml_compl_pool_param_t to its default values. + * + * @param[out] param Parameter structure to be initialized + */ +void odp_ml_compl_pool_param_init(odp_ml_compl_pool_param_t *param); + +/** + * Create ML completion event pool + * + * Creates a pool of ML completion events (ODP_EVENT_ML_COMPL). Pool type is ODP_POOL_ML_COMPL. + * The use of pool name is optional. Unique names are not required. However, odp_pool_lookup() + * returns only a single matching pool. Use odp_ml_compl_pool_param_init() to initialize pool + * parameters into their default values. Parameters values must not exceed pool capabilities + * (see odp_ml_compl_pool_capability_t). + * + * @param name Name of the pool or NULL. Maximum string length is #ODP_POOL_NAME_LEN. + * @param param Pool parameters + * + * @return Pool handle on success + * @retval ODP_POOL_INVALID on failure + */ +odp_pool_t odp_ml_compl_pool_create(const char *name, const odp_ml_compl_pool_param_t *param); + +/** + * Allocate ML completion event + * + * Allocates an ML completion event from a pool. The pool must have been created with + * odp_ml_compl_pool_create() call. All completion event metadata are set to their default values. + * + * @param pool ML completion event pool + * + * @return ML completion event handle + * @retval ODP_ML_COMPL_INVALID Completion event could not be allocated + */ +odp_ml_compl_t odp_ml_compl_alloc(odp_pool_t pool); + +/** + * Free ML completion event + * + * Frees an ML completion event into the pool it was allocated from. + * + * @param ml_compl ML completion event handle + */ +void odp_ml_compl_free(odp_ml_compl_t ml_compl); + +/** + * Check ML model run results from completion event + * + * Reads model run results from an ML completion event (ODP_EVENT_ML_COMPL). The event indicates + * completion of a previously started inference operation. Subtype of the completion event must be + * ODP_EVENT_ML_COMPL_RUN. Function return value indicates if the model run succeeded or failed. + * Additionally, outputs more detailed results into the provided odp_ml_run_result_t + * structure. Use NULL as 'result' pointer if those results are not required. + * + * @param ml_compl ML completion event (subtype ODP_EVENT_ML_COMPL_RUN) + * @param[out] result Pointer to ML run result structure for output, or NULL. + * + * @retval 0 Model run was successful + * @retval -1 Model run failed + * @retval <-1 Failed to read results from the event (e.g. bad handle) + */ +int odp_ml_compl_run_result(odp_ml_compl_t ml_compl, odp_ml_run_result_t *result); + +/** + * Check ML model load / unload results from completion event + * + * Reads model load / unload results from an ML completion event (ODP_EVENT_ML_COMPL). The event + * indicates completion of a previously started model load / unload operation. Subtype of the + * completion event must be ODP_EVENT_ML_COMPL_LOAD. Function return value indicates if the model + * load / unload succeeded or failed. Additionally, outputs more detailed results into the provided + * odp_ml_load_result_t structure. Use NULL as 'result' pointer if those results are not required. + * + * @param ml_compl ML completion event (subtype ODP_EVENT_ML_COMPL_LOAD) + * @param[out] result Pointer to model load / unload result structure for output, or NULL. + * + * @retval 0 Model load / unload was successful + * @retval -1 Model load / unload failed + * @retval <-1 Failed to read results from the event (e.g. bad handle) + */ +int odp_ml_compl_load_result(odp_ml_compl_t ml_compl, odp_ml_load_result_t *result); + +/** + * ML completion event user area + * + * Returns pointer to the user area associated with the completion event. Size of the area is + * fixed and defined in pool parameters. + * + * @param ml_compl ML completion event + * + * @return Pointer to the user area of the completion event + * @retval NULL The completion event does not have user area + */ +void *odp_ml_compl_user_area(odp_ml_compl_t ml_compl); + +/** + * Convert event to ML completion event + * + * Converts an ODP_EVENT_ML_COMPL type event to an ML completion event. + * + * @param event Event handle + * + * @return ML completion event handle + */ +odp_ml_compl_t odp_ml_compl_from_event(odp_event_t event); + +/** + * Convert ML completion event to event + * + * @param ml_compl ML completion event handle + * + * @return Event handle + */ +odp_event_t odp_ml_compl_to_event(odp_ml_compl_t ml_compl); + +/** + * Convert ML completion event handle to a uint64_t value for debugging + * + * @param ml_compl ML completion event handle to be converted + * + * @return uint64_t value that can be used for debugging (e.g. printed) + */ +uint64_t odp_ml_compl_to_u64(odp_ml_compl_t ml_compl); + +/** + * Initialize ML completion parameters + * + * Initialize an odp_ml_compl_param_t to its default values. + * + * @param[out] param Address of parameters structure to be initialized + */ +void odp_ml_compl_param_init(odp_ml_compl_param_t *param); + +/** + * Retrieve model information + * + * Retrieve information about the model. Model information includes e.g. version numbers and + * number of model inputs/outputs. Information about each input and output can be retrieved with + * odp_ml_model_input_info() and odp_ml_model_output_info() calls. + * + * @param model ML model handle + * @param[out] info Pointer to model information structure for output + * + * @retval 0 on success + * @retval <0 on failure + */ +int odp_ml_model_info(odp_ml_model_t model, odp_ml_model_info_t *info); + +/** + * Retrieve model input information + * + * Writes information about each model input into the array. If there are more inputs than array + * elements, writes only 'num' elements. Returns the number of model inputs on success, and zero on + * failure. When 'num' is zero, ignores value of 'info' and returns normally. + * + * @param model ML model handle + * @param[out] info Pointer to model input information array for output + * @param num Number of elements in the array + * + * @return Number of model inputs + * @retval 0 on failure + */ +uint32_t odp_ml_model_input_info(odp_ml_model_t model, odp_ml_input_info_t info[], uint32_t num); + +/** + * Retrieve model output information + * + * Writes information about each model output into the array. If there are more outputs than array + * elements, writes only 'num' elements. Returns the number of model outputs on success, and zero on + * failure. When 'num' is zero, ignores value of 'info' and returns normally. + * + * @param model ML model handle + * @param[out] info Pointer to model output information array for output + * @param num Number of elements in the array + * + * @return Number of model outputs + * @retval 0 on failure + */ +uint32_t odp_ml_model_output_info(odp_ml_model_t model, odp_ml_output_info_t info[], uint32_t num); + +/** + * Convert ML model handle to a uint64_t value for debugging + * + * @param model ML model handle + * + * @return uint64_t value that can be used for debugging (e.g. printed) + */ +uint64_t odp_ml_model_to_u64(odp_ml_model_t model); + +/** + * Print debug information about the model. + * + * Print implementation defined information about ML model to the ODP log. The information is + * intended to be used for debugging. + + * @param model ML model handle + */ +void odp_ml_model_print(odp_ml_model_t model); + +/** + * Print ML debug information + * + * Print implementation defined information about ML offload to the ODP log. The information is + * intended to be used for debugging. + */ +void odp_ml_print(void); + +/** + * Extra statistics counter information + * + * Returns the number of extra statistics counters supported by the ML offload, and outputs + * information (e.g. name) about those. Counters are implementation specific and maintained + * per model. Statistics counting is enabled through model create parameters. + * + * When 'info' pointer is not NULL, fills in up to 'num' counter info structures. If the return + * value is larger than 'num', there are more counters than the function was allowed to output. + * If the return value N is less than 'num' (on success), only first N structures have been written. + * + * Info array elements are filled in the same order than odp_ml_model_extra_stats() outputs + * counter values. + * + * @param model ML model + * @param[out] info Pointer to extra statistics counter information array for output. + * NULL may be used to query only the number of counters. + * @param num Number of elements in the array + * + * @return Number of extra statistics counters + * @retval <0 on failure + */ +int odp_ml_model_extra_stat_info(odp_ml_model_t model, odp_ml_extra_stat_info_t info[], int num); + +/** + * Read extra statistics counter values + * + * Reads extra statistics counter values and returns the number of supported counters. Outputs + * up to 'num' counter values into 'stats' array. If the return value is larger than 'num', + * there are more counters than the function was allowed to output. If the return value N is less + * than 'num' (on success), only first N counters have been written. The order of counters in + * the array matches the counter information array order on odp_ml_model_extra_stat_info() output. + * + * @param model ML model + * @param[out] stats Pointer to extra statistics counter array for output + * @param num Number of elements in the array + * + * @return Number of extra statistics counters + * @retval <0 on failure + */ +int odp_ml_model_extra_stats(odp_ml_model_t model, uint64_t stats[], int num); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#include <odp/visibility_end.h> +#endif diff --git a/include/odp/api/spec/ml_quantize.h b/include/odp/api/spec/ml_quantize.h new file mode 100644 index 000000000..25565ef27 --- /dev/null +++ b/include/odp/api/spec/ml_quantize.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +/** + * @file + * + * ODP Machine Learning (ML) quantization functions + */ + +#ifndef ODP_API_SPEC_ML_QUANTIZE_H_ +#define ODP_API_SPEC_ML_QUANTIZE_H_ +#include <odp/visibility_begin.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/std_types.h> + +/** @addtogroup odp_ml + * @{ + */ + +/** + * Quantize 32-bit float to uint8_t + * + * Quantizes 'num' 32-bit floating point values to uint8_t values using the provided scale and + * zero point. + * + * dst_u8 = (src_fp32 / scale) + zerop + * + * @param[out] dst_u8 Destination address for quantized values + * @param src_fp32 Source address of values to be quantized + * @param num Number of values + * @param scale Scale for quantization + * @param zerop Zero point for quantization + */ +void odp_ml_fp32_to_uint8(uint8_t *dst_u8, const float *src_fp32, uint32_t num, + float scale, uint8_t zerop); + +/** + * De-quantize 32-bit float from uint8_t + * + * De-quantizes 'num' 32-bit floating point values from uint8_t values using the provided scale and + * zero point. + * + * dst_fp32 = (src_u8 - zerop) * scale + * + * @param[out] dst_fp32 Destination address for de-quantized values + * @param src_u8 Source address of values to be de-quantized + * @param num Number of values + * @param scale Scale for de-quantization + * @param zerop Zero point for de-quantization + */ +void odp_ml_fp32_from_uint8(float *dst_fp32, const uint8_t *src_u8, uint32_t num, + float scale, uint8_t zerop); + +/** + * Quantize 32-bit float to int8_t + * + * Quantizes 'num' 32-bit floating point values to int8_t values using the provided scale and + * zero point. + * + * dst_i8 = (src_fp32 / scale) + zerop + * + * @param[out] dst_i8 Destination address for quantized values + * @param src_fp32 Source address of values to be quantized + * @param num Number of values + * @param scale Scale for quantization + * @param zerop Zero point for quantization + */ +void odp_ml_fp32_to_int8(int8_t *dst_i8, const float *src_fp32, uint32_t num, float scale, + int8_t zerop); + +/** + * De-quantize 32-bit float from int8_t + * + * De-quantizes 'num' 32-bit floating point values from int8_t values using the provided scale and + * zero point. + * + * dst_fp32 = (src_i8 - zerop) * scale + * + * @param[out] dst_fp32 Destination address for de-quantized values + * @param src_i8 Source address of values to be de-quantized + * @param num Number of values + * @param scale Scale for de-quantization + * @param zerop Zero point for de-quantization + */ +void odp_ml_fp32_from_int8(float *dst_fp32, const int8_t *src_i8, uint32_t num, float scale, + int8_t zerop); + +/** + * Quantize 32-bit float to 16-bit float + * + * Quantizes 'num' 32-bit floating point values to 16-bit floating point values. + * + * @param[out] dst_fp16 Destination address for quantized values + * @param src_fp32 Source address of values to be quantized + * @param num Number of values + */ +void odp_ml_fp32_to_fp16(uint16_t *dst_fp16, const float *src_fp32, uint32_t num); + +/** + * De-quantize 32-bit float from 16-bit float + * + * De-quantizes 'num' 32-bit floating point values from 16-bit floating point values. + * + * @param[out] dst_fp32 Destination address for de-quantized values + * @param src_fp16 Source address of values to be de-quantized + * @param num Number of values + */ +void odp_ml_fp32_from_fp16(float *dst_fp32, const uint16_t *src_fp16, uint32_t num); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#include <odp/visibility_end.h> +#endif diff --git a/include/odp/api/spec/ml_types.h b/include/odp/api/spec/ml_types.h new file mode 100644 index 000000000..2b8f9d6c8 --- /dev/null +++ b/include/odp/api/spec/ml_types.h @@ -0,0 +1,873 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021-2023 Nokia + * Copyright (c) 2021 Marvell + */ + +/** + * @file + * + * ODP Machine Learning (ML) types + */ + +#ifndef ODP_API_SPEC_ML_TYPES_H_ +#define ODP_API_SPEC_ML_TYPES_H_ +#include <odp/visibility_begin.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/event_types.h> +#include <odp/api/queue_types.h> +#include <odp/api/std_types.h> + +/** @defgroup odp_ml ODP ML + * @{ + */ + +/** + * @typedef odp_ml_model_t + * ODP ML model handle + */ + +/** + * @def ODP_ML_MODEL_INVALID + * Invalid ML model + */ + +/** + * @typedef odp_ml_compl_t + * ML completion event + */ + +/** + * @def ODP_ML_COMPL_INVALID + * Invalid ML completion event + */ + +/** + * @def ODP_ML_MODEL_NAME_LEN + * Maximum length of model name in chars (including null char) + */ + +/** + * @def ODP_ML_MODEL_IO_NAME_LEN + * Maximum length of model input/output name in chars (including null char) + */ + +/** + * @def ODP_ML_SHAPE_NAME_LEN + * Maximum length of data dimension name in chars (including null char) + */ + +/** + * @def ODP_ML_EXTRA_STAT_NAME_LEN + * Maximum length of extra statistics counter name in chars (including null char) + */ + +/** + * @typedef odp_ml_model_extra_param_t + * ODP implementation specific extra parameters for model creation + */ + +/** Maximum number of dimensions in input / output data shape */ +#define ODP_ML_MAX_DIMS 8 + +/** Dimension size is dynamic */ +#define ODP_ML_DIM_DYNAMIC 0 + +/** Synchronous operation */ +#define ODP_ML_COMPL_MODE_SYNC 0x1u + +/** + * Asynchronous poll mode operation + * + * A function call starts an operation and a status function call indicates when + * the operation has finished. + */ +#define ODP_ML_COMPL_MODE_POLL 0x2u + +/** + * Asynchronous event mode operation + * + * A function call starts an operation and a completion event indicates when + * the operation has finished. + */ +#define ODP_ML_COMPL_MODE_EVENT 0x4u + +/** ML completion mode */ +typedef uint32_t odp_ml_compl_mode_t; + +/** + * ML completion event pool capabilities + * + * Pool statistics are not supported with ML completion event pools. + */ +typedef struct odp_ml_compl_pool_capability_t { + /** + * Maximum number of ML completion event pools + * + * See odp_pool_capability_t.max_pools for maximum number of pools of any type. It + * includes also ML completion event pools. + */ + uint32_t max_pools; + + /** Maximum number of ML completion events in a pool */ + uint32_t max_num; + + /** Maximum user area size in bytes */ + uint32_t max_uarea_size; + + /** User area persistence + * + * See buf.uarea_persistence of odp_pool_capability_t for details + * (odp_pool_capability_t.uarea_persistence). + */ + odp_bool_t uarea_persistence; + + /** Maximum size of local thread cache */ + uint32_t max_cache_size; + + /** Minimum size of local thread cache */ + uint32_t min_cache_size; + +} odp_ml_compl_pool_capability_t; + +/** + * ML completion event pool parameters + * + * Use odp_ml_compl_pool_param_init() to initialize the structure to its default values. + */ +typedef struct odp_ml_compl_pool_param_t { + /** + * Number of ML completion events in the pool + * + * The maximum supported value is defined by ML pool capability 'max_num' + * (odp_ml_compl_pool_capability_t.max_num). + */ + uint32_t num; + + /** + * User area size in bytes + * + * The maximum supported value is defined by ML pool capability 'max_uarea_size'. + * Specify as zero if no user area is needed. The default value is 0. + */ + uint32_t uarea_size; + + /** Parameters for user area initialization */ + struct { + /** See uarea_init.init_fn of odp_pool_param_t for details + * (odp_pool_param_t.init_fn). Function is called during + * odp_ml_compl_pool_create(). The default value is NULL. */ + void (*init_fn)(void *uarea, uint32_t size, void *args, uint32_t index); + + /** See uarea_init.args of odp_pool_param_t for details + * (odp_pool_param_t.args). The default value is NULL. */ + void *args; + + } uarea_init; + + /** + * Maximum number of events cached locally per thread + * + * See odp_pool_param_t.cache_size documentation for details. Valid values range from + * 'min_cache_size' to 'max_cache_size' ML pool capability. The default value is + * implementation specific and set by odp_ml_compl_pool_param_init(). + */ + uint32_t cache_size; + +} odp_ml_compl_pool_param_t; + +/** Machine learning capabilities */ +typedef struct odp_ml_capability_t { + /** Maximum number of models + * + * Maximum number of models that can be created simultaneously. The value is zero when + * ML offload is not available. */ + uint32_t max_models; + + /** Maximum number of models that can be loaded simultaneously */ + uint32_t max_models_loaded; + + /** Maximum model size in bytes */ + uint64_t max_model_size; + + /** Maximum completion identifier value */ + uint32_t max_compl_id; + + /** Maximum number of model inputs */ + uint32_t max_inputs; + + /** Maximum number of model outputs */ + uint32_t max_outputs; + + /** + * Maximum number of data segments per model input + * + * Segmented input data is not supported when 1. + */ + uint32_t max_segs_per_input; + + /** + * Maximum number of data segments per model output + * + * Segmented output data is not supported when 1. + */ + uint32_t max_segs_per_output; + + /** + * Minimum input data alignment in bytes + * + * For each model input, the first data segment must start at this or a higher power of two + * memory alignment in bytes. The value is 1 when there is no alignment requirement. + */ + uint32_t min_input_align; + + /** + * Minimum output data alignment in bytes + * + * For each model output, the first data segment must start at this or a higher power of two + * memory alignment in bytes. The value is 1 when there is no alignment requirement. + */ + uint32_t min_output_align; + + /** + * Input data packing + * + * 0: Data packing is not required. + * 1: Data for all model inputs must be continuous in memory. The memory block starts with + * data for the first input and continues through all inputs in-order and without gaps + * between inputs. The minimum alignment requirement (min_input_align) applies only for + * the first input. + */ + odp_bool_t packed_input_data; + + /** + * Output data packing + * + * 0: Data packing is not required. + * 1: Data buffer space for all model outputs must be continuous in memory. The memory + * block starts with buffer space for the first output and continues through all outputs + * in-order and without gaps between outputs. The minimum alignment requirement + * (min_output_align) applies only for the first output. + */ + odp_bool_t packed_output_data; + + /** Model load / unload capabilities */ + struct { + /** + * Supported completion modes for model load / unload operations + * + * Mask of supported completion modes. Each supported mode has the corresponding + * flag (e.g. #ODP_ML_COMPL_MODE_SYNC) set in the mask. + */ + odp_ml_compl_mode_t compl_mode_mask; + + /** + * Support of model load / unload completion into plain queues + * + * Specifies if plain queues are supported as destination queues for + * load / unload completion events (#ODP_ML_COMPL_MODE_EVENT). + * + * 0: Plain queues are not supported as completion queues + * 1: Plain queues are supported as completion queues + */ + odp_bool_t compl_queue_plain; + + /** + * Support of model load / unload completion into scheduled queues + * + * Specifies if scheduled queues are supported as destination queues for + * load / unload completion events (#ODP_ML_COMPL_MODE_EVENT). + * + * 0: Scheduled queues are not supported as completion queues + * 1: Scheduled queues are supported as completion queues + */ + odp_bool_t compl_queue_sched; + + } load; + + /** Model run capabilities */ + struct { + /** + * Supported completion modes for model run operations + * + * Mask of supported completion modes. Each supported mode has the corresponding + * flag (e.g. #ODP_ML_COMPL_MODE_SYNC) set in the mask. + */ + odp_ml_compl_mode_t compl_mode_mask; + + /** + * Support of model run completion into plain queues + * + * Specifies if plain queues are supported as destination queues for + * run completion events (#ODP_ML_COMPL_MODE_EVENT). + * + * 0: Plain queues are not supported as completion queues + * 1: Plain queues are supported as completion queues + */ + odp_bool_t compl_queue_plain; + + /** + * Support of model run completion into scheduled queues + * + * Specifies if scheduled queues are supported as destination queues for + * run completion events (#ODP_ML_COMPL_MODE_EVENT). + * + * 0: Scheduled queues are not supported as completion queues + * 1: Scheduled queues are supported as completion queues + */ + odp_bool_t compl_queue_sched; + + } run; + + /** ML completion event pool capabilities */ + odp_ml_compl_pool_capability_t pool; + +} odp_ml_capability_t; + +/** Machine learning configuration parameters */ +typedef struct odp_ml_config_t { + /** + * Maximum number of models + * + * Application may create and use this many models simultaneously. The default value is 1. + */ + uint32_t max_models_created; + + /** + * Maximum number of models loaded + * + * Maximum number of models that the application will keep loaded simultaneously. + * The default value is 1. + */ + uint32_t max_models_loaded; + + /** + * Maximum model binary size in bytes + * + * All model binaries application will pass to odp_ml_model_create() are this size or + * smaller. + */ + uint64_t max_model_size; + + /** + * Load / unload completion modes + * + * Mask of completion modes that application will use with model load/unload operations. + * Multiple modes may be selected, but it is implementation specific if some combinations + * are not supported. In case of an unsupported combination odp_ml_config() returns + * failure. Check odp_ml_capability_t.load for supported modes. The default value is 0. + */ + odp_ml_compl_mode_t load_mode_mask; + + /** + * Run completion modes + * + * Mask of completion modes that application will use with model run operations. + * Multiple modes may be selected, but it is implementation specific if some combinations + * are not supported. In case of an unsupported combination odp_ml_config() returns + * failure. Check odp_ml_capability_t.run for supported modes. The default value is 0. + */ + odp_ml_compl_mode_t run_mode_mask; + +} odp_ml_config_t; + +/** Model input / output data type enumeration */ +typedef enum { + /** Data type is not defined */ + ODP_ML_DATA_TYPE_NONE = 0, + + /** 8-bit integer */ + ODP_ML_DATA_TYPE_INT8, + + /** 8-bit unsigned integer */ + ODP_ML_DATA_TYPE_UINT8, + + /** 16-bit integer */ + ODP_ML_DATA_TYPE_INT16, + + /** 16-bit unsigned integer */ + ODP_ML_DATA_TYPE_UINT16, + + /** 24-bit integer */ + ODP_ML_DATA_TYPE_INT24, + + /** 24-bit unsigned integer */ + ODP_ML_DATA_TYPE_UINT24, + + /** 32-bit integer */ + ODP_ML_DATA_TYPE_INT32, + + /** 32-bit unsigned integer */ + ODP_ML_DATA_TYPE_UINT32, + + /** 64-bit integer */ + ODP_ML_DATA_TYPE_INT64, + + /** 64-bit unsigned integer */ + ODP_ML_DATA_TYPE_UINT64, + + /** 16-bit floating point number */ + ODP_ML_DATA_TYPE_FP16, + + /** 16-bit brain floating point (bfloat16) number */ + ODP_ML_DATA_TYPE_BFP16, + + /** 32-bit floating point number */ + ODP_ML_DATA_TYPE_FP32, + + /** 64-bit floating point number */ + ODP_ML_DATA_TYPE_FP64, + +} odp_ml_data_type_t; + +/** Model input / output data shape type */ +typedef enum { + /** Type of shape is not defined */ + ODP_ML_SHAPE_NONE = 0, + + /** Static shape of data + * + * Shape is static when all dimensions have fixed sizes. + */ + ODP_ML_SHAPE_STATIC, + + /** Dynamic batch size + * + * Shape that has only one dynamic dimension, and the dimension is used as batch size of + * input / output data. The same batch size is applied for all inputs and outputs of + * the model. + */ + ODP_ML_SHAPE_BATCH, + +} odp_ml_shape_type_t; + +/** Model input / output data shape information */ +typedef struct odp_ml_shape_info_t { + /** Shape type */ + odp_ml_shape_type_t type; + + /** Number of dimensions + * + * Number of input / output data dimensions. When zero, the model does not have + * dimension information available. ODP API supports in maximum #ODP_ML_MAX_DIMS + * dimensions. + */ + uint32_t num_dim; + + /** Dimension sizes + * + * Number of data values in each ('num_dim') dimension. Type of the data is defined by + * odp_ml_data_type_t enumeration. Depending on the shape type, some dimensions may have + * dynamic size which is denoted with #ODP_ML_DIM_DYNAMIC value. When shape type is + * #ODP_ML_SHAPE_BATCH, the shape has one dynamic dimension which is used as the batch + * size. + * + * For example, a static (#ODP_ML_SHAPE_STATIC) NCHW tensor could be presented as: + * + * num_dim = 4; + * dim[0] = 1; // no batching, N = 1 + * dim[1] = 3; // 3 color channels + * dim[2] = 720; // height 720 pixels + * dim[3] = 1280; // width 1280 pixels + * + * ... and with dynamic batch size (#ODP_ML_SHAPE_BATCH): + * + * num_dim = 4; + * dim[0] = ODP_ML_DIM_DYNAMIC; // dynamic in range: dim_min[0] ... dim_max[0] + * dim[1] = 3; + * dim[2] = 720; + * dim[3] = 1280; + */ + uint32_t dim[ODP_ML_MAX_DIMS]; + + /** Minimum dimension sizes + * + * Defines the minimum value for each dynamic size (#ODP_ML_DIM_DYNAMIC) in dim[] array. + * Zero is used when the minimum value is unknown. When dimension size is static, the + * value is equal to dim[] array value. + */ + uint32_t dim_min[ODP_ML_MAX_DIMS]; + + /** Maximum dimension sizes + * + * Defines the maximum value for each dynamic size (#ODP_ML_DIM_DYNAMIC) in dim[] array. + * Zero is used when the maximum value is unknown. When dimension size is static, the + * value is equal to dim[] array value. + */ + uint32_t dim_max[ODP_ML_MAX_DIMS]; + + /** Dimension name + * + * Name of each ('num_dim') dimension as a null terminated string. Null string is used if + * a dimension does not have a name. Maximum string length is #ODP_ML_SHAPE_NAME_LEN + * including the null character. + * + * For example, an NCHW tensor could have dimensions named as: + * dim_name = {"N", "C", "H", "W"} + */ + char dim_name[ODP_ML_MAX_DIMS][ODP_ML_SHAPE_NAME_LEN]; + +} odp_ml_shape_info_t; + +/** Model input information */ +typedef struct odp_ml_input_info_t { + /** Model input name */ + char name[ODP_ML_MODEL_IO_NAME_LEN]; + + /** Model input data type */ + odp_ml_data_type_t data_type; + + /** Size of model input data type in bytes */ + uint32_t data_type_size; + + /** Model input data shape */ + odp_ml_shape_info_t shape; + +} odp_ml_input_info_t; + +/** Model output information */ +typedef struct odp_ml_output_info_t { + /** Model output name */ + char name[ODP_ML_MODEL_IO_NAME_LEN]; + + /** Model output data type */ + odp_ml_data_type_t data_type; + + /** Size of model output data type in bytes */ + uint32_t data_type_size; + + /** Model output data shape */ + odp_ml_shape_info_t shape; + +} odp_ml_output_info_t; + +/** Model information */ +typedef struct odp_ml_model_info_t { + /** Model name */ + char name[ODP_ML_MODEL_NAME_LEN]; + + /** + * Model version number + * + * Version number of the model binary. The number changes when the model is modified + * in any way. + */ + uint64_t model_version; + + /** + * Model interface version number + * + * The model interface version number changes only when model input or output data + * format is modified. Data formats are the same for two model versions that have + * the same interface version number. + */ + uint64_t interface_version; + + /** Model index assigned by the implementation */ + uint32_t index; + + /** Number of model inputs */ + uint32_t num_inputs; + + /** Number of model outputs */ + uint32_t num_outputs; + +} odp_ml_model_info_t; + +/** + * Model input / output data format + */ +typedef struct odp_ml_data_format_t { + /** Model input / output data type */ + odp_ml_data_type_t data_type; + + /** Size of data type in bytes */ + uint32_t data_type_size; + + /** Model input / output data shape */ + odp_ml_shape_info_t shape; + +} odp_ml_data_format_t; + +/** + * Machine learning model parameters + * + * Use odp_ml_model_param_init() to initialize the structure to its default values. + */ +typedef struct odp_ml_model_param_t { + /** + * Model binary + * + * Points to model binary stored into a memory buffer. Model format is + * implementation specific. */ + void *model; + + /** Size of the model binary in bytes */ + uint64_t size; + + /** + * Maximum completion identifier value + * + * When application uses asynchronous poll mode (#ODP_ML_COMPL_MODE_POLL) operations with + * the model, it will choose completion identifier values between 0 and this value. Valid + * values range from 0 to max_compl_id capability. The default value is zero. + */ + uint32_t max_compl_id; + + /** + * Enable / disable extra statistics counters + * + * Extra statistics may be read with odp_ml_model_extra_stats() when enabled. Statistics + * are disabled by default. + */ + odp_bool_t extra_stat_enable; + + /** + * Extra model information + * + * When model metadata misses some details of model input / output data format, user can + * pass those with this structure. When 'num_inputs' / 'num_outputs' is non-zero, data + * format of all model inputs / outputs are overridden by the provided values. Values are + * ignored when 'num_inputs' / 'num_outputs' is zero. + */ + struct { + /** + * Number of model inputs + * + * Number of model inputs and elements in 'input_format' array. When non-zero, + * the value must match the number of inputs defined in model metadata. The default + * value is 0. + */ + uint32_t num_inputs; + + /** + * Number of model outputs + * + * Number of model outputs and elements in 'output_format' array. When non-zero, + * the value must match the number of outputs defined in model metadata. The default + * value is 0. + */ + uint32_t num_outputs; + + /** + * Model input data format array + * + * Points to an array of data formats. The array has 'num_inputs' elements. Inputs + * are defined in the same order they are listed in model metadata. + * An odp_ml_model_create() call copies these values. The default value is NULL. + */ + const odp_ml_data_format_t *input_format; + + /** + * Model output data format array + * + * Points to an array of data formats. The array has 'num_outputs' elements. Outputs + * are defined in the same order they are listed in model metadata. + * An odp_ml_model_create() call copies these values. The default value is NULL. + */ + const odp_ml_data_format_t *output_format; + + } extra_info; + + /** + * ODP implementation specific extra parameters + * + * See ODP implementation documentation for details about extra parameter usage. For + * example, extra parameters may give hints about HW resource usage with the model to be + * created. An odp_ml_model_create() call copies these parameter values. When NULL, all + * extra parameters are set to their default values. The default value is NULL. + */ + const odp_ml_model_extra_param_t *extra_param; + +} odp_ml_model_param_t; + +/** Results of model run operation */ +typedef struct odp_ml_run_result_t { + /** Model run error code + * + * Zero when model run completed successfully. Otherwise, error code contains + * an implementation specific value. + */ + uint64_t error_code; + + /** User context pointer value from odp_ml_compl_param_t */ + void *user_ptr; + +} odp_ml_run_result_t; + +/** Result of model load / unload operation */ +typedef struct odp_ml_load_result_t { + /** Model load / unload error code + * + * Zero when model load / unload completed successfully. Otherwise, error code contains + * an implementation specific value. + */ + uint64_t error_code; + + /** User context pointer value from odp_ml_compl_param_t */ + void *user_ptr; + +} odp_ml_load_result_t; + +/** + * ML completion parameters + * + * Use odp_ml_compl_param_init() to initialize the structure to its default values. + */ +typedef struct odp_ml_compl_param_t { + /** + * Completion mode + * + * The selected completion mode defines which other parameters are used. When + * #ODP_ML_COMPL_MODE_EVENT mode is selected, 'event' and 'queue' must have valid values + * but value of 'compl_id' is ignored, or vice versa when #ODP_ML_COMPL_MODE_POLL mode is + * selected. + */ + odp_ml_compl_mode_t mode; + + /** + * Completion event + * + * Event to be enqueued by ML offload to the completion queue when ML operation + * is complete. Event type must be ODP_EVENT_ML_COMPL. ML offload sets the subtype of + * the event to ODP_EVENT_ML_COMPL_LOAD or ODP_EVENT_ML_COMPL_RUN based on the completed + * operation. + */ + odp_event_t event; + + /** + * Completion queue + * + * Destination queue for the completion event. + */ + odp_queue_t queue; + + /** + * Completion identifier + * + * When completion mode is #ODP_ML_COMPL_MODE_POLL, ML operation completion status is + * reported through this completion identifier. The value passed here is used in + * a following status call to check model load, unload, or inference completion + * (see e.g. odp_ml_model_load_status()). + * + * Application selects a value between 0 and max_compl_id defined in model creation + * parameters (see odp_ml_model_param_t). Only single ML operation (per model) may be + * started with the same identifier value at a time. A value may be reused for the next + * ML operation only after the previous operation is complete. + */ + uint32_t compl_id; + + /** + * User defined context pointer + * + * ODP implementation does not refer to the pointer, but just copies it to the result. + * For example, application may use this pointer to link a received completion event + * to the originating model run request and its input / output data. The default value + * is NULL. + */ + void *user_ptr; + +} odp_ml_compl_param_t; + +/** Model input / output data segment */ +typedef struct odp_ml_data_seg_t { + /** Segment start address */ + void *addr; + + /** Segment size in bytes */ + uint64_t size; + +} odp_ml_data_seg_t; + +/** Model input / output data for a model inference run */ +typedef struct odp_ml_data_t { + /** + * Number of input data segments + * + * Number of elements in 'input_seg' array (at least one per input). + */ + uint32_t num_input_seg; + + /** + * Number of output data segments + * + * Number of elements in 'output_seg' array (at least one per output). + */ + uint32_t num_output_seg; + + /** + * Model input data segments + * + * Points to an array of data segment descriptors for model input data. Each segment + * (odp_ml_data_seg_t) specifies data for one input only. Multiple consecutive segments may + * be used to specify data for the same input. Sum of those segment sizes must match data + * size of the input. Inputs are defined in the same order which odp_ml_model_input_info() + * reports those. + * + * Input data segments may overlap in memory. + */ + odp_ml_data_seg_t *input_seg; + + /** + * Model output data segments + * + * Points to an array of data segment descriptors for model output data. Each segment + * (odp_ml_data_seg_t) specifies data buffer space for one output only. Multiple + * consecutive segments may be used to specify buffer space for the same output. + * Sum of those segment sizes must match data size of the output. Outputs are defined + * in the same order which odp_ml_model_output_info() reports those. + * + * An output data segment must not overlap with any other (input or output) segment + * in memory. + */ + odp_ml_data_seg_t *output_seg; + +} odp_ml_data_t; + +/** + * Parameters for model run + * + * Use odp_ml_run_param_init() to initialize the structure to its default values. + */ +typedef struct odp_ml_run_param_t { + /** + * Batch size + * + * Batch size for all model inputs and outputs that have #ODP_ML_SHAPE_BATCH shape type. + * The default value is 0. + */ + uint32_t batch_size; + + /** + * Model run results + * + * Points to a result structure for model run result output. Results are output only + * in synchronous mode (#ODP_ML_COMPL_MODE_SYNC). The pointer value is ignored in + * asynchronous modes. Use NULL when results are not required. The default value is NULL. + */ + odp_ml_run_result_t *result; + +} odp_ml_run_param_t; + +/** + * ML extra statistics counter information + */ +typedef struct odp_ml_extra_stat_info_t { + /** Name of the statistics counter */ + char name[ODP_ML_EXTRA_STAT_NAME_LEN]; + +} odp_ml_extra_stat_info_t; + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#include <odp/visibility_end.h> +#endif diff --git a/include/odp/arch/arm32-linux/odp/api/abi/ml_types.h b/include/odp/arch/arm32-linux/odp/api/abi/ml_types.h new file mode 100644 index 000000000..4201f92e6 --- /dev/null +++ b/include/odp/arch/arm32-linux/odp/api/abi/ml_types.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +#include <odp/api/abi-default/ml_types.h> diff --git a/include/odp/arch/arm64-linux/odp/api/abi/ml_types.h b/include/odp/arch/arm64-linux/odp/api/abi/ml_types.h new file mode 100644 index 000000000..4201f92e6 --- /dev/null +++ b/include/odp/arch/arm64-linux/odp/api/abi/ml_types.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +#include <odp/api/abi-default/ml_types.h> diff --git a/include/odp/arch/default-linux/odp/api/abi/ml_types.h b/include/odp/arch/default-linux/odp/api/abi/ml_types.h new file mode 100644 index 000000000..4201f92e6 --- /dev/null +++ b/include/odp/arch/default-linux/odp/api/abi/ml_types.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +#include <odp/api/abi-default/ml_types.h> diff --git a/include/odp/arch/power64-linux/odp/api/abi/ml_types.h b/include/odp/arch/power64-linux/odp/api/abi/ml_types.h new file mode 100644 index 000000000..4201f92e6 --- /dev/null +++ b/include/odp/arch/power64-linux/odp/api/abi/ml_types.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +#include <odp/api/abi-default/ml_types.h> diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/ml_types.h b/include/odp/arch/x86_32-linux/odp/api/abi/ml_types.h new file mode 100644 index 000000000..4201f92e6 --- /dev/null +++ b/include/odp/arch/x86_32-linux/odp/api/abi/ml_types.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +#include <odp/api/abi-default/ml_types.h> diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/ml_types.h b/include/odp/arch/x86_64-linux/odp/api/abi/ml_types.h new file mode 100644 index 000000000..4201f92e6 --- /dev/null +++ b/include/odp/arch/x86_64-linux/odp/api/abi/ml_types.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +#include <odp/api/abi-default/ml_types.h> diff --git a/include/odp_api.h b/include/odp_api.h index e7cad266f..cbd1af691 100644 --- a/include/odp_api.h +++ b/include/odp_api.h @@ -55,6 +55,8 @@ extern "C" { #include <odp/api/errno.h> #include <odp/api/thrmask.h> #include <odp/api/traffic_mngr.h> +#include <odp/api/ml.h> +#include <odp/api/ml_quantize.h> #include <odp/api/spinlock_recursive.h> #include <odp/api/rwlock_recursive.h> #include <odp/api/std.h> diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index ab4f98720..c2aec362e 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -90,6 +90,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/init.h \ include-abi/odp/api/abi/ipsec.h \ include-abi/odp/api/abi/ipsec_types.h \ + include-abi/odp/api/abi/ml_types.h \ include-abi/odp/api/abi/packet.h \ include-abi/odp/api/abi/packet_types.h \ include-abi/odp/api/abi/packet_flags.h \ diff --git a/platform/linux-generic/include-abi/odp/api/abi/ml_types.h b/platform/linux-generic/include-abi/odp/api/abi/ml_types.h new file mode 100644 index 000000000..0fdb7a8dc --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/ml_types.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021-2023 Nokia + */ + +#ifndef ODP_API_ABI_ML_TYPES_H_ +#define ODP_API_ABI_ML_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/std_types.h> +#include <odp/api/plat/strong_types.h> + +/** @internal Implementation specific ML parameters */ +struct _odp_ml_model_extra_param_t { + /** @internal Dummy field to avoid empty struct */ + char dummy; +}; + +/** @addtogroup odp_ml + * @{ + */ + +typedef ODP_HANDLE_T(odp_ml_model_t); +typedef ODP_HANDLE_T(odp_ml_compl_t); +typedef struct _odp_ml_model_extra_param_t odp_ml_model_extra_param_t; + +#define ODP_ML_MODEL_INVALID _odp_cast_scalar(odp_ml_model_t, 0) +#define ODP_ML_COMPL_INVALID _odp_cast_scalar(odp_ml_compl_t, 0) + +#define ODP_ML_MODEL_NAME_LEN 64 +#define ODP_ML_MODEL_IO_NAME_LEN 64 +#define ODP_ML_SHAPE_NAME_LEN 16 +#define ODP_ML_EXTRA_STAT_NAME_LEN 64 + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif From ebb3606c308666f276dfd5f3c90fc520d6aaec09 Mon Sep 17 00:00:00 2001 From: Petri Savolainen <petri.savolainen@nokia.com> Date: Mon, 30 Oct 2023 14:01:24 +0200 Subject: [PATCH 113/147] api: feature: add ML API into ODP features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added ML API into ODP features bit-field. Signed-off-by: Petri Savolainen <petri.savolainen@nokia.com> Reviewed-by: Jere Leppänen <jere.leppanen@nokia.com> Reviewed-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- include/odp/api/spec/std_types.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/odp/api/spec/std_types.h b/include/odp/api/spec/std_types.h index e23dfd224..5428631b6 100644 --- a/include/odp/api/spec/std_types.h +++ b/include/odp/api/spec/std_types.h @@ -138,6 +138,9 @@ typedef union odp_feature_t { /** IPsec APIs, e.g., odp_ipsec_xxx() */ uint32_t ipsec:1; + /** Machine Learning APIs, e.g., odp_ml_xxx() */ + uint32_t ml:1; + /** Scheduler APIs, e.g., odp_schedule_xxx() */ uint32_t schedule:1; From 1417a4724d52254ddba851cbcb22ecb1f798283d Mon Sep 17 00:00:00 2001 From: Petri Savolainen <petri.savolainen@nokia.com> Date: Mon, 30 Oct 2023 13:41:03 +0200 Subject: [PATCH 114/147] api: event: add ML completion event type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added event type and subtype enumerations for ML completion event. Signed-off-by: Petri Savolainen <petri.savolainen@nokia.com> Reviewed-by: Jere Leppänen <jere.leppanen@nokia.com> Reviewed-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- include/odp/api/abi-default/event_types.h | 7 +++++-- include/odp/api/spec/event_types.h | 10 +++++++++- .../include-abi/odp/api/abi/event_types.h | 7 +++++-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/include/odp/api/abi-default/event_types.h b/include/odp/api/abi-default/event_types.h index 7169a3bd4..e5b50d9c0 100644 --- a/include/odp/api/abi-default/event_types.h +++ b/include/odp/api/abi-default/event_types.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2017-2018 Linaro Limited - * Copyright (c) 2022 Nokia + * Copyright (c) 2022-2023 Nokia */ #ifndef ODP_ABI_EVENT_TYPES_H_ @@ -31,6 +31,7 @@ typedef enum { ODP_EVENT_PACKET_VECTOR = 6, ODP_EVENT_PACKET_TX_COMPL = 7, ODP_EVENT_DMA_COMPL = 8, + ODP_EVENT_ML_COMPL = 9 } odp_event_type_t; typedef enum { @@ -38,7 +39,9 @@ typedef enum { ODP_EVENT_PACKET_BASIC = 1, ODP_EVENT_PACKET_CRYPTO = 2, ODP_EVENT_PACKET_IPSEC = 3, - ODP_EVENT_PACKET_COMP = 4 + ODP_EVENT_PACKET_COMP = 4, + ODP_EVENT_ML_COMPL_LOAD = 5, + ODP_EVENT_ML_COMPL_RUN = 6 } odp_event_subtype_t; /** diff --git a/include/odp/api/spec/event_types.h b/include/odp/api/spec/event_types.h index 05c4d1f3c..b967e2871 100644 --- a/include/odp/api/spec/event_types.h +++ b/include/odp/api/spec/event_types.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2015-2018 Linaro Limited - * Copyright (c) 2022 Nokia + * Copyright (c) 2022-2023 Nokia */ /** @@ -59,6 +59,8 @@ extern "C" { * completion. * - ODP_EVENT_DMA_COMPL * - DMA completion event (odp_dma_compl_t) indicates that a DMA transfer has finished + * - ODP_EVENT_ML_COMPL + * - ML completion event (odp_ml_compl_t) indicates that an ML operation has finished */ /** @@ -92,6 +94,12 @@ extern "C" { * packet metadata. * - ODP_EVENT_NO_SUBTYPE * - An event type does not have any subtypes defined + * - ODP_EVENT_ML_COMPL_LOAD + * - ML completion event (odp_ml_compl_t) that contains results from a completed model load or + * unload operation. + * - ODP_EVENT_ML_COMPL_RUN + * - ML completion event (odp_ml_compl_t) that contains results from a completed model run + * operation. */ /** diff --git a/platform/linux-generic/include-abi/odp/api/abi/event_types.h b/platform/linux-generic/include-abi/odp/api/abi/event_types.h index 8208ad5a8..01ee66cd3 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/event_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/event_types.h @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2022, Nokia + * Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -36,6 +36,7 @@ typedef enum odp_event_type_t { ODP_EVENT_PACKET_VECTOR = 6, ODP_EVENT_PACKET_TX_COMPL = 7, ODP_EVENT_DMA_COMPL = 8, + ODP_EVENT_ML_COMPL = 9 } odp_event_type_t; typedef enum odp_event_subtype_t { @@ -43,7 +44,9 @@ typedef enum odp_event_subtype_t { ODP_EVENT_PACKET_BASIC = 1, ODP_EVENT_PACKET_CRYPTO = 2, ODP_EVENT_PACKET_IPSEC = 3, - ODP_EVENT_PACKET_COMP = 4 + ODP_EVENT_PACKET_COMP = 4, + ODP_EVENT_ML_COMPL_LOAD = 5, + ODP_EVENT_ML_COMPL_RUN = 6 } odp_event_subtype_t; /** From 0c30d10ab56567e6eaef1a6c4814405bee79e98a Mon Sep 17 00:00:00 2001 From: Petri Savolainen <petri.savolainen@nokia.com> Date: Mon, 30 Oct 2023 13:26:18 +0200 Subject: [PATCH 115/147] api: pool: add ML completion event pool type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added pool type enumeration for ML completion event pool. Signed-off-by: Petri Savolainen <petri.savolainen@nokia.com> Reviewed-by: Jere Leppänen <jere.leppanen@nokia.com> Reviewed-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- include/odp/api/spec/pool_types.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/include/odp/api/spec/pool_types.h b/include/odp/api/spec/pool_types.h index da9f0d9e2..cb3db4737 100644 --- a/include/odp/api/spec/pool_types.h +++ b/include/odp/api/spec/pool_types.h @@ -18,6 +18,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/dma_types.h> +#include <odp/api/ml_types.h> /** @defgroup odp_pool ODP POOL * @{ @@ -417,7 +418,10 @@ typedef enum odp_pool_type_t { ODP_POOL_VECTOR, /** DMA completion event pool */ - ODP_POOL_DMA_COMPL + ODP_POOL_DMA_COMPL, + + /** ML completion event pool */ + ODP_POOL_ML_COMPL } odp_pool_type_t; @@ -891,6 +895,9 @@ typedef struct odp_pool_info_t { /** Copy of pool parameters when pool type is ODP_POOL_DMA_COMPL. */ odp_dma_pool_param_t dma_pool_param; + + /** Copy of pool parameters when pool type is ODP_POOL_ML_COMPL. */ + odp_ml_compl_pool_param_t ml_pool_param; }; /** Additional info for packet pools */ From 10753512c8a4c04350092c5faaa014ad485427b9 Mon Sep 17 00:00:00 2001 From: Lifang Zhang <lifang.zhang@nokia.com> Date: Sat, 2 Dec 2023 07:04:55 +0200 Subject: [PATCH 116/147] linux-gen: pool: add buffer_subtype_set() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ML need to use the function to set buffer subtypes. Signed-off-by: Lifang Zhang <lifang.zhang@nokia.com> Reviewed-by: Jere Leppänen <jere.leppanen@nokia.com> Reviewed-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-generic/include/odp_buffer_internal.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 1cececb99..676b9f116 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -53,6 +53,13 @@ static inline odp_buffer_hdr_t *_odp_buf_hdr(odp_buffer_t buf) return (odp_buffer_hdr_t *)(uintptr_t)buf; } +static inline void _odp_buffer_subtype_set(odp_buffer_t buffer, int subtype) +{ + odp_buffer_hdr_t *buf_hdr = _odp_buf_hdr(buffer); + + buf_hdr->event_hdr.subtype = subtype; +} + #ifdef __cplusplus } #endif From b04851ad21929cc143fa0a6f8c04a191c85f9c6e Mon Sep 17 00:00:00 2001 From: Lifang Zhang <lifang.zhang@nokia.com> Date: Sat, 2 Dec 2023 07:05:44 +0200 Subject: [PATCH 117/147] linux-gen: ml: implement the new ML API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added implementation of the new ML API. Signed-off-by: Lifang Zhang <lifang.zhang@nokia.com> Signed-off-by: Jere Leppänen <jere.leppanen@nokia.com> Reviewed-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- DEPENDENCIES | 29 + config/odp-linux-generic.conf | 33 +- platform/linux-generic/Makefile.am | 14 + .../include/odp/api/plat/event_inlines.h | 2 + .../include/odp_config_internal.h | 9 + .../linux-generic/include/odp_global_data.h | 1 + .../linux-generic/include/odp_init_internal.h | 3 + platform/linux-generic/include/odp_ml_fp16.h | 21 + platform/linux-generic/libodp-linux.pc.in | 2 +- platform/linux-generic/m4/configure.m4 | 4 +- platform/linux-generic/m4/odp_libconfig.m4 | 2 +- platform/linux-generic/m4/odp_ml.m4 | 46 + platform/linux-generic/odp_event.c | 6 + platform/linux-generic/odp_init.c | 15 + platform/linux-generic/odp_ml.c | 2633 +++++++++++++++++ platform/linux-generic/odp_ml_fp16.c | 425 +++ platform/linux-generic/odp_ml_null.c | 229 ++ platform/linux-generic/odp_ml_quantize.c | 79 + platform/linux-generic/odp_pool.c | 9 + platform/linux-generic/odp_system_info.c | 3 + platform/linux-generic/test/inline-timer.conf | 2 +- platform/linux-generic/test/packet_align.conf | 2 +- platform/linux-generic/test/process-mode.conf | 2 +- platform/linux-generic/test/sched-basic.conf | 2 +- platform/linux-generic/test/stash-custom.conf | 2 +- 25 files changed, 3566 insertions(+), 9 deletions(-) create mode 100644 platform/linux-generic/include/odp_ml_fp16.h create mode 100644 platform/linux-generic/m4/odp_ml.m4 create mode 100644 platform/linux-generic/odp_ml.c create mode 100644 platform/linux-generic/odp_ml_fp16.c create mode 100644 platform/linux-generic/odp_ml_null.c create mode 100644 platform/linux-generic/odp_ml_quantize.c diff --git a/DEPENDENCIES b/DEPENDENCIES index 7dbe86489..5af0d4ac3 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -305,6 +305,35 @@ Prerequisites for building the OpenDataPlane (ODP) API which would be the first actual queue in case 5 regular combined queues were configured (zero-indexing). +3.7 Machine Learning API support (optional) + Use ML API for model inferencing. ML implementation uses ONNX Runtime library + (https://github.com/microsoft/onnxruntime). ODP has been tested with ONNX + Runtime version 1.16.3. + +3.7.1 Prebuilt onnxruntime download + Download a default CPU version onnxruntime-linux-x64-*.tgz and unzip it to + any folder. + + $ wget -P ~ https://github.com/microsoft/onnxruntime/releases/download/v<version>/onnxruntime-linux-x64-<version>.tgz + $ mkdir <onnxruntime path> + $ cd <onnxruntime path>/ + $ tar --strip=1 -zxvf ~/onnxruntime-linux-x64-<version>.tgz + +3.7.1 Build onnxruntime from source + $ git clone --recursive https://github.com/Microsoft/onnxruntime.git + $ cd onnxruntime + + # Configure + $ ./build.sh --config RelWithDebInfo --build_shared_lib --parallel + $ tools/ci_build/github/linux/copy_strip_binary.sh -r build/Linux/ -a onnxruntime -l libonnxruntime.so.1.14.0 -c RelWithDebInfo -s . -t <commit id> + $ cp -r build/Linux/onnxruntime/ <onnxruntime path> + +3.7.2 Build ODP with ML support + After installing onnxruntime and example dependencies, ODP can be configured to be + built with ML support by giving onnxruntime path with --with-ort-path. + + $ ../configure --with-ort-path=<onnxruntime path> + 4.0 Packages needed to build API tests CUnit test framework version 2.1-3 is required diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index 2d27752b2..93997ecb3 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.27" +config_file_version = "0.1.28" # System options system: { @@ -365,3 +365,34 @@ ipsec: { async_outbound = 0 } } + +ml: { + # Enable onnxruntime profiling, when enabled, a json file will be + # generated after inference. chrome://tracing/ can be used to check + # the profiling. Use 0 to disable and 1 to enable profiling. + enable_profiling = 0 + + # Choose onnxruntime execution mode, which can be "SEQUENTIAL" or + # "PARALLEL" + execution_mode = "SEQUENTIAL" + + # Set the number of threads used to parallelize the execution of the + # graph across nodes. A value of 0 means onnxruntime will pick a default. + inter_op_num_threads = 0 + + # Set the number of threads used to parallelize the execution within + # a node. A value of 0 means onnxruntime will pick a default. + intra_op_num_threads = 0 + + # Set graph optimization level. Valid values are: + # DISABLE_ALL: disables all optimizations + # ENABLE_BASIC: enables basic optimizations + # ENABLE_EXTENDED: enables basic and extended optimizations + # ENABLE_ALL: enables all available optimizations including layout optimization + graph_optimization_level = "ENABLE_ALL" + + # Serialize the optimized model to disk. When initializing a session + # with the same model, no need to apply optimization anymore, thus + # reducing model startup time. + optimized_model_filepath = "" +} diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index c2aec362e..11cdb4c64 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -13,6 +13,7 @@ AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/default AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/common AM_CPPFLAGS += $(OPENSSL_CPPFLAGS) +AM_CPPFLAGS += $(ORT_CPPFLAGS) AM_CFLAGS += $(AARCH64CRYPTO_CFLAGS) AM_CFLAGS += $(DPDK_CFLAGS) @@ -141,6 +142,7 @@ noinst_HEADERS = \ include/odp_event_validation_internal.h \ include/odp_fdserver_internal.h \ include/odp_forward_typedefs_internal.h \ + include/odp_ml_fp16.h \ include/odp_global_data.h \ include/odp_init_internal.h \ include/odp_ipsec_internal.h \ @@ -229,6 +231,8 @@ __LIB__libodp_linux_la_SOURCES = \ odp_ishmphy.c \ odp_ishmpool.c \ odp_libconfig.c \ + odp_ml_fp16.c \ + odp_ml_quantize.c \ odp_name_table.c \ odp_packet.c \ odp_packet_vector.c \ @@ -298,6 +302,15 @@ __LIB__libodp_linux_la_SOURCES += \ endif endif endif + +if WITH_ML +__LIB__libodp_linux_la_SOURCES += \ + odp_ml.c +else +__LIB__libodp_linux_la_SOURCES += \ + odp_ml_null.c +endif + if ODP_ABI_COMPAT __LIB__libodp_linux_la_SOURCES += \ odp_atomic_api.c \ @@ -473,6 +486,7 @@ __LIB__libodp_linux_la_LIBADD += $(PTHREAD_LIBS) __LIB__libodp_linux_la_LIBADD += $(TIMER_LIBS) __LIB__libodp_linux_la_LIBADD += $(LIBXDP_LIBS) __LIB__libodp_linux_la_LIBADD += $(IPSEC_MB_LIBS) +__LIB__libodp_linux_la_LIBADD += $(ORT_LIBS) if ODP_PKTIO_PCAP __LIB__libodp_linux_la_LIBADD += $(PCAP_LIBS) diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index d30c6acbb..990575166 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -99,6 +99,7 @@ _ODP_INLINE void *odp_event_user_area(odp_event_t event) switch (type) { case ODP_EVENT_BUFFER: + case ODP_EVENT_ML_COMPL: case ODP_EVENT_DMA_COMPL: return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); case ODP_EVENT_PACKET: @@ -121,6 +122,7 @@ _ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) switch (type) { case ODP_EVENT_BUFFER: case ODP_EVENT_DMA_COMPL: + case ODP_EVENT_ML_COMPL: *flag = -1; return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); case ODP_EVENT_PACKET: diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 8fd8c4be7..89d89936c 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -199,6 +199,15 @@ extern "C" { /* Enable timer scan performance benchmark. This works with inline enabled. */ #define CONFIG_TIMER_PROFILE_INLINE 0 +/* Maximum number of ML models that can be created or loaded. */ +#define CONFIG_ML_MAX_MODELS 4 + +/* Maximum number of inputs for a ML model. */ +#define CONFIG_ML_MAX_INPUTS 4 + +/* Maximum number of outputs for a ML model. */ +#define CONFIG_ML_MAX_OUTPUTS 4 + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index f00e155de..2a87192df 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -80,6 +80,7 @@ typedef struct odp_global_data_ro_t { uint8_t ipsec; uint8_t stash; uint8_t traffic_mngr; + uint8_t ml; } disable; diff --git a/platform/linux-generic/include/odp_init_internal.h b/platform/linux-generic/include/odp_init_internal.h index 24e8346ad..ca5d68c87 100644 --- a/platform/linux-generic/include/odp_init_internal.h +++ b/platform/linux-generic/include/odp_init_internal.h @@ -105,6 +105,9 @@ int _odp_stash_term_global(void); int _odp_dma_init_global(void); int _odp_dma_term_global(void); +int _odp_ml_init_global(void); +int _odp_ml_term_global(void); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_ml_fp16.h b/platform/linux-generic/include/odp_ml_fp16.h new file mode 100644 index 000000000..5294a7c0b --- /dev/null +++ b/platform/linux-generic/include/odp_ml_fp16.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ML_FP16_H_ +#define ODP_ML_FP16_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +uint16_t _odp_float32_to_float16(float x); +float _odp_float16_to_float32(uint16_t f16); +uint16_t _odp_float32_to_bfloat16(float x); +float _odp_bfloat16_to_float32(uint16_t f16); + +#ifdef __cplusplus +} +#endif + +#endif /* ODP_ML_FP16_H_ */ diff --git a/platform/linux-generic/libodp-linux.pc.in b/platform/linux-generic/libodp-linux.pc.in index 05ba5b9d6..62589c1a3 100644 --- a/platform/linux-generic/libodp-linux.pc.in +++ b/platform/linux-generic/libodp-linux.pc.in @@ -8,5 +8,5 @@ Description: The ODP packet processing engine Version: @PKGCONFIG_VERSION@ Requires.private: libconfig@AARCH64CRYPTO_PKG@ Libs: -L${libdir} -l@ODP_LIB_NAME@ @ATOMIC_LIBS_NON_ABI_COMPAT@ -Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ @LIBXDP_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ @IPSEC_MB_LIBS@ +Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ @LIBXDP_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ @IPSEC_MB_LIBS@ @ORT_LIBS@ Cflags: -I${includedir} diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 61b65540f..161bf48c5 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -31,10 +31,11 @@ m4_include([platform/linux-generic/m4/odp_pcapng.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) m4_include([platform/linux-generic/m4/odp_wfe.m4]) m4_include([platform/linux-generic/m4/odp_xdp.m4]) +m4_include([platform/linux-generic/m4/odp_ml.m4]) ODP_EVENT_VALIDATION ODP_SCHEDULER -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${IPSEC_MB_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${LIBXDP_LIBS}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${IPSEC_MB_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${LIBXDP_LIBS} ${ORT_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. @@ -46,6 +47,7 @@ AS_VAR_APPEND([PLAT_CFG_TEXT], [" pcap: ${have_pcap} pcapng: ${have_pcapng} wfe_locks: ${use_wfe_locks} + ml_support: ${ml_support} default_config_path: ${default_config_path}"]) # Ignore Clang specific errors about fields with variable sized type not at the diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4 index a6d19f661..77095e0fe 100644 --- a/platform/linux-generic/m4/odp_libconfig.m4 +++ b/platform/linux-generic/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [27]) +m4_define([_odp_config_version_minor], [28]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-generic/m4/odp_ml.m4 b/platform/linux-generic/m4/odp_ml.m4 new file mode 100644 index 000000000..a7b9a4fd6 --- /dev/null +++ b/platform/linux-generic/m4/odp_ml.m4 @@ -0,0 +1,46 @@ +########################################################################## +# Onnxruntime library path and name +########################################################################## +# Optional configure parameter for a non-standard install prefix of onnxruntime +AC_ARG_WITH([ort-path], + [AS_HELP_STRING([--with-ort-path=DIR], + [path to onnxruntime libs and headers [default=system]])], + [ort_path_given=yes + ORT_CPPFLAGS="-I$withval/include" + ORT_LIBS="-L$withval/lib" + ORT_RPATH="-R$withval/lib"], + []) + +########################################################################## +# Save and set temporary compilation flags +########################################################################## +OLD_CPPFLAGS=$CPPFLAGS +OLD_LIBS=$LIBS +CPPFLAGS="$ORT_CPPFLAGS $CPPFLAGS" +LIBS="$ORT_LIBS $LIBS" + +######################################################################### +# If ort is available, enable ML API +######################################################################### +ml_support=no +AC_CHECK_HEADERS([onnxruntime_c_api.h], + [AC_CHECK_LIB(onnxruntime, OrtGetApiBase, [ml_support=yes], [], [])], + [AS_IF([test "x$ort_path_given" = "xyes"], + [AC_MSG_ERROR([ort not found at the specified path (--with-ort-path)])])]) + +AS_IF([test "x$ml_support" != "xno"], + [ORT_LIBS="$ORT_RPATH $ORT_LIBS -lonnxruntime -lm"], + [ORT_CPPFLAGS="" ORT_LIBS="-lm"]) + +AC_CONFIG_COMMANDS_PRE([dnl +AM_CONDITIONAL([WITH_ML], [test x$ml_support = xyes ]) +]) + +########################################################################## +# Restore old saved variables +########################################################################## +LIBS=$OLD_LIBS +CPPFLAGS=$OLD_CPPFLAGS + +AC_SUBST([ORT_CPPFLAGS]) +AC_SUBST([ORT_LIBS]) diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c index 9ec4b4bfb..f3644f02b 100644 --- a/platform/linux-generic/odp_event.c +++ b/platform/linux-generic/odp_event.c @@ -12,6 +12,7 @@ #include <odp/api/packet.h> #include <odp/api/timer.h> #include <odp/api/pool.h> +#include <odp/api/ml.h> #include <odp_buffer_internal.h> #include <odp_ipsec_internal.h> @@ -69,6 +70,9 @@ static inline void event_free(odp_event_t event, _odp_ev_id_t id) case ODP_EVENT_DMA_COMPL: odp_dma_compl_free(odp_dma_compl_from_event(event)); break; + case ODP_EVENT_ML_COMPL: + odp_ml_compl_free(odp_ml_compl_from_event(event)); + break; default: _ODP_ABORT("Invalid event type: %d\n", odp_event_type(event)); } @@ -117,6 +121,8 @@ int odp_event_is_valid(odp_event_t event) /* Fall through */ case ODP_EVENT_DMA_COMPL: /* Fall through */ + case ODP_EVENT_ML_COMPL: + /* Fall through */ case ODP_EVENT_PACKET_TX_COMPL: break; default: diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 05b693c94..795252df1 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -51,6 +51,7 @@ enum init_stage { IPSEC_SAD_INIT, IPSEC_INIT, DMA_INIT, + ML_INIT, ALL_INIT /* All init stages completed */ }; @@ -95,6 +96,7 @@ static void disable_features(odp_global_data_ro_t *global_ro, global_ro->disable.traffic_mngr = init_param->not_used.feat.tm; global_ro->disable.compress = init_param->not_used.feat.compress; + global_ro->disable.ml = init_param->not_used.feat.ml; } void odp_init_param_init(odp_init_t *param) @@ -145,6 +147,13 @@ static int term_global(enum init_stage stage) switch (stage) { case ALL_INIT: + case ML_INIT: + if (_odp_ml_term_global()) { + _ODP_ERR("ODP ML term failed.\n"); + rc = -1; + } + /* Fall through */ + case DMA_INIT: if (_odp_dma_term_global()) { _ODP_ERR("ODP DMA term failed.\n"); @@ -509,6 +518,12 @@ int odp_init_global(odp_instance_t *instance, } stage = DMA_INIT; + if (_odp_ml_init_global()) { + _ODP_ERR("ODP ML init failed.\n"); + goto init_failed; + } + stage = ML_INIT; + *instance = (odp_instance_t)odp_global_ro.main_pid; return 0; diff --git a/platform/linux-generic/odp_ml.c b/platform/linux-generic/odp_ml.c new file mode 100644 index 000000000..fda06e7cb --- /dev/null +++ b/platform/linux-generic/odp_ml.c @@ -0,0 +1,2633 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp/api/ml.h> +#include <odp/api/queue.h> +#include <odp/api/plat/event_inline_types.h> + +#include <odp_global_data.h> +#include <odp_debug_internal.h> +#include <odp_init_internal.h> +#include <odp_pool_internal.h> +#include <odp_config_internal.h> +#include <odp_macros_internal.h> +#include <odp_libconfig_internal.h> + +#include <onnxruntime_c_api.h> + +#include <stdint.h> +#include <stdio.h> +#include <inttypes.h> +#include <string.h> + +#define ML_MAX_IO_SEGS UINT32_MAX +#define ML_MAX_COMPL_ID 32 +#define ML_MAX_CONFIG_STR_LEN 65 +#define ML_MAX_MODEL_SIZE (1024 * 1024 * 1024) +#define ML_MAX_MODELS_CREATED CONFIG_ML_MAX_MODELS +#define ML_MAX_MODELS_LOADED CONFIG_ML_MAX_MODELS + +/* Error codes */ +enum { + /* Feature not supported */ + ML_FEATURE_NOT_SUPPORTED = 1, + + /* Model is not created */ + ML_NOT_CREATED, + + /* Model was not loaded */ + ML_NOT_LOADED, + + /* Model has already loaded */ + ML_LOADED, + + /* Bad input */ + ML_BAD_INPUT, + + /* Fail from underlying library onnxruntime */ + ML_LIB_FAILED, + + /* Bad output */ + ML_BAD_OUTPUT, + + /* Bad handle */ + ML_BAD_HDL +}; + +typedef struct ort_run_opts_t { + int enable_profiling; + + ExecutionMode execution_mode; + + int inter_op_num_threads; + + int intra_op_num_threads; + + GraphOptimizationLevel graph_opt_level; + + char opt_model_filepath[ML_MAX_CONFIG_STR_LEN]; +} ort_run_opts_t; + +typedef struct ml_input_t { + /* Combined input start address */ + void *addr; + /* Data size in bytes */ + uint64_t size; +} ml_input_t; + +/* Onnxruntime model info */ +typedef struct ml_model_t { + /* Guards state, which must be accessed atomically */ + odp_ticketlock_t lock; + + enum { + ML_STATE_FREE = 0, /* Not allocated */ + ML_STATE_CREATED, /* Model is created */ + ML_STATE_LOADED, /* Model is loaded */ + ML_STATE_INFERENCING, /* Model is inferencing */ + } state; + + OrtSession *session; + OrtSessionOptions *session_opts; + uint32_t max_compl_id; + odp_atomic_u32_t compl_status[ML_MAX_COMPL_ID]; + + odp_ml_model_info_t info; + odp_ml_input_info_t input_info[CONFIG_ML_MAX_INPUTS]; + uint64_t input_sizes[CONFIG_ML_MAX_INPUTS]; + odp_ml_output_info_t output_info[CONFIG_ML_MAX_OUTPUTS]; + uint64_t output_sizes[CONFIG_ML_MAX_OUTPUTS]; + + struct { + void *user_ptr; + } result[ML_MAX_COMPL_ID]; +} ml_model_t; + +typedef struct ml_global_t { + odp_shm_t shm; + + odp_ml_capability_t capa; + odp_ml_config_t ml_config; + + odp_pool_param_t pool_param; + + const OrtApi *ort_api; + OrtEnv *env; + ort_run_opts_t ort_run_opts; + + ml_model_t models[ML_MAX_MODELS_CREATED]; + +} ml_global_t; + +static ml_global_t *_odp_ml_glb; + +static inline ml_model_t *ml_model_from_handle(odp_ml_model_t model) +{ + return (ml_model_t *)(uintptr_t)model; +} + +int odp_ml_capability(odp_ml_capability_t *capa) +{ + odp_pool_capability_t pool_capa; + + memset(capa, 0, sizeof(odp_ml_capability_t)); + + if (odp_global_ro.disable.ml) { + _ODP_PRINT("ML is disabled\n"); + return 0; + } + + capa->max_model_size = ML_MAX_MODEL_SIZE; + capa->max_models = ML_MAX_MODELS_CREATED; + capa->max_models_loaded = ML_MAX_MODELS_LOADED; + capa->max_compl_id = ML_MAX_COMPL_ID; + capa->max_inputs = CONFIG_ML_MAX_INPUTS; + capa->max_outputs = CONFIG_ML_MAX_OUTPUTS; + capa->max_segs_per_input = ML_MAX_IO_SEGS; + capa->max_segs_per_output = ML_MAX_IO_SEGS; + capa->min_input_align = 1; + capa->min_output_align = 1; + + capa->load.compl_mode_mask = ODP_ML_COMPL_MODE_SYNC | + ODP_ML_COMPL_MODE_POLL | + ODP_ML_COMPL_MODE_EVENT; + capa->load.compl_queue_plain = 1; + capa->load.compl_queue_sched = 1; + + capa->run.compl_mode_mask = ODP_ML_COMPL_MODE_SYNC | + ODP_ML_COMPL_MODE_POLL | + ODP_ML_COMPL_MODE_EVENT; + capa->run.compl_queue_plain = 1; + capa->run.compl_queue_sched = 1; + + if (odp_pool_capability(&pool_capa)) { + _ODP_ERR("Pool capability failed\n"); + return -1; + } + + capa->pool.max_pools = pool_capa.buf.max_pools; + capa->pool.max_num = pool_capa.buf.max_num; + capa->pool.max_uarea_size = pool_capa.buf.max_uarea_size; + capa->pool.uarea_persistence = pool_capa.buf.uarea_persistence; + capa->pool.max_cache_size = pool_capa.buf.max_cache_size; + capa->pool.min_cache_size = pool_capa.buf.min_cache_size; + + return 0; +} + +void odp_ml_config_init(odp_ml_config_t *config) +{ + memset(config, 0, sizeof(odp_ml_config_t)); + config->max_models_created = 1; + config->max_models_loaded = 1; +} + +int odp_ml_config(const odp_ml_config_t *config) +{ + if (!config) { + _ODP_ERR("Error: config must not be NULL\n"); + return -1; + } + + if (config->max_model_size == 0 || config->max_models_created == 0 || + config->max_models_loaded == 0) { + _ODP_ERR("Error: max_model_size, max_models_created and max_models_loaded" + " must be bigger than 0\n"); + return -1; + } + + if (config->max_models_loaded > config->max_models_created) { + _ODP_ERR("Error: max_models_loaded %d exceeds max_models_created %d\n", + config->max_models_loaded, config->max_models_created); + return -1; + } + + if (config->max_models_created > ML_MAX_MODELS_CREATED) { + _ODP_ERR("Error: max_models_created %d exceeds maximum number" + " of models that can be created in this driver %d\n", + config->max_models_created, ML_MAX_MODELS_CREATED); + return -1; + } + + if (config->max_models_loaded > ML_MAX_MODELS_LOADED) { + _ODP_ERR("Error: max_models_loaded %d exceeds maximum number" + " of models that can be loaded in this driver %d\n", + config->max_models_loaded, ML_MAX_MODELS_LOADED); + return -1; + } + + if (config->max_model_size > ML_MAX_MODEL_SIZE) { + _ODP_ERR("max_model_size %" PRIu64 " exceeds supported maximum model size %d\n", + config->max_model_size, ML_MAX_MODEL_SIZE); + return -1; + } + + _odp_ml_glb->ml_config = *config; + return 0; +} + +void odp_ml_model_param_init(odp_ml_model_param_t *param) +{ + memset(param, 0, sizeof(odp_ml_model_param_t)); +} + +static int check_ortstatus(OrtStatus * const status) +{ + if (status != NULL) { + const char *msg = _odp_ml_glb->ort_api->GetErrorMessage(status); + + _ODP_ERR("%s\n", msg); + _odp_ml_glb->ort_api->ReleaseStatus(status); + return -1; + } + + return 0; +} + +/* Get model input and output count */ +static int get_model_io_count(OrtSession *model, uint32_t *num_inputs, uint32_t *num_outputs) +{ + size_t num = 0; + OrtStatus *status = NULL; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + status = ort_api->SessionGetInputCount(model, &num); + if (check_ortstatus(status)) { + _ODP_ERR("Get model input count failed\n"); + return -1; + } + + *num_inputs = num; + _ODP_DBG("num_inputs: %u\n", *num_inputs); + + status = ort_api->SessionGetOutputCount(model, &num); + if (check_ortstatus(status)) { + _ODP_ERR("Get model output count failed\n"); + return -1; + } + + *num_outputs = num; + _ODP_DBG("num_outputs: %u\n", *num_outputs); + + return 0; +} + +static odp_ml_data_type_t onnx_dtype_to_odp_dtype(ONNXTensorElementDataType onnx_dtype) +{ + switch (onnx_dtype) { + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: + return ODP_ML_DATA_TYPE_FP32; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: + return ODP_ML_DATA_TYPE_UINT8; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: + return ODP_ML_DATA_TYPE_INT8; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: + return ODP_ML_DATA_TYPE_UINT16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: + return ODP_ML_DATA_TYPE_INT16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: + return ODP_ML_DATA_TYPE_INT32; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: + return ODP_ML_DATA_TYPE_UINT32; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: + return ODP_ML_DATA_TYPE_INT64; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: + return ODP_ML_DATA_TYPE_UINT64; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: + return ODP_ML_DATA_TYPE_FP16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: + return ODP_ML_DATA_TYPE_BFP16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: + return ODP_ML_DATA_TYPE_FP64; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: + /* Fall through */ + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64: + /* Fall through */ + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128: + /* Fall through */ + default: + _ODP_ERR("onnx_dtype %d not supported by odp_ml\n", onnx_dtype); + return ODP_ML_DATA_TYPE_NONE; + } +} + +/* Get the size of given odp_ml_data_type_t in bytes */ +static uint32_t size_of_odp_ml_data_type(odp_ml_data_type_t data_type) +{ + switch (data_type) { + case ODP_ML_DATA_TYPE_NONE: + return 0; + + case ODP_ML_DATA_TYPE_INT8: + /* Fall through */ + case ODP_ML_DATA_TYPE_UINT8: + return 1; + + case ODP_ML_DATA_TYPE_INT16: + /* Fall through */ + case ODP_ML_DATA_TYPE_UINT16: + /* Fall through */ + case ODP_ML_DATA_TYPE_FP16: + /* Fall through */ + case ODP_ML_DATA_TYPE_BFP16: + return 2; + + case ODP_ML_DATA_TYPE_INT24: + /* Fall through */ + case ODP_ML_DATA_TYPE_UINT24: + return 3; + + case ODP_ML_DATA_TYPE_INT32: + /* Fall through */ + case ODP_ML_DATA_TYPE_UINT32: + /* Fall through */ + case ODP_ML_DATA_TYPE_FP32: + return 4; + + case ODP_ML_DATA_TYPE_INT64: + /* Fall through */ + case ODP_ML_DATA_TYPE_UINT64: + /* Fall through */ + case ODP_ML_DATA_TYPE_FP64: + return 8; + + default: + return 0; + } +} + +static int get_shape(int64_t dims[], odp_ml_shape_info_t *shape) +{ + uint32_t dyn_cnt = 0; + + for (uint32_t i = 0; i < shape->num_dim; i++) { + if (dims[i] == 0) { + _ODP_ERR("Dimension value: %" PRId64 " must be at least 1\n", dims[i]); + return -1; + } else if (dims[i] == -1) { /* Symbolic dimension */ + dyn_cnt++; + shape->dim[i] = ODP_ML_DIM_DYNAMIC; + shape->dim_min[i] = 0; /*unknown*/ + shape->dim_max[i] = 0; /*unknown*/ + } else if (dims[i] > 0 && dims[i] < UINT32_MAX) { + shape->dim[i] = dims[i]; + shape->dim_min[i] = dims[i]; + shape->dim_max[i] = dims[i]; + } else { + _ODP_ERR("Dimension value: %" PRId64 " invalid\n", dims[i]); + return -1; + } + } + + if (dyn_cnt == 0) { + shape->type = ODP_ML_SHAPE_STATIC; + } else if (dyn_cnt == 1) { + shape->type = ODP_ML_SHAPE_BATCH; + } else { + _ODP_ERR("Data shape type not supported by ODP\n"); + return -1; + } + + return 0; +} + +static inline void calculate_model_io_size(const odp_ml_shape_info_t *shape, uint64_t *size) +{ + /* Calculate the data size in bytes of this tensor, 0 for tensors with + * dynamic batch sizes */ + for (size_t i = 0; i < shape->num_dim; i++) { + /* Skip dynamic dimension size */ + if (shape->dim[i] == ODP_ML_DIM_DYNAMIC) { + *size = 0; + break; + } + (*size) *= shape->dim[i]; + } +} + +static int get_model_io_type_shape_size(OrtTypeInfo *type_info, odp_ml_shape_info_t *shape, + odp_ml_data_type_t *data_type, uint32_t *data_type_size, + uint64_t *size) +{ + ONNXTensorElementDataType tensor_type; + const OrtTensorTypeAndShapeInfo *tensor_info; + size_t num_dim = 0; + OrtStatus *status = NULL; + int64_t dims[ODP_ML_MAX_DIMS] = {0}; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + status = ort_api->CastTypeInfoToTensorInfo(type_info, &tensor_info); + if (check_ortstatus(status)) { + _ODP_ERR("CastTypeInfoToTensorInfo failed\n"); + return -1; + } + + status = ort_api->GetTensorElementType(tensor_info, &tensor_type); + if (check_ortstatus(status)) { + _ODP_ERR("GetTensorElementType failed\n"); + return -1; + } + + *data_type = onnx_dtype_to_odp_dtype(tensor_type); + if (*data_type == ODP_ML_DATA_TYPE_NONE) /* Type not supported by odp */ + return -1; + + status = ort_api->GetDimensionsCount(tensor_info, &num_dim); + if (check_ortstatus(status)) { + _ODP_ERR("GetDimensionsCount failed\n"); + return -1; + } + + if (num_dim > ODP_ML_MAX_DIMS) { + _ODP_ERR("Number of dimensions: %zu exceeds supported maximum number" + " of dimensions: %d\n", num_dim, ODP_ML_MAX_DIMS); + return -1; + } + shape->num_dim = num_dim; + + status = ort_api->GetDimensions(tensor_info, dims, num_dim); + if (check_ortstatus(status)) { + _ODP_ERR("GetDimensions failed\n"); + return -1; + } + + if (get_shape(dims, shape)) + return -1; + + *data_type_size = size_of_odp_ml_data_type(*data_type); + + *size = *data_type_size; + calculate_model_io_size(shape, size); + + return 0; +} + +/* Get model input and output info */ +static int get_model_io_info(OrtSession *session, ml_model_t *mdl, + const odp_ml_model_param_t *param) +{ + char *name; + OrtTypeInfo *type_info; + const odp_ml_data_format_t *data_format; + OrtStatus *status = NULL; + OrtAllocator *allocator = NULL; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + odp_ml_input_info_t *input_info = mdl->input_info; + odp_ml_output_info_t *output_info = mdl->output_info; + + status = ort_api->GetAllocatorWithDefaultOptions(&allocator); + if (check_ortstatus(status)) { + _ODP_ERR("GetAllocatorWithDefaultOptions failed\n"); + return -1; + } + + /* Retrieve info about input array. */ + memset(input_info, 0, sizeof(mdl->input_info)); + for (uint32_t i = 0; i < mdl->info.num_inputs; i++) { + name = NULL; + status = ort_api->SessionGetInputName(session, i, allocator, &name); + if (check_ortstatus(status)) { + _ODP_ERR("Get %uth input name failed\n", i); + return -1; + } + + strncpy(input_info[i].name, name, ODP_ML_MODEL_IO_NAME_LEN - 1); + input_info[i].name[ODP_ML_MODEL_IO_NAME_LEN - 1] = 0; + + /* Free memory allocated by SessionGetInputName */ + status = ort_api->AllocatorFree(allocator, name); + if (check_ortstatus(status)) { + _ODP_ERR("AllocatorFree %uth input_name failed\n", i); + return -1; + } + + if (param->extra_info.num_inputs) { + data_format = ¶m->extra_info.input_format[i]; + + input_info[i].shape = data_format->shape; + input_info[i].data_type = data_format->data_type; + input_info[i].data_type_size = data_format->data_type_size; + + mdl->input_sizes[i] = input_info[i].data_type_size; + calculate_model_io_size(&data_format->shape, &mdl->input_sizes[i]); + continue; + } + + type_info = NULL; + status = ort_api->SessionGetInputTypeInfo(session, i, &type_info); + if (check_ortstatus(status)) { + _ODP_ERR("SessionGetInputTypeInfo failed\n"); + return -1; + } + + if (get_model_io_type_shape_size(type_info, &input_info[i].shape, + &input_info[i].data_type, + &input_info[i].data_type_size, + &mdl->input_sizes[i])) { + _ODP_ERR("get_model_io_type_shape_size() for input failed\n"); + ort_api->ReleaseTypeInfo(type_info); + return -1; + } + + ort_api->ReleaseTypeInfo(type_info); + } + + /* Retrieve info about output array. */ + memset(output_info, 0, sizeof(mdl->output_info)); + for (uint32_t i = 0; i < mdl->info.num_outputs; i++) { + name = NULL; + status = ort_api->SessionGetOutputName(session, i, allocator, &name); + if (check_ortstatus(status)) { + _ODP_ERR("Get %uth output name failed\n", i); + return -1; + } + + strncpy(output_info[i].name, name, ODP_ML_MODEL_IO_NAME_LEN - 1); + output_info[i].name[ODP_ML_MODEL_IO_NAME_LEN - 1] = 0; + + /* Free memory allocated by SessionGetOutputName */ + status = ort_api->AllocatorFree(allocator, name); + if (check_ortstatus(status)) { + _ODP_ERR("AllocatorFree %uth output_name failed\n", i); + return -1; + } + + if (param->extra_info.num_outputs) { + data_format = ¶m->extra_info.output_format[i]; + + output_info[i].shape = data_format->shape; + output_info[i].data_type = data_format->data_type; + output_info[i].data_type_size = data_format->data_type_size; + + mdl->output_sizes[i] = output_info[i].data_type_size; + calculate_model_io_size(&data_format->shape, &mdl->output_sizes[i]); + continue; + } + + type_info = NULL; + status = ort_api->SessionGetOutputTypeInfo(session, i, &type_info); + if (check_ortstatus(status)) { + _ODP_ERR("SessionGetOutputTypeInfo failed\n"); + return -1; + } + + if (get_model_io_type_shape_size(type_info, &output_info[i].shape, + &output_info[i].data_type, + &output_info[i].data_type_size, + &mdl->output_sizes[i])) { + _ODP_ERR("get_model_io_type_shape_size() for output failed\n"); + ort_api->ReleaseTypeInfo(type_info); + return -1; + } + + ort_api->ReleaseTypeInfo(type_info); + } + + return 0; +} + +static inline int check_model_io_num(const odp_ml_model_param_t *param, + uint32_t num_inputs, uint32_t num_outputs) +{ + /* Make sure the number of inputs/outputs not exceeding the supported + * model max inputs/outputs */ + if (num_inputs > CONFIG_ML_MAX_INPUTS) { + _ODP_ERR("The model's number of inputs %u exceeds the maximum " + "number of inputs supported in a model %u\n", + num_inputs, CONFIG_ML_MAX_INPUTS); + return -1; + } + + if (num_outputs > CONFIG_ML_MAX_OUTPUTS) { + _ODP_ERR("The model's number of outputs %u exceeds the maximum " + "number of outputs supported in a model %u\n", + num_outputs, CONFIG_ML_MAX_OUTPUTS); + + return -1; + } + + /* Make sure the numbers of inputs/outputs provided in the extra_info of + * param match the numbers defined in model metadata. */ + if (param->extra_info.num_inputs && + param->extra_info.num_inputs != num_inputs) { + _ODP_ERR("Provided param->extra_info.num_inputs %u does not match the" + " number of inputs defined in model metadata: %u\n", + param->extra_info.num_inputs, num_inputs); + return -1; + } + + if (param->extra_info.num_outputs && param->extra_info.num_outputs != num_outputs) { + _ODP_ERR("Provided param->extra_info.num_outputs %u does not match the" + " number of outputs defined in model metadata: %u\n", + param->extra_info.num_outputs, num_outputs); + return -1; + } + + if (param->extra_info.num_inputs && !param->extra_info.input_format) { + _ODP_ERR("num_inputs is provided but not input_format in param->extra_info\n"); + return -1; + } + + if (param->extra_info.num_outputs && !param->extra_info.output_format) { + _ODP_ERR("num_outputs is provided but not output_format in param->extra_info\n"); + return -1; + } + + return 0; +} + +static int create_ort_model(const odp_ml_model_param_t *param, OrtSession **session, + ml_model_t *mdl, OrtSessionOptions *session_opts) +{ + OrtStatus *status; + int64_t model_version; + uint32_t num_inputs = 0; + uint32_t num_outputs = 0; + OrtModelMetadata *metadata = {0}; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + status = ort_api->CreateSessionFromArray(_odp_ml_glb->env, + param->model, + param->size, + session_opts, + session); + if (check_ortstatus(status) || !(*session)) { + _ODP_ERR("CreateSessionFromArray failed\n"); + return -1; + } + + if (get_model_io_count(*session, &num_inputs, &num_outputs)) { + _ODP_ERR("get_model_io_count() failed\n"); + ort_api->ReleaseSession(*session); + return -1; + } + + if (check_model_io_num(param, num_inputs, num_outputs)) { + ort_api->ReleaseSession(*session); + return -1; + } + + mdl->max_compl_id = param->max_compl_id; + mdl->info.num_inputs = num_inputs; + mdl->info.num_outputs = num_outputs; + + /* Get metadata */ + status = ort_api->SessionGetModelMetadata(*session, &metadata); + if (check_ortstatus(status) || !metadata) { + _ODP_ERR("SessionGetModelMetadata failed\n"); + ort_api->ReleaseSession(*session); + return -1; + } + + /* Get model version */ + status = ort_api->ModelMetadataGetVersion(metadata, &model_version); + if (check_ortstatus(status)) { + _ODP_ERR("ModelMetadataGetVersion failed\n"); + ort_api->ReleaseModelMetadata(metadata); + ort_api->ReleaseSession(*session); + return -1; + } + mdl->info.model_version = model_version; + mdl->info.interface_version = 0; + + if (get_model_io_info(*session, mdl, param)) { + _ODP_ERR("get_model_io_info() failed\n"); + ort_api->ReleaseModelMetadata(metadata); + ort_api->ReleaseSession(*session); + return -1; + } + + ort_api->ReleaseModelMetadata(metadata); + return 0; +} + +static int set_ort_run_opts(const char *name, OrtSessionOptions *se_opts) +{ + OrtStatus *status; + ort_run_opts_t *opts = &_odp_ml_glb->ort_run_opts; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + if (opts->enable_profiling) { + status = ort_api->EnableProfiling(se_opts, name); + if (check_ortstatus(status)) { + _ODP_ERR("Enable profiling failed\n"); + return -1; + } + } + + status = ort_api->SetSessionExecutionMode(se_opts, opts->execution_mode); + if (check_ortstatus(status)) { + _ODP_ERR("SetSessionExecutionMode failed\n"); + return -1; + } + + if (opts->intra_op_num_threads) { + status = ort_api->SetIntraOpNumThreads(se_opts, opts->intra_op_num_threads); + if (check_ortstatus(status)) { + _ODP_ERR("SetIntraOpNumThreads failed\n"); + return -1; + } + } + + if (opts->inter_op_num_threads) { + status = ort_api->SetInterOpNumThreads(se_opts, opts->inter_op_num_threads); + if (check_ortstatus(status)) { + _ODP_ERR("SetInterOpNumThreads failed\n"); + return -1; + } + } + + status = ort_api->SetSessionGraphOptimizationLevel(se_opts, opts->graph_opt_level); + if (check_ortstatus(status)) { + _ODP_ERR("SetSessionGraphOptimizationLevel failed\n"); + return -1; + } + + /* Optimized model file path is not provided */ + if (opts->opt_model_filepath[0] == '\0') + return 0; + + status = ort_api->SetOptimizedModelFilePath(se_opts, opts->opt_model_filepath); + if (check_ortstatus(status)) { + _ODP_ERR("SetOptimizedModelFilePath failed\n"); + return -1; + } + + return 0; +} + +static inline void reset_mdl_info_sizes(ml_model_t *mdl) +{ + memset(&mdl->info, 0, sizeof(odp_ml_model_info_t)); + memset(mdl->input_info, 0, sizeof(mdl->input_info)); + memset(mdl->output_info, 0, sizeof(mdl->output_info)); + memset(mdl->input_sizes, 0, sizeof(mdl->input_sizes)); + memset(mdl->output_sizes, 0, sizeof(mdl->output_sizes)); +} + +static int check_io_shape(ml_model_t *mdl) +{ + odp_ml_shape_info_t *shape; + + for (uint32_t i = 0; i < mdl->info.num_inputs; i++) { + shape = &mdl->input_info[i].shape; + + if (shape->type == ODP_ML_SHAPE_NONE) { + _ODP_ERR("Undefined shape type for model input[%u]\n", i); + return -1; + } + + if (shape->type == ODP_ML_SHAPE_STATIC) + continue; + + /* shape->type == ODP_ML_SHAPE_BATCH */ + for (uint32_t j = 0; j < shape->num_dim; j++) { + if (shape->dim[j] == ODP_ML_DIM_DYNAMIC && !shape->dim_max[j]) { + _ODP_ERR("Missing dim_max[%u] for dynamic sized input[%u], please" + " provide via the extra_info of model param\n", j, i); + return -1; + } + } + } + + for (uint32_t i = 0; i < mdl->info.num_outputs; i++) { + if (mdl->output_info[i].shape.type == ODP_ML_SHAPE_NONE) { + _ODP_ERR("Undefined shape type for model output[%u]\n", i); + return -1; + } + } + + return 0; +} + +odp_ml_model_t odp_ml_model_create(const char *name, const odp_ml_model_param_t *param) +{ + OrtStatus *status; + odp_ml_model_info_t *info; + OrtSessionOptions *session_opts; + uint32_t i = 0; + ml_model_t *mdl = NULL; + OrtSession *session = NULL; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + if (odp_unlikely(odp_global_ro.disable.ml)) { + _ODP_ERR("ML is disabled\n"); + return ODP_ML_MODEL_INVALID; + } + + if (odp_unlikely(param->size > _odp_ml_glb->ml_config.max_model_size)) { + _ODP_ERR("Model size %" PRIu64 " exceeds maximum model size configured %" PRIu64 "\n", + param->size, _odp_ml_glb->ml_config.max_model_size); + return ODP_ML_MODEL_INVALID; + } + + if (odp_unlikely(!param->size || !param->model)) { + _ODP_ERR("Invalid model param: param->model: %p, param->size: %" PRIu64 "\n", + param->model, param->size); + return ODP_ML_MODEL_INVALID; + } + + if (odp_unlikely(param->max_compl_id > ML_MAX_COMPL_ID)) { + _ODP_ERR("param->max_compl_id: %u exceeds maximum completion id supported: %d\n", + param->max_compl_id, ML_MAX_COMPL_ID); + return ODP_ML_MODEL_INVALID; + } + + /* Find an emtpy slot to store the new model */ + for (i = 0; i < ML_MAX_MODELS_CREATED; i++) { + if (_odp_ml_glb->models[i].state) + continue; + + odp_ticketlock_lock(&_odp_ml_glb->models[i].lock); + + if (_odp_ml_glb->models[i].state) { + odp_ticketlock_unlock(&_odp_ml_glb->models[i].lock); + continue; + } + + mdl = &_odp_ml_glb->models[i]; + break; + } + + if (i == ML_MAX_MODELS_CREATED) { + _ODP_ERR("Maximum number of models has already been created!\n"); + return ODP_ML_MODEL_INVALID; + } + + /* Free model entry was found and is now locked */ + mdl->state = ML_STATE_CREATED; + + status = ort_api->CreateSessionOptions(&session_opts); + if (check_ortstatus(status) || !session_opts) { + _ODP_ERR("Error: CreateSessionOptions failed.\n"); + mdl->state = ML_STATE_FREE; + odp_ticketlock_unlock(&mdl->lock); + return ODP_ML_MODEL_INVALID; + } + + if (set_ort_run_opts(name, session_opts)) { + _odp_ml_glb->ort_api->ReleaseSessionOptions(session_opts); + mdl->state = ML_STATE_FREE; + odp_ticketlock_unlock(&mdl->lock); + return ODP_ML_MODEL_INVALID; + } + + /* Store model info */ + info = &mdl->info; + memset(info, 0, sizeof(odp_ml_model_info_t)); + + if (create_ort_model(param, &session, mdl, session_opts)) { + mdl->state = ML_STATE_FREE; + + /* Initialize info back to 0 when some fields have been filled + * while later failed */ + reset_mdl_info_sizes(mdl); + odp_ticketlock_unlock(&mdl->lock); + + _odp_ml_glb->ort_api->ReleaseSessionOptions(session_opts); + _ODP_ERR("create_ort_model() failed\n"); + return ODP_ML_MODEL_INVALID; + } + + if (check_io_shape(mdl)) { + mdl->state = ML_STATE_FREE; + reset_mdl_info_sizes(mdl); + odp_ticketlock_unlock(&mdl->lock); + + ort_api->ReleaseSession(session); + _odp_ml_glb->ort_api->ReleaseSessionOptions(session_opts); + return ODP_ML_MODEL_INVALID; + } + + mdl->session = session; + mdl->session_opts = session_opts; + info->index = i; + + if (name) { + strncpy(info->name, name, ODP_ML_MODEL_NAME_LEN - 1); + info->name[ODP_ML_MODEL_NAME_LEN - 1] = 0; + } + + mdl->max_compl_id = param->max_compl_id; + for (uint32_t j = 0; j < ML_MAX_COMPL_ID; j++) + odp_atomic_init_u32(&mdl->compl_status[j], 1); + + odp_ticketlock_unlock(&mdl->lock); + return (odp_ml_model_t)mdl; +} + +int odp_ml_model_destroy(odp_ml_model_t model) +{ + ml_model_t *mdl = ml_model_from_handle(model); + + if (model == ODP_ML_MODEL_INVALID) { + _ODP_ERR("Bad ML model handle\n"); + return -1; + } + + odp_ticketlock_lock(&mdl->lock); + + if (mdl->state != ML_STATE_CREATED) { + _ODP_ERR("Model not created\n"); + odp_ticketlock_unlock(&mdl->lock); + return -1; + } + + _odp_ml_glb->ort_api->ReleaseSessionOptions(mdl->session_opts); + _odp_ml_glb->ort_api->ReleaseSession(mdl->session); + mdl->state = ML_STATE_FREE; + mdl->session = NULL; + odp_ticketlock_unlock(&mdl->lock); + + return 0; +} + +int odp_ml_model_info(odp_ml_model_t model, odp_ml_model_info_t *info) +{ + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return -1; + } + + if (odp_unlikely(!info)) { + _ODP_ERR("info must not be NULL\n"); + return -1; + } + + odp_ticketlock_lock(&mdl->lock); + if (odp_unlikely(mdl->state == ML_STATE_FREE)) { + _ODP_ERR("Model not created\n"); + odp_ticketlock_unlock(&mdl->lock); + return -1; + } + + *info = mdl->info; + + odp_ticketlock_unlock(&mdl->lock); + return 0; +} + +uint32_t odp_ml_model_input_info(odp_ml_model_t model, odp_ml_input_info_t info[], uint32_t num) +{ + uint32_t num_model_inputs; + uint32_t num_written; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return 0; + } + + odp_ticketlock_lock(&mdl->lock); + num_model_inputs = mdl->info.num_inputs; + num_written = num_model_inputs >= num ? num : num_model_inputs; + + if (num == 0) { + odp_ticketlock_unlock(&mdl->lock); + return num_model_inputs; + } + + for (uint32_t i = 0; i < num_written; i++) + info[i] = mdl->input_info[i]; + + odp_ticketlock_unlock(&mdl->lock); + return num_model_inputs; +} + +uint32_t odp_ml_model_output_info(odp_ml_model_t model, odp_ml_output_info_t info[], uint32_t num) +{ + uint32_t num_model_outputs; + uint32_t num_written; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return 0; + } + + odp_ticketlock_lock(&mdl->lock); + num_model_outputs = mdl->info.num_outputs; + num_written = num_model_outputs >= num ? num : num_model_outputs; + + if (num == 0) { + odp_ticketlock_unlock(&mdl->lock); + return num_model_outputs; + } + + for (uint32_t i = 0; i < num_written; i++) + info[i] = mdl->output_info[i]; + + odp_ticketlock_unlock(&mdl->lock); + return num_model_outputs; +} + +odp_ml_model_t odp_ml_model_lookup(const char *name) +{ + uint32_t i; + ml_model_t *mdl; + + for (i = 0; i < ML_MAX_MODELS_CREATED; i++) { + mdl = &_odp_ml_glb->models[i]; + + odp_ticketlock_lock(&mdl->lock); + + if (mdl->state == ML_STATE_FREE) { + odp_ticketlock_unlock(&mdl->lock); + continue; + } + + if (!strcmp(mdl->info.name, name)) { + /* found it */ + odp_ticketlock_unlock(&mdl->lock); + return (odp_ml_model_t)mdl; + } + odp_ticketlock_unlock(&mdl->lock); + } + + return ODP_ML_MODEL_INVALID; +} + +uint64_t odp_ml_model_to_u64(odp_ml_model_t model) +{ + return _odp_pri(model); +} + +static const char *data_type_str(odp_ml_data_type_t data_type) +{ + switch (data_type) { + case ODP_ML_DATA_TYPE_INT8: + return "int8"; + case ODP_ML_DATA_TYPE_UINT8: + return "uint8"; + case ODP_ML_DATA_TYPE_UINT16: + return "uint16"; + case ODP_ML_DATA_TYPE_INT16: + return "int16"; + case ODP_ML_DATA_TYPE_INT32: + return "int32"; + case ODP_ML_DATA_TYPE_UINT32: + return "uint32"; + case ODP_ML_DATA_TYPE_INT64: + return "int64"; + case ODP_ML_DATA_TYPE_UINT64: + return "uint64"; + case ODP_ML_DATA_TYPE_FP16: + return "fp16"; + case ODP_ML_DATA_TYPE_FP32: + return "fp32"; + case ODP_ML_DATA_TYPE_BFP16: + return "bfp16"; + default: + return "unknown"; + } +} + +static const char *shape_type_str(odp_ml_shape_type_t shape_type) +{ + switch (shape_type) { + case ODP_ML_SHAPE_NONE: + return "none"; + case ODP_ML_SHAPE_STATIC: + return "static"; + case ODP_ML_SHAPE_BATCH: + return "batch"; + default: + return "Unknown"; + } +} + +static void print_shape(const odp_ml_shape_info_t *shape) +{ + /* Print shape */ + _ODP_PRINT("Shape: %s [", shape_type_str(shape->type)); + + for (uint32_t i = 0; i < shape->num_dim; i++) { + if (shape->dim[i] == ODP_ML_DIM_DYNAMIC) + _ODP_PRINT("Dyn"); + else + _ODP_PRINT("%" PRIu32, shape->dim[i]); + + if (i == (shape->num_dim - 1)) + _ODP_PRINT("]\n"); + else + _ODP_PRINT(", "); + } + + /* The number of dimensions for a scalar input is 0, in which case did not + * go into above for loop */ + if (shape->num_dim == 0) + _ODP_PRINT("]\n"); +} + +void odp_ml_model_print(odp_ml_model_t model) +{ + ml_model_t *mdl = ml_model_from_handle(model); + const odp_ml_model_info_t * const info = &mdl->info; + const odp_ml_input_info_t * const input_info = mdl->input_info; + const odp_ml_output_info_t * const output_info = mdl->output_info; + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return; + } + + odp_ticketlock_lock(&mdl->lock); + if (odp_unlikely(mdl->state == ML_STATE_FREE)) { + odp_ticketlock_unlock(&mdl->lock); + _ODP_ERR("Model not created\n"); + return; + } + + _ODP_PRINT("\nModel info\n"); + _ODP_PRINT("----------\n"); + _ODP_PRINT(" Model handle: 0x%" PRIx64 "\n", odp_ml_model_to_u64(model)); + _ODP_PRINT(" Name: %s\n", info->name); + _ODP_PRINT(" Model version: %" PRIu64 "\n", info->model_version); + _ODP_PRINT(" Model interface version: %" PRIu64 "\n", info->interface_version); + _ODP_PRINT(" Index: %u\n", info->index); + _ODP_PRINT(" Number of inputs: %u\n", info->num_inputs); + + for (uint32_t i = 0; i < info->num_inputs; i++) { + _ODP_PRINT(" Input[%u]: ", i); + _ODP_PRINT("Name: %s, ", input_info[i].name); + _ODP_PRINT("Data_type: %s, ", data_type_str(input_info[i].data_type)); + print_shape(&input_info[i].shape); + } + + _ODP_PRINT(" Number of outputs: %u\n", info->num_outputs); + for (uint32_t i = 0; i < info->num_outputs; i++) { + _ODP_PRINT(" Output[%u]: ", i); + _ODP_PRINT("Name: %s, ", output_info[i].name); + _ODP_PRINT("Data_type: %s, ", data_type_str(output_info[i].data_type)); + print_shape(&output_info[i].shape); + } + + odp_ticketlock_unlock(&mdl->lock); + + _ODP_PRINT("\n"); +} + +static inline void mode_print(odp_ml_compl_mode_t compl_mode_mask) +{ + if (compl_mode_mask & ODP_ML_COMPL_MODE_SYNC) + _ODP_PRINT(" syn"); + + if (compl_mode_mask & ODP_ML_COMPL_MODE_POLL) + _ODP_PRINT(" poll"); + + if (compl_mode_mask & ODP_ML_COMPL_MODE_EVENT) + _ODP_PRINT(" event"); +} + +void odp_ml_print(void) +{ + _ODP_PRINT("\nML info\n"); + _ODP_PRINT("-----------\n"); + _ODP_PRINT(" max_model_size: %u\n", ML_MAX_MODEL_SIZE); + _ODP_PRINT(" max_compl_id: %u\n", ML_MAX_COMPL_ID); + _ODP_PRINT(" max_models_created: %u\n", ML_MAX_MODELS_CREATED); + _ODP_PRINT(" max_models_loaded: %u\n", ML_MAX_MODELS_LOADED); + _ODP_PRINT(" model_max_inputs: %u\n", CONFIG_ML_MAX_INPUTS); + _ODP_PRINT(" model_max_outputs: %u\n", CONFIG_ML_MAX_OUTPUTS); + + _ODP_PRINT(" load:\n"); + _ODP_PRINT(" completion mode: "); + mode_print(_odp_ml_glb->capa.load.compl_mode_mask); + _ODP_PRINT(", plain queue: %c, schedule queue: %c\n", + _odp_ml_glb->capa.load.compl_queue_plain ? 'Y' : 'N', + _odp_ml_glb->capa.load.compl_queue_sched ? 'Y' : 'N'); + + _ODP_PRINT(" run:\n"); + _ODP_PRINT(" completion mode:"); + mode_print(_odp_ml_glb->capa.run.compl_mode_mask); + _ODP_PRINT(", plain queue: %c, schedule queue: %c\n", + _odp_ml_glb->capa.run.compl_queue_plain ? 'Y' : 'N', + _odp_ml_glb->capa.run.compl_queue_sched ? 'Y' : 'N'); + _ODP_PRINT("\n"); +} + +int odp_ml_model_extra_stat_info(odp_ml_model_t model, + odp_ml_extra_stat_info_t info[] ODP_UNUSED, + int num ODP_UNUSED) +{ + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return -1; + } + + return 0; +} + +int odp_ml_model_extra_stats(odp_ml_model_t model, uint64_t stats[] ODP_UNUSED, int num ODP_UNUSED) +{ + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return -1; + } + + return 0; +} + +void odp_ml_compl_pool_param_init(odp_ml_compl_pool_param_t *pool_param) +{ + if (odp_unlikely(!pool_param)) { + _ODP_ERR("Param 'pool_param' must not NULL\n"); + return; + } + + memset(pool_param, 0, sizeof(odp_ml_compl_pool_param_t)); + + pool_param->cache_size = _odp_ml_glb->pool_param.buf.cache_size; +} + +odp_pool_t odp_ml_compl_pool_create(const char *name, const odp_ml_compl_pool_param_t *pool_param) +{ + odp_pool_t pool; + odp_pool_param_t ml_pool_param; + uint32_t num = pool_param->num; + uint32_t uarea_size = pool_param->uarea_size; + uint32_t cache_size = pool_param->cache_size; + uint32_t buf_size = _ODP_MAX(sizeof(odp_ml_run_result_t), + sizeof(odp_ml_load_result_t)); + + if (num > _odp_ml_glb->capa.pool.max_num) { + _ODP_ERR("Too many ML completion events: %u\n", num); + return ODP_POOL_INVALID; + } + + if (uarea_size > _odp_ml_glb->capa.pool.max_uarea_size) { + _ODP_ERR("Bad uarea size: %u\n", uarea_size); + return ODP_POOL_INVALID; + } + + if (cache_size < _odp_ml_glb->capa.pool.min_cache_size || + cache_size > _odp_ml_glb->capa.pool.max_cache_size) { + _ODP_ERR("Bad cache size: %u\n", cache_size); + return ODP_POOL_INVALID; + } + + odp_pool_param_init(&ml_pool_param); + ml_pool_param.type = ODP_POOL_BUFFER; + ml_pool_param.uarea_init.init_fn = pool_param->uarea_init.init_fn; + ml_pool_param.uarea_init.args = pool_param->uarea_init.args; + ml_pool_param.buf.num = num; + ml_pool_param.buf.cache_size = cache_size; + ml_pool_param.buf.size = buf_size; + ml_pool_param.buf.uarea_size = uarea_size; + + pool = _odp_pool_create(name, &ml_pool_param, ODP_POOL_ML_COMPL); + + return pool; +} + +odp_ml_compl_t odp_ml_compl_alloc(odp_pool_t pool) +{ + odp_buffer_t buf; + odp_event_t ev; + odp_ml_run_result_t *result; + uint32_t buf_size = _ODP_MAX(sizeof(odp_ml_run_result_t), + sizeof(odp_ml_load_result_t)); + + buf = odp_buffer_alloc(pool); + + if (odp_unlikely(buf == ODP_BUFFER_INVALID)) + return ODP_ML_COMPL_INVALID; + + result = odp_buffer_addr(buf); + memset(result, 0, buf_size); + + ev = odp_buffer_to_event(buf); + _odp_event_type_set(ev, ODP_EVENT_ML_COMPL); + + return (odp_ml_compl_t)(uintptr_t)buf; +} + +void odp_ml_compl_free(odp_ml_compl_t ml_compl) +{ + odp_event_t ev; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)ml_compl; + + if (odp_unlikely(ml_compl == ODP_ML_COMPL_INVALID)) { + _ODP_ERR("Bad ML job completion handle\n"); + return; + } + + ev = odp_buffer_to_event(buf); + _odp_event_type_set(ev, ODP_EVENT_BUFFER); + + odp_buffer_free(buf); +} + +int odp_ml_compl_run_result(odp_ml_compl_t ml_compl, odp_ml_run_result_t *result) +{ + odp_event_subtype_t subtype; + odp_ml_run_result_t *run_result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)ml_compl; + odp_event_t ev = odp_buffer_to_event(buf); + + if (odp_unlikely(ml_compl == ODP_ML_COMPL_INVALID)) { + _ODP_ERR("Given ML completion event is invalid\n"); + return -2; + } + + if (odp_event_types(ev, &subtype) != ODP_EVENT_ML_COMPL || + subtype != ODP_EVENT_ML_COMPL_RUN) { + _ODP_ERR("Given completion event has wrong event type or subtype\n"); + return -2; + } + + run_result = odp_buffer_addr(buf); + if (result) + *result = *run_result; + + return run_result->error_code ? -1 : 0; +} + +int odp_ml_compl_load_result(odp_ml_compl_t ml_compl, odp_ml_load_result_t *result) +{ + odp_event_subtype_t subtype; + odp_ml_load_result_t *load_result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)ml_compl; + odp_event_t ev = odp_buffer_to_event(buf); + + if (odp_unlikely(ml_compl == ODP_ML_COMPL_INVALID)) { + _ODP_ERR("Given ML completion event is invalid\n"); + return -2; + } + + if (odp_event_types(ev, &subtype) != ODP_EVENT_ML_COMPL || + subtype != ODP_EVENT_ML_COMPL_LOAD) { + _ODP_ERR("Given completion event has wrong event type or subtype\n"); + return -2; + } + + load_result = odp_buffer_addr(buf); + if (result) + *result = *load_result; + + return load_result->error_code ? -1 : 0; +} + +void *odp_ml_compl_user_area(odp_ml_compl_t ml_compl) +{ + return odp_buffer_user_area((odp_buffer_t)(uintptr_t)ml_compl); +} + +odp_ml_compl_t odp_ml_compl_from_event(odp_event_t event) +{ + _ODP_ASSERT(_odp_event_hdr_field(event, int8_t, event_type) == ODP_EVENT_ML_COMPL); + + return (odp_ml_compl_t)(uintptr_t)event; +} + +odp_event_t odp_ml_compl_to_event(odp_ml_compl_t ml_compl) +{ + return (odp_event_t)(uintptr_t)ml_compl; +} + +uint64_t odp_ml_compl_to_u64(odp_ml_compl_t ml_compl) +{ + return (uint64_t)(uintptr_t)ml_compl; +} + +void odp_ml_compl_param_init(odp_ml_compl_param_t *compl_param) +{ + memset(compl_param, 0, sizeof(odp_ml_compl_param_t)); + + compl_param->queue = ODP_QUEUE_INVALID; + compl_param->event = ODP_EVENT_INVALID; +} + +int odp_ml_model_load(odp_ml_model_t model, odp_ml_load_result_t *result) +{ + odp_ml_load_result_t result_local; + int ret = -1; + ml_model_t *mdl = ml_model_from_handle(model); + + memset(&result_local, 0, sizeof(result_local)); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + result_local.error_code = ML_BAD_HDL; + goto load_fail; + } + + odp_ticketlock_lock(&mdl->lock); + if (odp_unlikely(mdl->state != ML_STATE_CREATED)) { + _ODP_ERR("Model has not been created yet or is already loaded\n"); + odp_ticketlock_unlock(&mdl->lock); + result_local.error_code = ML_NOT_CREATED; + goto load_fail; + } + + mdl->state = ML_STATE_LOADED; + odp_ticketlock_unlock(&mdl->lock); + ret = 0; + +load_fail: + if (result) + *result = result_local; + + return ret; +} + +static inline int check_compl_param(const odp_ml_compl_param_t *compl_param, + uint32_t max_compl_id, odp_bool_t is_load) +{ + odp_ml_config_t *config = &_odp_ml_glb->ml_config; + + switch (compl_param->mode) { + case ODP_ML_COMPL_MODE_POLL: + if (is_load && !(config->load_mode_mask & ODP_ML_COMPL_MODE_POLL)) { + _ODP_ERR("Poll mode loading/unloading is not configured\n"); + return -1; + } + + if (!is_load && !(config->run_mode_mask & ODP_ML_COMPL_MODE_POLL)) { + _ODP_ERR("Poll mode run is not configured\n"); + return -1; + } + + if (compl_param->compl_id > max_compl_id) { + _ODP_ERR("Bad compl_id: %u, exceeding model max completion id %u\n", + compl_param->compl_id, max_compl_id); + return -1; + } + break; + case ODP_ML_COMPL_MODE_EVENT: + if (is_load && !(config->load_mode_mask & ODP_ML_COMPL_MODE_EVENT)) { + _ODP_ERR("Event mode loading/unloading is not configured\n"); + return -1; + } + + if (!is_load && !(config->run_mode_mask & ODP_ML_COMPL_MODE_EVENT)) { + _ODP_ERR("Event mode run is not configured\n"); + return -1; + } + + if (compl_param->event == ODP_EVENT_INVALID || + compl_param->queue == ODP_QUEUE_INVALID) { + _ODP_ERR("Bad event or queue\n"); + return -1; + } + + if (odp_event_type(compl_param->event) != ODP_EVENT_ML_COMPL) { + _ODP_ERR("Bad completion event type\n"); + return -1; + } + break; + default: + /* Including ODP_ML_COMPL_MODE_SYNC, which is not supported by + * asynchrous functions (e.g. *_start()) either. + */ + _ODP_ERR("Invalid completion mode %u\n", compl_param->mode); + return -1; + } + + return 0; +} + +int odp_ml_model_load_start(odp_ml_model_t model, const odp_ml_compl_param_t *compl_param) +{ + int ret; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad model handle\n"); + return -1; + } + + if (odp_unlikely(check_compl_param(compl_param, mdl->max_compl_id, true))) + return -1; + + if (compl_param->mode == ODP_ML_COMPL_MODE_POLL) + odp_atomic_store_rel_u32(&mdl->compl_status[compl_param->compl_id], 0); + + ret = odp_ml_model_load(model, NULL); + + if (odp_unlikely(ret)) + return -1; + + /* Send a completion event to the given queue */ + if (compl_param->mode == ODP_ML_COMPL_MODE_EVENT) { + odp_ml_load_result_t *result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)compl_param->event; + + _odp_buffer_subtype_set(buf, ODP_EVENT_ML_COMPL_LOAD); + + result = odp_buffer_addr(buf); + result->error_code = 0; + result->user_ptr = compl_param->user_ptr; + + if (odp_unlikely(odp_queue_enq(compl_param->queue, compl_param->event))) { + _ODP_ERR("Completion event enqueue failed %" PRIu64 "\n", + odp_queue_to_u64(compl_param->queue)); + if (odp_ml_model_unload(model, NULL)) + _ODP_ERR("Failed to unload model\n"); + return -1; + } + + return 0; + } + + mdl->result[compl_param->compl_id].user_ptr = compl_param->user_ptr; + odp_atomic_store_rel_u32(&mdl->compl_status[compl_param->compl_id], 1); + return 0; +} + +int odp_ml_model_load_status(odp_ml_model_t model, uint32_t compl_id, odp_ml_load_result_t *result) +{ + int ret; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID || compl_id > mdl->max_compl_id)) { + _ODP_ERR("Invalid model or compl_id: %u\n", compl_id); + return -2; + } + + ret = odp_atomic_load_acq_u32(&mdl->compl_status[compl_id]); + + if (ret && result) { + result->error_code = 0; + result->user_ptr = mdl->result[compl_id].user_ptr; + } + + return ret; +} + +int odp_ml_model_unload(odp_ml_model_t model, odp_ml_load_result_t *result) +{ + odp_ml_load_result_t result_local; + int ret = -1; + ml_model_t *mdl = ml_model_from_handle(model); + + memset(&result_local, 0, sizeof(result_local)); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + result_local.error_code = ML_BAD_HDL; + _ODP_ERR("Bad ML model handle\n"); + goto unload_fail; + } + + odp_ticketlock_lock(&mdl->lock); + /* mdl->state == ML_STATE_FREE, ML_STATE_CREATED, ML_STATE_INFERENCING */ + if (odp_unlikely(mdl->state != ML_STATE_LOADED)) { + _ODP_ERR("Model has not been created/loaded or inferencing has not finished yet\n"); + odp_ticketlock_unlock(&mdl->lock); + result_local.error_code = ML_NOT_LOADED; + goto unload_fail; + } + + mdl->state = ML_STATE_CREATED; + odp_ticketlock_unlock(&mdl->lock); + + ret = 0; + +unload_fail: + if (result) + *result = result_local; + + return ret; +} + +int odp_ml_model_unload_start(odp_ml_model_t model, const odp_ml_compl_param_t *compl_param) +{ + int ret; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad model handle\n"); + return -1; + } + + if (odp_unlikely(check_compl_param(compl_param, mdl->max_compl_id, true))) + return -1; + + if (compl_param->mode == ODP_ML_COMPL_MODE_POLL) + odp_atomic_store_rel_u32(&mdl->compl_status[compl_param->compl_id], 0); + + ret = odp_ml_model_unload(model, NULL); + + if (odp_unlikely(ret)) + return -1; + + /* Upon successful unloading, send a completion event to the given queue */ + if (compl_param->mode == ODP_ML_COMPL_MODE_EVENT) { + odp_ml_load_result_t *result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)compl_param->event; + + _odp_buffer_subtype_set(buf, ODP_EVENT_ML_COMPL_LOAD); + + result = odp_buffer_addr(buf); + result->error_code = 0; + result->user_ptr = compl_param->user_ptr; + + if (odp_unlikely(odp_queue_enq(compl_param->queue, compl_param->event))) { + _ODP_ERR("Completion event enqueue failed %" PRIu64 "\n", + odp_queue_to_u64(compl_param->queue)); + return -1; + } + + return 0; + } + + mdl->result[compl_param->compl_id].user_ptr = compl_param->user_ptr; + odp_atomic_store_rel_u32(&mdl->compl_status[compl_param->compl_id], 1); + return 0; +} + +int odp_ml_model_unload_status(odp_ml_model_t model, uint32_t compl_id, + odp_ml_load_result_t *result) +{ + return odp_ml_model_load_status(model, compl_id, result); +} + +void odp_ml_run_param_init(odp_ml_run_param_t *param) +{ + memset(param, 0, sizeof(odp_ml_run_param_t)); +} + +static void ml_shape_to_int64(const odp_ml_shape_info_t *shape, uint32_t batch_size, int64_t *array) +{ + for (uint32_t i = 0; i < shape->num_dim; i++) { + /* Replace dynamic dimension size with provided batch_size */ + if (shape->dim[i] == ODP_ML_DIM_DYNAMIC) + array[i] = batch_size; + else + array[i] = shape->dim[i]; + } +} + +/* Get the number of elements in given shape */ +static inline uint64_t get_num_elem(uint32_t batch_size, const odp_ml_shape_info_t *shape) +{ + uint64_t num_elements = 1; + int64_t dim[ODP_ML_MAX_DIMS] = {0}; + + ml_shape_to_int64(shape, batch_size, dim); + + for (uint32_t i = 0; i < shape->num_dim; i++) + num_elements *= (uint64_t)dim[i]; + + return num_elements; +} + +static inline uint32_t dyn_io_size(const odp_ml_shape_info_t *shape, uint32_t data_type_size, + const odp_ml_run_param_t *param) +{ + uint32_t size; + + if (!param || !param->batch_size) { + _ODP_ERR("Parameter 'param' must not be NULL and batch_size must be " + "provided when a input/output has dynamic dimension size\n"); + return 0; + } + + size = get_num_elem(param->batch_size, shape); + size *= data_type_size; + + return size; +} + +static int verify_run_params(odp_ml_model_t model, const odp_ml_data_t *data, + const odp_ml_run_param_t *param) +{ + const ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return -1; + } + + if (odp_unlikely(!data)) { + _ODP_ERR("Parameter 'data' must not be NULL\n"); + return -1; + } + + /* Make sure that the number of input data segments equals or bigger than + * the number of model inputs. */ + if (mdl->info.num_inputs > data->num_input_seg) { + _ODP_ERR("The num of input data segments %u must not less than " + "the number of model inputs %u\n", data->num_input_seg, + mdl->info.num_inputs); + return -1; + } + + if (mdl->info.num_outputs > data->num_output_seg) { + _ODP_ERR("The num of output data segments %u must not less than " + "the number of model outputs %u\n", data->num_output_seg, + mdl->info.num_outputs); + return -1; + } + + if (data->num_input_seg > mdl->info.num_inputs && + (_odp_ml_glb->capa.max_segs_per_input == 1)) { + _ODP_ERR("Segmented input data is not supported\n"); + return -1; + } + + if (data->num_output_seg > mdl->info.num_outputs && + (_odp_ml_glb->capa.max_segs_per_output == 1)) { + _ODP_ERR("Segmented output data is not supported"); + return -1; + } + + uint32_t size = 0; + uint32_t input_index = 0; + uint32_t seg_size_sum = 0; + odp_bool_t index_new = true; + uint32_t segs_per_input = 1; + + for (uint32_t i = 0; i < data->num_input_seg; i++) { + if (data->input_seg[i].addr == NULL) { + _ODP_ERR("data->input_seg[%u].addr must not NULL\n", i); + return -1; + }; + + if (index_new) { + if (input_index > mdl->info.num_inputs - 1) { + _ODP_ERR("Too much number of input segments given\n"); + return -1; + } + + /* Input with dynamic batch size */ + if (mdl->input_info[input_index].shape.type == ODP_ML_SHAPE_BATCH) + size = dyn_io_size(&mdl->input_info[input_index].shape, + mdl->input_info[input_index].data_type_size, + param); + else + size = mdl->input_sizes[input_index]; + + if (!size) { + _ODP_ERR("Size for %uth input is 0\n", input_index); + return -1; + } + } + + seg_size_sum += data->input_seg[i].size; + + if (seg_size_sum > size) { + _ODP_ERR("Sum of segment sizes %u exceeds %uth input data size %u\n", + seg_size_sum, input_index, size); + return -1; + } + + if (seg_size_sum == size) { + if (segs_per_input > _odp_ml_glb->capa.max_segs_per_input) { + _ODP_ERR("Number of segments %u for input[%u] exceeds maximum" + " number of data segments per model input %u\n", + segs_per_input, input_index, + _odp_ml_glb->capa.max_segs_per_input); + return -1; + } + input_index++; + index_new = true; + seg_size_sum = 0; + segs_per_input = 1; + } else { + segs_per_input++; + index_new = false; + } + } + + if (input_index != mdl->info.num_inputs) { + _ODP_ERR("Data is not provided for all model inputs\n"); + return -1; + } + + seg_size_sum = 0; + index_new = true; + uint32_t output_index = 0; + uint32_t segs_per_output = 1; + + for (uint32_t i = 0; i < data->num_output_seg; i++) { + if (data->output_seg[i].addr == NULL) { + _ODP_ERR("data->output_seg[%u].addr must not NULL\n", i); + return -1; + } + + if (index_new) { + if (output_index > mdl->info.num_outputs - 1) { + _ODP_ERR("Too much number of output segments given\n"); + return -1; + } + + /* Output with dynamic batch size */ + if (mdl->output_info[output_index].shape.type == ODP_ML_SHAPE_BATCH) + size = dyn_io_size(&mdl->output_info[output_index].shape, + mdl->output_info[output_index].data_type_size, + param); + else + size = mdl->output_sizes[output_index]; + + if (!size) { + _ODP_ERR("Size for %uth output is 0\n", output_index); + return -1; + } + } + + seg_size_sum += data->output_seg[i].size; + + if (seg_size_sum > size) { + _ODP_ERR("Sum of segment sizes %u exceeds %uth output data size %u\n", + seg_size_sum, output_index, size); + return -1; + } + + if (seg_size_sum >= size) { + if (segs_per_output > _odp_ml_glb->capa.max_segs_per_output) { + _ODP_ERR("Number of segments %u for output[%u] exceeds maximum" + " number of data segments per model output %u\n", + segs_per_output, output_index, + _odp_ml_glb->capa.max_segs_per_output); + return -1; + } + output_index++; + index_new = true; + seg_size_sum = 0; + segs_per_output = 1; + } else { + segs_per_output++; + index_new = false; + } + } + + if (output_index != mdl->info.num_outputs) { + _ODP_ERR("Not enough output_segs to hold all output data\n"); + return -1; + } + + return 0; +} + +static ONNXTensorElementDataType onnx_dtype_from_odp_dtype(odp_ml_data_type_t data_type) +{ + switch (data_type) { + case ODP_ML_DATA_TYPE_NONE: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + case ODP_ML_DATA_TYPE_INT8: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; + case ODP_ML_DATA_TYPE_UINT8: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; + case ODP_ML_DATA_TYPE_INT16: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; + case ODP_ML_DATA_TYPE_UINT16: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; + case ODP_ML_DATA_TYPE_INT24: + /* Fall through*/ + case ODP_ML_DATA_TYPE_UINT24: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + case ODP_ML_DATA_TYPE_FP64: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; + case ODP_ML_DATA_TYPE_INT32: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; + case ODP_ML_DATA_TYPE_UINT32: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; + case ODP_ML_DATA_TYPE_INT64: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; + case ODP_ML_DATA_TYPE_UINT64: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; + case ODP_ML_DATA_TYPE_FP16: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; + case ODP_ML_DATA_TYPE_FP32: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; + case ODP_ML_DATA_TYPE_BFP16: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16; + default: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + } +} + +static int verify_tensor(const OrtValue *tensor, odp_ml_data_type_t expected_type, + const odp_ml_shape_info_t *expected_shape, uint32_t batch_size) +{ + OrtTensorTypeAndShapeInfo *tensor_info; + ONNXTensorElementDataType tensor_type; + size_t dim_count; + OrtStatus *status = NULL; + int64_t dims[ODP_ML_MAX_DIMS] = {0}; + int64_t shape_arr[ODP_ML_MAX_DIMS] = {0}; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + status = ort_api->GetTensorTypeAndShape(tensor, &tensor_info); + if (check_ortstatus(status)) { + _ODP_ERR("GetTensorTypeAndShape() failed\n"); + return -1; + } + + status = ort_api->GetTensorElementType(tensor_info, &tensor_type); + if (check_ortstatus(status)) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("GetTensorElementType() failed\n"); + return -1; + } + + if (onnx_dtype_to_odp_dtype(tensor_type) != expected_type) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("Tensor type does not match model type\n"); + return -1; + } + + status = ort_api->GetDimensionsCount(tensor_info, &dim_count); + if (check_ortstatus(status)) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("GetDimensionsCount() failed\n"); + return -1; + } + + if (dim_count != expected_shape->num_dim) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("Tensor dimension does not match shape_dim\n"); + return -1; + } + + status = ort_api->GetDimensions(tensor_info, dims, dim_count); + if (check_ortstatus(status)) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("GetDimensions() failed\n"); + return -1; + } + + ml_shape_to_int64(expected_shape, batch_size, shape_arr); + + for (uint32_t i = 0; i < dim_count; i++) { + if (dims[i] != shape_arr[i]) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("Shape[%u]: %" PRIu64 " does not match expected: %" PRIu64 "\n", + i, dims[i], shape_arr[i]); + return -1; + } + } + + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + return 0; +} + +static int input_data_to_tensor(const odp_ml_input_info_t *input_info, uint32_t num_seg, + const odp_ml_data_seg_t *input_seg, uint32_t *seg_idx, + uint32_t batch_size, OrtValue **input_tensor) +{ + int is_tensor; + uint64_t input_size; + OrtAllocator *allocator; + void *data = NULL; + OrtStatus *status = NULL; + int64_t shape[ODP_ML_MAX_DIMS] = {0}; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + ONNXTensorElementDataType onnx_dtype = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + + ml_shape_to_int64(&input_info->shape, batch_size, shape); + + onnx_dtype = onnx_dtype_from_odp_dtype(input_info->data_type); + _ODP_ASSERT(onnx_dtype != ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); + + status = ort_api->GetAllocatorWithDefaultOptions(&allocator); + if (check_ortstatus(status)) { + _ODP_ERR("GetAllocatorWithDefaultOptions() failed\n"); + return -1; + } + + status = ort_api->CreateTensorAsOrtValue(allocator, + shape, + input_info->shape.num_dim, + onnx_dtype, + input_tensor); + if (check_ortstatus(status) || !input_tensor[0]) { + _ODP_ERR("CreateTensorWithDataAsOrtValue() failed\n"); + return -1; + } + + input_size = input_info->data_type_size * get_num_elem(batch_size, &input_info->shape); + + status = ort_api->GetTensorMutableData(input_tensor[0], &data); + if (check_ortstatus(status) || !data) { + _ODP_ERR("GetTensorMutableData() failed\n"); + return -1; + } + + for (uint64_t i = 0; i < input_size; ) { + if (*seg_idx >= num_seg) { + _ODP_ERR("Insufficient input data\n"); + return -1; + } + + uint64_t seg_size = input_seg[*seg_idx].size; + + if (i + seg_size > input_size) { + _ODP_ERR("Excess input data in segment %" PRIu32 "\n", *seg_idx); + return -1; + } + + memcpy((uint8_t *)data + i, input_seg[(*seg_idx)++].addr, seg_size); + i += seg_size; + } + + if (!ODP_DEBUG) + return 0; + + status = ort_api->IsTensor(input_tensor[0], &is_tensor); + if (check_ortstatus(status) || !is_tensor) { + _ODP_ERR("input_tensor IsTensor failed\n"); + return -1; + } + + /* Make sure tensor shape matches input_shape */ + if (verify_tensor(input_tensor[0], input_info->data_type, + &input_info->shape, batch_size)) { + _ODP_ERR("Verify input_tensor failed\n"); + return -1; + } + + return 0; +} + +static int verify_output_tensor(OrtValue *output_tensor, odp_ml_data_type_t expected_type, + const odp_ml_shape_info_t *expected_shape, uint32_t batch_size) +{ + int is_tensor = 0; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + OrtStatus *status = ort_api->IsTensor(output_tensor, &is_tensor); + + if (check_ortstatus(status) || !is_tensor) { + _ODP_ERR("output_tensor IsTensor failed\n"); + return -1; + } + + /* Make sure tensor shape matches output_shape */ + if (verify_tensor(output_tensor, expected_type, expected_shape, batch_size)) { + _ODP_ERR("Verify output_tensor failed\n"); + return -1; + } + + return 0; +} + +static int get_tensor_data_size(OrtValue *tensor, uint32_t *size, uint32_t data_type_size) +{ + size_t num_elem; + OrtStatus *status; + OrtTensorTypeAndShapeInfo *tensor_info; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + status = ort_api->GetTensorTypeAndShape(tensor, &tensor_info); + if (check_ortstatus(status)) { + _ODP_ERR("GetTensorTypeAndShape() failed\n"); + return -1; + } + + status = ort_api->GetTensorShapeElementCount(tensor_info, &num_elem); + if (check_ortstatus(status)) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("GetTensorShapeElementCount() failed\n"); + return -1; + } + *size = data_type_size * num_elem; + + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + return 0; +} + +static int check_output_size(odp_bool_t is_segmented, uint32_t output_idx, uint32_t seg_idx, + uint64_t out_tensor_data_size, const odp_ml_data_t data[]) +{ + uint64_t output_size = 0; + + /* Output is not segmented */ + if (!is_segmented) { + /* Make sure tensor data size does not exceed size allocated for + * data->output_seg[seg_idx].addr */ + if (out_tensor_data_size > data->output_seg[seg_idx].size) { + _ODP_ERR("Malloc at least %" PRIu64 " bytes for %dth output tensor\n", + out_tensor_data_size, output_idx); + return -1; + } + + return 0; + } + + /* Output is segmented, first calculate total size for one tensor */ + for (; seg_idx < data->num_output_seg; seg_idx++) { + output_size += data->output_seg[seg_idx].size; + if (output_size >= out_tensor_data_size) + break; + } + + if (0 == output_size) { + _ODP_ERR("No output data segments for %uth output tensor\n", output_idx); + return -1; + } + + if (out_tensor_data_size > output_size) { + _ODP_ERR("Output segments (%" PRIu64 " bytes in total) for %uth output" + " is expected to be at least %" PRIu64 " bytes\n", + output_size, output_idx, out_tensor_data_size); + return -1; + } + + return 0; +} + +static int output_tensors_to_data(OrtValue **output_tensors, + uint32_t model_num_outputs, + const odp_ml_run_param_t *param, + const odp_ml_output_info_t *output_info, + const odp_ml_data_t *data, + odp_ml_run_result_t *result_local) +{ + uint32_t seg_idx; + uint64_t seg_size; + uint64_t cpy_size; + uint64_t left_size; + uint64_t output_val_offset; + uint32_t out_tensor_data_size; + void *output_val = NULL; /* Pointer to store one raw output value */ + OrtStatus *status = NULL; + uint32_t batch_size = (param && param->batch_size) ? param->batch_size : 0; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + odp_bool_t is_segmented = (data->num_output_seg != model_num_outputs); + + seg_idx = 0; + for (uint32_t i = 0; i < model_num_outputs; i++) { + if (ODP_DEBUG && + verify_output_tensor(output_tensors[i], output_info[i].data_type, + &output_info[i].shape, batch_size)){ + result_local->error_code = ML_BAD_OUTPUT; + return -1; + } + + /* Get tensor data size */ + if (get_tensor_data_size(output_tensors[i], &out_tensor_data_size, + output_info[i].data_type_size)) { + result_local->error_code = ML_LIB_FAILED; + return -1; + } + + /* When output_tensor is an empty tensor [], skip getting data */ + if (out_tensor_data_size == 0) + continue; + + if (ODP_DEBUG && check_output_size(is_segmented, i, seg_idx, + out_tensor_data_size, data)) { + result_local->error_code = ML_BAD_OUTPUT; + return -1; + } + + /* Following assumes param and data->output_seg are valid */ + /* Get tensor data */ + output_val = NULL; + status = ort_api->GetTensorMutableData(output_tensors[i], &output_val); + if (check_ortstatus(status) || !output_val) { + result_local->error_code = ML_LIB_FAILED; + return -1; + } + + /* Output is not segmented */ + if (!is_segmented) { + /* Store output data to data->output_seg[i].addr */ + memcpy(data->output_seg[i].addr, output_val, out_tensor_data_size); + seg_idx++; + continue; + } + + /* Output is segmented */ + output_val_offset = 0; + left_size = out_tensor_data_size; + for (; seg_idx < data->num_output_seg; seg_idx++) { + seg_size = data->output_seg[seg_idx].size; + cpy_size = left_size > seg_size ? seg_size : left_size; + memcpy(data->output_seg[seg_idx].addr, + ((char *)output_val) + output_val_offset, cpy_size); + + output_val_offset += cpy_size; + left_size = out_tensor_data_size - output_val_offset; + + if (!left_size) { + seg_idx++; + break; + } + } + } + + return 0; +} + +int odp_ml_run(odp_ml_model_t model, const odp_ml_data_t *data, const odp_ml_run_param_t *param) +{ + odp_ml_run_result_t result_local; + + int retval = -1; /* Return value of this function */ + int ret = 0; + OrtStatus *status = NULL; + uint32_t batch_size = 0; + + OrtValue *input_tensor[CONFIG_ML_MAX_INPUTS] = {0}; + OrtValue *output_tensors[CONFIG_ML_MAX_OUTPUTS] = {0}; + const char *input_names[CONFIG_ML_MAX_INPUTS] = {0}; + const char *output_names[CONFIG_ML_MAX_OUTPUTS] = {0}; + + const OrtApi *ort_api = _odp_ml_glb->ort_api; + ml_model_t *mdl = ml_model_from_handle(model); + const odp_ml_model_info_t *ml_info = &mdl->info; + const odp_ml_input_info_t *input_info = mdl->input_info; + const odp_ml_output_info_t *output_info = mdl->output_info; + OrtSession *session = mdl->session; + + odp_ticketlock_lock(&mdl->lock); + if (odp_unlikely(mdl->state == ML_STATE_INFERENCING)) { + odp_ticketlock_unlock(&mdl->lock); + return 0; + } + if (odp_unlikely(mdl->state != ML_STATE_LOADED)) { + _ODP_ERR("Wrong model state: not created or not loaded\n"); + odp_ticketlock_unlock(&mdl->lock); + return -1; + } + mdl->state = ML_STATE_INFERENCING; + odp_ticketlock_unlock(&mdl->lock); + + memset(&result_local, 0, sizeof(result_local)); + + if (ODP_DEBUG && verify_run_params(model, data, param)) { + result_local.error_code = ML_BAD_INPUT; + goto init_fail; + } + + if (param && param->batch_size) + batch_size = param->batch_size; + + uint32_t seg_idx = 0; + + /* Transfer input data to tensor */ + for (uint32_t i = 0; i < ml_info->num_inputs; i++) { + ret = input_data_to_tensor(&input_info[i], + data->num_input_seg, + data->input_seg, + &seg_idx, + batch_size, + &input_tensor[i]); + if (ret) { + _ODP_ERR("%uth input data to tensor failed\n", i); + result_local.error_code = ML_LIB_FAILED; + goto release_input_tensors; + } + + _ODP_DBG("input_tensor[%u]: %p\n", i, input_tensor[i]); + + /* Model input names */ + input_names[i] = input_info[i].name; + } + + if (seg_idx < data->num_input_seg) { + _ODP_ERR("Excess input segments\n"); + ret = -1; + } + + for (uint32_t i = 0; i < ml_info->num_outputs; i++) + output_names[i] = output_info[i].name; + + /* Run inference */ + status = ort_api->Run(session, + NULL, + (const char * const *)input_names, + (const OrtValue * const*)input_tensor, + ml_info->num_inputs, + (const char * const *)output_names, + ml_info->num_outputs, + output_tensors); + + if (check_ortstatus(status)) { + _ODP_ERR("Run inference failed\n"); + result_local.error_code = ML_LIB_FAILED; + goto release_all_tensors; + } + + /* Verify output tensors and store them to output */ + if (output_tensors_to_data(output_tensors, ml_info->num_outputs, param, + output_info, data, &result_local)) { + _ODP_ERR("Output tensors to data failed\n"); + goto release_all_tensors; + } + + retval = 1; + +release_all_tensors: + for (uint32_t i = 0; i < ml_info->num_outputs; i++) + ort_api->ReleaseValue(output_tensors[i]); + +release_input_tensors: + for (uint32_t i = 0; i < ml_info->num_inputs; i++) + ort_api->ReleaseValue(input_tensor[i]); + +init_fail: + if (param && param->result) + *param->result = result_local; + + odp_ticketlock_lock(&mdl->lock); + mdl->state = ML_STATE_LOADED; + odp_ticketlock_unlock(&mdl->lock); + + return retval; +} + +int odp_ml_run_multi(odp_ml_model_t model, const odp_ml_data_t data[], + const odp_ml_run_param_t param[], int num) +{ + int i; + int ret; + + if (odp_unlikely(num < 1)) { + _ODP_ERR("Bad number of runs\n"); + return -1; + } + + for (i = 0; i < num; i++) { + if (param) + ret = odp_ml_run(model, &data[i], ¶m[i]); + else + ret = odp_ml_run(model, &data[i], NULL); + + if (odp_unlikely(ret != 1)) + break; + } + + if (odp_unlikely(i == 0)) + return ret; + + return i; +} + +int odp_ml_run_start(odp_ml_model_t model, const odp_ml_data_t *data, + const odp_ml_compl_param_t *compl_param, + const odp_ml_run_param_t *run_param) +{ + int ret; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad model handle\n"); + return -1; + } + + if (odp_unlikely(!compl_param)) { + _ODP_ERR("Completion parameter is NULL\n"); + return -1; + } + + /* Check completion mode */ + if (odp_unlikely(check_compl_param(compl_param, mdl->max_compl_id, false))) { + _ODP_ERR("Bad ML job completion parameter\n"); + return -1; + } + + if (compl_param->mode == ODP_ML_COMPL_MODE_POLL) + odp_atomic_store_rel_u32(&mdl->compl_status[compl_param->compl_id], 0); + + ret = odp_ml_run(model, data, run_param); + + if (odp_unlikely(ret < 1)) + return ret; + + /* Send a completion event to the given queue */ + if (compl_param->mode == ODP_ML_COMPL_MODE_EVENT) { + odp_ml_run_result_t *result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)compl_param->event; + + _odp_buffer_subtype_set(buf, ODP_EVENT_ML_COMPL_RUN); + + result = odp_buffer_addr(buf); + result->error_code = 0; + result->user_ptr = compl_param->user_ptr; + + if (odp_unlikely(odp_queue_enq(compl_param->queue, compl_param->event))) { + _ODP_ERR("Completion event enqueue failed %" PRIu64 "\n", + odp_queue_to_u64(compl_param->queue)); + return -1; + } + + return 1; + } + + /* compl_param->mode == ODP_ML_COMPL_MODE_POLL */ + mdl->result[compl_param->compl_id].user_ptr = compl_param->user_ptr; + odp_atomic_store_rel_u32(&mdl->compl_status[compl_param->compl_id], 1); + + return 1; +} + +int odp_ml_run_start_multi(odp_ml_model_t model, const odp_ml_data_t data[], + const odp_ml_compl_param_t compl_param[], + const odp_ml_run_param_t run_param[], int num) +{ + int i; + int ret = 0; + + if (odp_unlikely(num < 1)) { + _ODP_ERR("Bad number of runs\n"); + return -1; + } + + for (i = 0; i < num; i++) { + if (run_param) + ret = odp_ml_run_start(model, &data[i], &compl_param[i], &run_param[i]); + else + ret = odp_ml_run_start(model, &data[i], &compl_param[i], NULL); + + if (odp_unlikely(ret != 1)) + break; + } + + if (odp_unlikely(i == 0)) + return ret; + + return i; +} + +int odp_ml_run_status(odp_ml_model_t model, uint32_t compl_id, odp_ml_run_result_t *result) +{ + int ret; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID || + compl_id > mdl->max_compl_id)) { + _ODP_ERR("Invalid model handle or completion id: %u\n", compl_id); + return -2; + } + + ret = odp_atomic_load_acq_u32(&mdl->compl_status[compl_id]); + + if (result) { + result->error_code = 0; + result->user_ptr = mdl->result[compl_id].user_ptr; + } + + return ret; +} + +static int opt_level_from_str(const char *level_str, GraphOptimizationLevel *level) +{ + if (strcmp(level_str, "DISABLE_ALL") == 0) + *level = ORT_DISABLE_ALL; + else if (strcmp(level_str, "ENABLE_BASIC") == 0) + *level = ORT_ENABLE_BASIC; + else if (strcmp(level_str, "ENABLE_EXTENDED") == 0) + *level = ORT_ENABLE_EXTENDED; + else if (strcmp(level_str, "ENABLE_ALL") == 0) + *level = ORT_ENABLE_ALL; + else + return -1; + + return 0; +} + +static int execution_mode_from_str(const char *mode_str, ExecutionMode *mode) +{ + if (strcmp(mode_str, "SEQUENTIAL") == 0) + *mode = ORT_SEQUENTIAL; + else if (strcmp(mode_str, "PARALLEL") == 0) + *mode = ORT_PARALLEL; + else + return -1; + + return 0; +} + +static int read_config_file(ort_run_opts_t *opts) +{ + const char *conf_str; + char mode_str[ML_MAX_CONFIG_STR_LEN]; + char opt_level_str[ML_MAX_CONFIG_STR_LEN]; + + _ODP_PRINT("ML config:\n"); + + conf_str = "ml.enable_profiling"; + if (!_odp_libconfig_lookup_int(conf_str, &opts->enable_profiling)) { + _ODP_ERR("Config option '%s' not found.\n", conf_str); + return -1; + } + _ODP_PRINT(" %s: %i\n", conf_str, opts->enable_profiling); + + conf_str = "ml.execution_mode"; + if (_odp_libconfig_lookup_str(conf_str, mode_str, ML_MAX_CONFIG_STR_LEN) < 0) { + _ODP_ERR("Config option '%s' not found.\n", conf_str); + return -1; + } + + if (execution_mode_from_str(mode_str, &opts->execution_mode)) { + _ODP_ERR("Unsupported execution mode: %s\n", mode_str); + return -1; + } + _ODP_PRINT(" %s: %s\n", conf_str, mode_str); + + conf_str = "ml.inter_op_num_threads"; + if (!_odp_libconfig_lookup_int(conf_str, &opts->inter_op_num_threads)) { + _ODP_ERR("Config option '%s' not found.\n", conf_str); + return -1; + } + _ODP_PRINT(" %s: %i\n", conf_str, opts->inter_op_num_threads); + + conf_str = "ml.intra_op_num_threads"; + if (!_odp_libconfig_lookup_int(conf_str, &opts->intra_op_num_threads)) { + _ODP_ERR("Config option '%s' not found.\n", conf_str); + return -1; + } + _ODP_PRINT(" %s: %i\n", conf_str, opts->intra_op_num_threads); + + conf_str = "ml.graph_optimization_level"; + if (_odp_libconfig_lookup_str(conf_str, opt_level_str, + ML_MAX_CONFIG_STR_LEN) < 0) { + _ODP_ERR("Config option '%s' not found.\n", conf_str); + return -1; + } + + if (opt_level_from_str(opt_level_str, &opts->graph_opt_level)) { + _ODP_ERR("Graph optimize level %s not supported\n", opt_level_str); + return -1; + } + _ODP_PRINT(" %s: %s\n", conf_str, opt_level_str); + + conf_str = "ml.optimized_model_filepath"; + if (_odp_libconfig_lookup_str(conf_str, opts->opt_model_filepath, + ML_MAX_CONFIG_STR_LEN) < 0) { + _ODP_ERR("Config option '%s' not found.\n", conf_str); + return -1; + } + _ODP_PRINT(" %s: %s\n", conf_str, opts->opt_model_filepath); + + return 0; +} + +int _odp_ml_init_global(void) +{ + int i; + OrtEnv *env; + odp_shm_t shm; + OrtStatus *status; + const OrtApi *ort_api; + + if (odp_global_ro.disable.ml) { + _ODP_ERR("ML is disabled\n"); + return 0; + } + + shm = odp_shm_reserve("_odp_ml_global", sizeof(ml_global_t), ODP_CACHE_LINE_SIZE, 0); + _odp_ml_glb = odp_shm_addr(shm); + + if (_odp_ml_glb == NULL) { + _ODP_ERR("SHM reserve failed for odp_ml\n"); + return -1; + } + + memset(_odp_ml_glb, 0, sizeof(ml_global_t)); + _odp_ml_glb->shm = shm; + + if (odp_ml_capability(&_odp_ml_glb->capa)) { + _ODP_ERR("ML capability failed\n"); + return -1; + } + + odp_pool_param_init(&_odp_ml_glb->pool_param); + + if (read_config_file(&_odp_ml_glb->ort_run_opts)) + return -1; + + ort_api = OrtGetApiBase()->GetApi(ORT_API_VERSION); + if (!ort_api) { + _ODP_ERR("Failed to init ONNX Runtime engine.\n"); + return -1; + } + _odp_ml_glb->ort_api = ort_api; + + status = ort_api->CreateEnv(ORT_LOGGING_LEVEL_WARNING, "Default", &env); + if (check_ortstatus(status) || !env) { + _ODP_ERR("ort_api->CreateEnv() failed.\n"); + return -1; + } + _odp_ml_glb->env = env; + + for (i = 0; i < ML_MAX_MODELS_CREATED; i++) + odp_ticketlock_init(&_odp_ml_glb->models[i].lock); + + return 0; +} + +int _odp_ml_term_global(void) +{ + if (odp_global_ro.disable.ml) + return 0; + + if (_odp_ml_glb == NULL) + return 0; + + if (_odp_ml_glb->env) + _odp_ml_glb->ort_api->ReleaseEnv(_odp_ml_glb->env); + + if (odp_shm_free(_odp_ml_glb->shm)) { + _ODP_ERR("Shm free failed for odp_ml\n"); + return -1; + } + + return 0; +} diff --git a/platform/linux-generic/odp_ml_fp16.c b/platform/linux-generic/odp_ml_fp16.c new file mode 100644 index 000000000..f135f8b5a --- /dev/null +++ b/platform/linux-generic/odp_ml_fp16.c @@ -0,0 +1,425 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2022-2023 Marvell. + * Copyright (c) 2023 Nokia + * + * Based on + * - dpdk/lib/mldev/mldev_utils_scalar.h + * - dpdk/lib/mldev/mldev_utils_scalar.c + * - dpdk/lib/mldev/mldev_utils_scalar_bfloat16.c + */ + +#include <errno.h> +#include <stdint.h> + +#include <odp_ml_fp16.h> + +#ifndef BIT +#define BIT(nr) (1UL << (nr)) +#endif + +#ifndef BITS_PER_LONG +#define BITS_PER_LONG (__SIZEOF_LONG__ * 8) +#endif + +#ifndef GENMASK_U32 +#define GENMASK_U32(h, l) (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h)))) +#endif + +/* float32: bit index of MSB & LSB of sign, exponent and mantissa */ +#define FP32_LSB_M 0 +#define FP32_MSB_M 22 +#define FP32_LSB_E 23 +#define FP32_MSB_E 30 +#define FP32_LSB_S 31 +#define FP32_MSB_S 31 + +/* float32: bitmask for sign, exponent and mantissa */ +#define FP32_MASK_S GENMASK_U32(FP32_MSB_S, FP32_LSB_S) +#define FP32_MASK_E GENMASK_U32(FP32_MSB_E, FP32_LSB_E) +#define FP32_MASK_M GENMASK_U32(FP32_MSB_M, FP32_LSB_M) + +/* float16: bit index of MSB & LSB of sign, exponent and mantissa */ +#define FP16_LSB_M 0 +#define FP16_MSB_M 9 +#define FP16_LSB_E 10 +#define FP16_MSB_E 14 +#define FP16_LSB_S 15 +#define FP16_MSB_S 15 + +/* float16: bitmask for sign, exponent and mantissa */ +#define FP16_MASK_S GENMASK_U32(FP16_MSB_S, FP16_LSB_S) +#define FP16_MASK_E GENMASK_U32(FP16_MSB_E, FP16_LSB_E) +#define FP16_MASK_M GENMASK_U32(FP16_MSB_M, FP16_LSB_M) + +/* bfloat16: bit index of MSB & LSB of sign, exponent and mantissa */ +#define BF16_LSB_M 0 +#define BF16_MSB_M 6 +#define BF16_LSB_E 7 +#define BF16_MSB_E 14 +#define BF16_LSB_S 15 +#define BF16_MSB_S 15 + +/* bfloat16: bitmask for sign, exponent and mantissa */ +#define BF16_MASK_S GENMASK_U32(BF16_MSB_S, BF16_LSB_S) +#define BF16_MASK_E GENMASK_U32(BF16_MSB_E, BF16_LSB_E) +#define BF16_MASK_M GENMASK_U32(BF16_MSB_M, BF16_LSB_M) + +/* Exponent bias */ +#define FP32_BIAS_E 127 +#define FP16_BIAS_E 15 +#define BF16_BIAS_E 127 + +#define FP32_PACK(sign, exponent, mantissa) \ + (((sign) << FP32_LSB_S) | ((exponent) << FP32_LSB_E) | (mantissa)) + +#define FP16_PACK(sign, exponent, mantissa) \ + (((sign) << FP16_LSB_S) | ((exponent) << FP16_LSB_E) | (mantissa)) + +#define BF16_PACK(sign, exponent, mantissa) \ + (((sign) << BF16_LSB_S) | ((exponent) << BF16_LSB_E) | (mantissa)) + +/* Represent float32 as float and uint32_t */ +union float32 { + float f; + uint32_t u; +}; + +/* Convert a single precision floating point number (float32) into a half precision + * floating point number (float16) using round to nearest rounding mode. + */ +static uint16_t +__float32_to_float16_scalar_rtn(float x) +{ + union float32 f32; /* float32 input */ + uint32_t f32_s; /* float32 sign */ + uint32_t f32_e; /* float32 exponent */ + uint32_t f32_m; /* float32 mantissa */ + uint16_t f16_s; /* float16 sign */ + uint16_t f16_e; /* float16 exponent */ + uint16_t f16_m; /* float16 mantissa */ + uint32_t tbits; /* number of truncated bits */ + uint32_t tmsb; /* MSB position of truncated bits */ + uint32_t m_32; /* temporary float32 mantissa */ + uint16_t m_16; /* temporary float16 mantissa */ + uint16_t u16; /* float16 output */ + int be_16; /* float16 biased exponent, signed */ + + f32.f = x; + f32_s = (f32.u & FP32_MASK_S) >> FP32_LSB_S; + f32_e = (f32.u & FP32_MASK_E) >> FP32_LSB_E; + f32_m = (f32.u & FP32_MASK_M) >> FP32_LSB_M; + + f16_s = f32_s; + f16_e = 0; + f16_m = 0; + + switch (f32_e) { + case (0): /* float32: zero or subnormal number */ + f16_e = 0; + f16_m = 0; /* convert to zero */ + break; + case (FP32_MASK_E >> FP32_LSB_E): /* float32: infinity or nan */ + f16_e = FP16_MASK_E >> FP16_LSB_E; + if (f32_m == 0) { /* infinity */ + f16_m = 0; + } else { /* nan, propagate mantissa and set MSB of mantissa to 1 */ + f16_m = f32_m >> (FP32_MSB_M - FP16_MSB_M); + f16_m |= BIT(FP16_MSB_M); + } + break; + default: /* float32: normal number */ + /* compute biased exponent for float16 */ + be_16 = (int)f32_e - FP32_BIAS_E + FP16_BIAS_E; + + /* overflow, be_16 = [31-INF], set to infinity */ + if (be_16 >= (int)(FP16_MASK_E >> FP16_LSB_E)) { + f16_e = FP16_MASK_E >> FP16_LSB_E; + f16_m = 0; + } else if ((be_16 >= 1) && (be_16 < (int)(FP16_MASK_E >> FP16_LSB_E))) { + /* normal float16, be_16 = [1:30]*/ + f16_e = be_16; + m_16 = f32_m >> (FP32_LSB_E - FP16_LSB_E); + tmsb = FP32_MSB_M - FP16_MSB_M - 1; + if ((f32_m & GENMASK_U32(tmsb, 0)) > BIT(tmsb)) { + /* round: non-zero truncated bits except MSB */ + m_16++; + + /* overflow into exponent */ + if (((m_16 & FP16_MASK_E) >> FP16_LSB_E) == 0x1) + f16_e++; + } else if ((f32_m & GENMASK_U32(tmsb, 0)) == BIT(tmsb)) { + /* round: MSB of truncated bits and LSB of m_16 is set */ + if ((m_16 & 0x1) == 0x1) { + m_16++; + + /* overflow into exponent */ + if (((m_16 & FP16_MASK_E) >> FP16_LSB_E) == 0x1) + f16_e++; + } + } + f16_m = m_16 & FP16_MASK_M; + } else if ((be_16 >= -(int)(FP16_MSB_M)) && (be_16 < 1)) { + /* underflow: zero / subnormal, be_16 = [-9:0] */ + f16_e = 0; + + /* add implicit leading zero */ + m_32 = f32_m | BIT(FP32_LSB_E); + tbits = FP32_LSB_E - FP16_LSB_E - be_16 + 1; + m_16 = m_32 >> tbits; + + /* if non-leading truncated bits are set */ + if ((f32_m & GENMASK_U32(tbits - 1, 0)) > BIT(tbits - 1)) { + m_16++; + + /* overflow into exponent */ + if (((m_16 & FP16_MASK_E) >> FP16_LSB_E) == 0x1) + f16_e++; + } else if ((f32_m & GENMASK_U32(tbits - 1, 0)) == BIT(tbits - 1)) { + /* if leading truncated bit is set */ + if ((m_16 & 0x1) == 0x1) { + m_16++; + + /* overflow into exponent */ + if (((m_16 & FP16_MASK_E) >> FP16_LSB_E) == 0x1) + f16_e++; + } + } + f16_m = m_16 & FP16_MASK_M; + } else if (be_16 == -(int)(FP16_MSB_M + 1)) { + /* underflow: zero, be_16 = [-10] */ + f16_e = 0; + if (f32_m != 0) + f16_m = 1; + else + f16_m = 0; + } else { + /* underflow: zero, be_16 = [-INF:-11] */ + f16_e = 0; + f16_m = 0; + } + + break; + } + + u16 = FP16_PACK(f16_s, f16_e, f16_m); + + return u16; +} + +/* Convert a half precision floating point number (float16) into a single precision + * floating point number (float32). + */ +static float +__float16_to_float32_scalar_rtx(uint16_t f16) +{ + union float32 f32; /* float32 output */ + uint16_t f16_s; /* float16 sign */ + uint16_t f16_e; /* float16 exponent */ + uint16_t f16_m; /* float16 mantissa */ + uint32_t f32_s; /* float32 sign */ + uint32_t f32_e; /* float32 exponent */ + uint32_t f32_m; /* float32 mantissa*/ + uint8_t shift; /* number of bits to be shifted */ + uint32_t clz; /* count of leading zeroes */ + int e_16; /* float16 exponent unbiased */ + + f16_s = (f16 & FP16_MASK_S) >> FP16_LSB_S; + f16_e = (f16 & FP16_MASK_E) >> FP16_LSB_E; + f16_m = (f16 & FP16_MASK_M) >> FP16_LSB_M; + + f32_s = f16_s; + switch (f16_e) { + case (FP16_MASK_E >> FP16_LSB_E): /* float16: infinity or nan */ + f32_e = FP32_MASK_E >> FP32_LSB_E; + if (f16_m == 0x0) { /* infinity */ + f32_m = f16_m; + } else { /* nan, propagate mantissa, set MSB of mantissa to 1 */ + f32_m = f16_m; + shift = FP32_MSB_M - FP16_MSB_M; + f32_m = (f32_m << shift) & FP32_MASK_M; + f32_m |= BIT(FP32_MSB_M); + } + break; + case 0: /* float16: zero or sub-normal */ + f32_m = f16_m; + if (f16_m == 0) { /* zero signed */ + f32_e = 0; + } else { /* subnormal numbers */ + clz = __builtin_clz((uint32_t)f16_m) - sizeof(uint32_t) * 8 + FP16_LSB_E; + e_16 = (int)f16_e - clz; + f32_e = FP32_BIAS_E + e_16 - FP16_BIAS_E; + + shift = clz + (FP32_MSB_M - FP16_MSB_M) + 1; + f32_m = (f32_m << shift) & FP32_MASK_M; + } + break; + default: /* normal numbers */ + f32_m = f16_m; + e_16 = (int)f16_e; + f32_e = FP32_BIAS_E + e_16 - FP16_BIAS_E; + + shift = (FP32_MSB_M - FP16_MSB_M); + f32_m = (f32_m << shift) & FP32_MASK_M; + } + + f32.u = FP32_PACK(f32_s, f32_e, f32_m); + + return f32.f; +} + +/* Convert a single precision floating point number (float32) into a + * brain float number (bfloat16) using round to nearest rounding mode. + */ +static uint16_t +__float32_to_bfloat16_scalar_rtn(float x) +{ + union float32 f32; /* float32 input */ + uint32_t f32_s; /* float32 sign */ + uint32_t f32_e; /* float32 exponent */ + uint32_t f32_m; /* float32 mantissa */ + uint16_t b16_s; /* float16 sign */ + uint16_t b16_e; /* float16 exponent */ + uint16_t b16_m; /* float16 mantissa */ + uint32_t tbits; /* number of truncated bits */ + uint16_t u16; /* float16 output */ + + f32.f = x; + f32_s = (f32.u & FP32_MASK_S) >> FP32_LSB_S; + f32_e = (f32.u & FP32_MASK_E) >> FP32_LSB_E; + f32_m = (f32.u & FP32_MASK_M) >> FP32_LSB_M; + + b16_s = f32_s; + b16_e = 0; + b16_m = 0; + + switch (f32_e) { + case (0): /* float32: zero or subnormal number */ + b16_e = 0; + if (f32_m == 0) /* zero */ + b16_m = 0; + else /* subnormal float32 number, normal bfloat16 */ + goto bf16_normal; + break; + case (FP32_MASK_E >> FP32_LSB_E): /* float32: infinity or nan */ + b16_e = BF16_MASK_E >> BF16_LSB_E; + if (f32_m == 0) { /* infinity */ + b16_m = 0; + } else { /* nan, propagate mantissa and set MSB of mantissa to 1 */ + b16_m = f32_m >> (FP32_MSB_M - BF16_MSB_M); + b16_m |= BIT(BF16_MSB_M); + } + break; + default: /* float32: normal number, normal bfloat16 */ + goto bf16_normal; + } + + goto bf16_pack; + +bf16_normal: + b16_e = f32_e; + tbits = FP32_MSB_M - BF16_MSB_M; + b16_m = f32_m >> tbits; + + /* if non-leading truncated bits are set */ + if ((f32_m & GENMASK_U32(tbits - 1, 0)) > BIT(tbits - 1)) { + b16_m++; + + /* if overflow into exponent */ + if (((b16_m & BF16_MASK_E) >> BF16_LSB_E) == 0x1) + b16_e++; + } else if ((f32_m & GENMASK_U32(tbits - 1, 0)) == BIT(tbits - 1)) { + /* if only leading truncated bit is set */ + if ((b16_m & 0x1) == 0x1) { + b16_m++; + + /* if overflow into exponent */ + if (((b16_m & BF16_MASK_E) >> BF16_LSB_E) == 0x1) + b16_e++; + } + } + b16_m = b16_m & BF16_MASK_M; + +bf16_pack: + u16 = BF16_PACK(b16_s, b16_e, b16_m); + + return u16; +} + +/* Convert a brain float number (bfloat16) into a + * single precision floating point number (float32). + */ +static float +__bfloat16_to_float32_scalar_rtx(uint16_t f16) +{ + union float32 f32; /* float32 output */ + uint16_t b16_s; /* float16 sign */ + uint16_t b16_e; /* float16 exponent */ + uint16_t b16_m; /* float16 mantissa */ + uint32_t f32_s; /* float32 sign */ + uint32_t f32_e; /* float32 exponent */ + uint32_t f32_m; /* float32 mantissa*/ + uint8_t shift; /* number of bits to be shifted */ + + b16_s = (f16 & BF16_MASK_S) >> BF16_LSB_S; + b16_e = (f16 & BF16_MASK_E) >> BF16_LSB_E; + b16_m = (f16 & BF16_MASK_M) >> BF16_LSB_M; + + f32_s = b16_s; + switch (b16_e) { + case (BF16_MASK_E >> BF16_LSB_E): /* bfloat16: infinity or nan */ + f32_e = FP32_MASK_E >> FP32_LSB_E; + if (b16_m == 0x0) { /* infinity */ + f32_m = 0; + } else { /* nan, propagate mantissa, set MSB of mantissa to 1 */ + f32_m = b16_m; + shift = FP32_MSB_M - BF16_MSB_M; + f32_m = (f32_m << shift) & FP32_MASK_M; + f32_m |= BIT(FP32_MSB_M); + } + break; + case 0: /* bfloat16: zero or subnormal */ + f32_m = b16_m; + if (b16_m == 0) { /* zero signed */ + f32_e = 0; + } else { /* subnormal numbers */ + goto fp32_normal; + } + break; + default: /* bfloat16: normal number */ + goto fp32_normal; + } + + goto fp32_pack; + +fp32_normal: + f32_m = b16_m; + f32_e = FP32_BIAS_E + b16_e - BF16_BIAS_E; + + shift = (FP32_MSB_M - BF16_MSB_M); + f32_m = (f32_m << shift) & FP32_MASK_M; + +fp32_pack: + f32.u = FP32_PACK(f32_s, f32_e, f32_m); + + return f32.f; +} + +uint16_t _odp_float32_to_float16(float x) +{ + return __float32_to_float16_scalar_rtn(x); +} + +float _odp_float16_to_float32(uint16_t f16) +{ + return __float16_to_float32_scalar_rtx(f16); +} + +uint16_t _odp_float32_to_bfloat16(float x) +{ + return __float32_to_bfloat16_scalar_rtn(x); +} + +float _odp_bfloat16_to_float32(uint16_t f16) +{ + return __bfloat16_to_float32_scalar_rtx(f16); +} diff --git a/platform/linux-generic/odp_ml_null.c b/platform/linux-generic/odp_ml_null.c new file mode 100644 index 000000000..faf431997 --- /dev/null +++ b/platform/linux-generic/odp_ml_null.c @@ -0,0 +1,229 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp/api/ml.h> +#include <odp_init_internal.h> + +#include <string.h> + +/* Dummy ML API implementation, no capability and just return error for + * other functions. + */ +int _odp_ml_init_global(void) +{ + return 0; +} + +int _odp_ml_term_global(void) +{ + return 0; +} + +int odp_ml_capability(odp_ml_capability_t *capa) +{ + memset(capa, 0, sizeof(odp_ml_capability_t)); + return 0; +} + +void odp_ml_config_init(odp_ml_config_t *config ODP_UNUSED) +{ +} + +int odp_ml_config(const odp_ml_config_t *config ODP_UNUSED) +{ + return -1; +} + +void odp_ml_model_param_init(odp_ml_model_param_t *param ODP_UNUSED) +{ +} + +odp_ml_model_t odp_ml_model_create(const char *name ODP_UNUSED, + const odp_ml_model_param_t *param ODP_UNUSED) +{ + return ODP_ML_MODEL_INVALID; +} + +int odp_ml_model_destroy(odp_ml_model_t model ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_info(odp_ml_model_t model ODP_UNUSED, odp_ml_model_info_t *info ODP_UNUSED) +{ + return -1; +} + +uint32_t odp_ml_model_input_info(odp_ml_model_t model ODP_UNUSED, + odp_ml_input_info_t info[] ODP_UNUSED, + uint32_t num ODP_UNUSED) +{ + return 0; +} + +uint32_t odp_ml_model_output_info(odp_ml_model_t model ODP_UNUSED, + odp_ml_output_info_t info[] ODP_UNUSED, + uint32_t num ODP_UNUSED) +{ + return 0; +} + +odp_ml_model_t odp_ml_model_lookup(const char *name ODP_UNUSED) +{ + return ODP_ML_MODEL_INVALID; +} + +uint64_t odp_ml_model_to_u64(odp_ml_model_t model ODP_UNUSED) +{ + return 0; +} + +void odp_ml_model_print(odp_ml_model_t model ODP_UNUSED) +{ +} + +void odp_ml_print(void) +{ +} + +void odp_ml_compl_pool_param_init(odp_ml_compl_pool_param_t *pool_param) +{ + memset(pool_param, 0, sizeof(odp_ml_compl_pool_param_t)); +} + +odp_pool_t odp_ml_compl_pool_create(const char *name ODP_UNUSED, + const odp_ml_compl_pool_param_t *pool_param ODP_UNUSED) +{ + return ODP_POOL_INVALID; +} + +odp_ml_compl_t odp_ml_compl_alloc(odp_pool_t pool ODP_UNUSED) +{ + return ODP_ML_COMPL_INVALID; +} + +void odp_ml_compl_free(odp_ml_compl_t ml_compl ODP_UNUSED) +{ +} + +int odp_ml_compl_run_result(odp_ml_compl_t ml_compl ODP_UNUSED, + odp_ml_run_result_t *result ODP_UNUSED) +{ + return -1; +} + +int odp_ml_compl_load_result(odp_ml_compl_t ml_compl ODP_UNUSED, + odp_ml_load_result_t *result ODP_UNUSED) +{ + return -1; +} + +void *odp_ml_compl_user_area(odp_ml_compl_t ml_compl ODP_UNUSED) +{ + return NULL; +} + +odp_ml_compl_t odp_ml_compl_from_event(odp_event_t event ODP_UNUSED) +{ + return ODP_ML_COMPL_INVALID; +} + +odp_event_t odp_ml_compl_to_event(odp_ml_compl_t ml_compl ODP_UNUSED) +{ + return ODP_EVENT_INVALID; +} + +uint64_t odp_ml_compl_to_u64(odp_ml_compl_t ml_compl ODP_UNUSED) +{ + return 0; +} + +void odp_ml_compl_param_init(odp_ml_compl_param_t *compl_param ODP_UNUSED) +{ +} + +int odp_ml_model_load(odp_ml_model_t model ODP_UNUSED, odp_ml_load_result_t *result ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_load_start(odp_ml_model_t model ODP_UNUSED, + const odp_ml_compl_param_t *compl_param ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_load_status(odp_ml_model_t model ODP_UNUSED, uint32_t compl_id ODP_UNUSED, + odp_ml_load_result_t *result ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_unload(odp_ml_model_t model ODP_UNUSED, odp_ml_load_result_t *result ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_unload_start(odp_ml_model_t model ODP_UNUSED, + const odp_ml_compl_param_t *compl_param ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_unload_status(odp_ml_model_t model ODP_UNUSED, uint32_t compl_id ODP_UNUSED, + odp_ml_load_result_t *result ODP_UNUSED) +{ + return -1; +} + +void odp_ml_run_param_init(odp_ml_run_param_t *param ODP_UNUSED) +{ +} + +int odp_ml_run(odp_ml_model_t model ODP_UNUSED, const odp_ml_data_t *data ODP_UNUSED, + const odp_ml_run_param_t *param ODP_UNUSED) +{ + return -1; +} + +int odp_ml_run_multi(odp_ml_model_t model ODP_UNUSED, const odp_ml_data_t data[] ODP_UNUSED, + const odp_ml_run_param_t param[] ODP_UNUSED, int num ODP_UNUSED) +{ + return -1; +} + +int odp_ml_run_start(odp_ml_model_t model ODP_UNUSED, const odp_ml_data_t *data ODP_UNUSED, + const odp_ml_compl_param_t *compl_param ODP_UNUSED, + const odp_ml_run_param_t *run_param ODP_UNUSED) +{ + return -1; +} + +int odp_ml_run_start_multi(odp_ml_model_t model ODP_UNUSED, + const odp_ml_data_t data[] ODP_UNUSED, + const odp_ml_compl_param_t compl_param[] ODP_UNUSED, + const odp_ml_run_param_t run_param[] ODP_UNUSED, + int num ODP_UNUSED) +{ + return -1; +} + +int odp_ml_run_status(odp_ml_model_t model ODP_UNUSED, uint32_t compl_id ODP_UNUSED, + odp_ml_run_result_t *result ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_extra_stat_info(odp_ml_model_t model ODP_UNUSED, + odp_ml_extra_stat_info_t info[] ODP_UNUSED, + int num ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_extra_stats(odp_ml_model_t model ODP_UNUSED, + uint64_t stats[] ODP_UNUSED, int num ODP_UNUSED) +{ + return -1; +} diff --git a/platform/linux-generic/odp_ml_quantize.c b/platform/linux-generic/odp_ml_quantize.c new file mode 100644 index 000000000..0678f15ef --- /dev/null +++ b/platform/linux-generic/odp_ml_quantize.c @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <stdint.h> +#include <math.h> + +#include <odp/api/ml_quantize.h> + +#include <odp_macros_internal.h> +#include <odp_debug_internal.h> +#include <odp_ml_fp16.h> + +void odp_ml_fp32_to_uint8(uint8_t *u8, const float *fp32, uint32_t num, float scale, + uint8_t zerop) +{ + float fval; + + _ODP_ASSERT(scale != 0); + + for (uint32_t i = 0; i < num; i++) { + /* Range mapping: map real values to signed integer */ + fval = nearbyintf(fp32[i] / scale) + (float)zerop; + + /* clip */ + fval = _ODP_MAX(fval, 0.f); + fval = _ODP_MIN(fval, 255.f); + u8[i] = (uint8_t)(int32_t)fval; + } +} + +void odp_ml_fp32_from_uint8(float *fp32, const uint8_t *u8, uint32_t num, float scale, + uint8_t zerop) +{ + for (uint32_t i = 0; i < num; i++) + fp32[i] = (float)(u8[i] - zerop) * scale; +} + +void odp_ml_fp32_to_int8(int8_t *i8, const float *fp32, uint32_t num, float scale, int8_t zerop) +{ + float fval; + + _ODP_ASSERT(scale != 0); + + for (uint32_t i = 0; i < num; i++) { + /* Range mapping: map real values to signed integer */ + fval = nearbyintf(fp32[i] / scale) + (float)zerop; + + /* NOTE: Clamps signed quantization values to [-127,127] instead of [-128,127]. + * This is to ensure that symmetric quantization results in a zero + * point of exactly 0 for signed 8 bit ints. + */ + fval = _ODP_MAX(fval, -127.f); + fval = _ODP_MIN(fval, 127.f); + i8[i] = (int8_t)(int32_t)fval; + } +} + +void odp_ml_fp32_from_int8(float *fp32, const int8_t *i8, uint32_t num, float scale, int8_t zerop) +{ + for (uint32_t i = 0; i < num; i++) + fp32[i] = (float)(i8[i] - zerop) * scale; +} + +void odp_ml_fp32_to_fp16(uint16_t *fp16, const float *fp32, uint32_t num) +{ + uint32_t i; + + for (i = 0; i < num; i++) + fp16[i] = _odp_float32_to_float16(fp32[i]); +} + +void odp_ml_fp32_from_fp16(float *fp32, const uint16_t *fp16, uint32_t num) +{ + uint32_t i; + + for (i = 0; i < num; i++) + fp32[i] = _odp_float16_to_float32(fp16[i]); +} diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 1e9767821..d3fde70f6 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -1257,6 +1257,10 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info) info->dma_pool_param.uarea_size = pool->params.buf.uarea_size; info->dma_pool_param.cache_size = pool->params.buf.cache_size; + } else if (pool->type_2 == ODP_POOL_ML_COMPL) { + info->ml_pool_param.num = pool->params.buf.num; + info->ml_pool_param.uarea_size = pool->params.buf.uarea_size; + info->ml_pool_param.cache_size = pool->params.buf.cache_size; } else { info->params = pool->params; } @@ -1559,6 +1563,8 @@ static const char *get_long_type_str(odp_pool_type_t type) return "vector"; case ODP_POOL_DMA_COMPL: return "dma completion"; + case ODP_POOL_ML_COMPL: + return "ml completion"; default: return "unknown"; } @@ -1577,6 +1583,8 @@ static const char *get_short_type_str(odp_pool_type_t type) return "V"; case ODP_POOL_DMA_COMPL: return "D"; + case ODP_POOL_ML_COMPL: + return "M"; default: return "-"; } @@ -1875,6 +1883,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, odp_pool_ext_capability_t *cap case ODP_POOL_TIMEOUT: case ODP_POOL_VECTOR: case ODP_POOL_DMA_COMPL: + case ODP_POOL_ML_COMPL: memset(capa, 0, sizeof(odp_pool_ext_capability_t)); return 0; default: diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index bea77fb23..a2593b531 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -627,5 +627,8 @@ void odp_sys_config_print(void) _ODP_PRINT("CONFIG_IPSEC_MAX_NUM_SA: %i\n", CONFIG_IPSEC_MAX_NUM_SA); _ODP_PRINT("CONFIG_TIMER_128BIT_ATOMICS: %i\n", CONFIG_TIMER_128BIT_ATOMICS); _ODP_PRINT("CONFIG_TIMER_PROFILE_INLINE: %i\n", CONFIG_TIMER_PROFILE_INLINE); + _ODP_PRINT("CONFIG_ML_MAX_MODELS: %i\n", CONFIG_ML_MAX_MODELS); + _ODP_PRINT("CONFIG_ML_MAX_INPUTS: %i\n", CONFIG_ML_MAX_INPUTS); + _ODP_PRINT("CONFIG_ML_MAX_OUTPUTS: %i\n", CONFIG_ML_MAX_OUTPUTS); _ODP_PRINT("\n"); } diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index d645bef3c..fa3b6982f 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.27" +config_file_version = "0.1.28" timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf index 427674bb2..fb1418348 100644 --- a/platform/linux-generic/test/packet_align.conf +++ b/platform/linux-generic/test/packet_align.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.27" +config_file_version = "0.1.28" pool: { pkt: { diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index 5bfcb9f2f..f4c6f7952 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.27" +config_file_version = "0.1.28" # Shared memory options shm: { diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf index 1a401298e..8a6d0ac98 100644 --- a/platform/linux-generic/test/sched-basic.conf +++ b/platform/linux-generic/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.27" +config_file_version = "0.1.28" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { diff --git a/platform/linux-generic/test/stash-custom.conf b/platform/linux-generic/test/stash-custom.conf index b96c1cf45..6a2496303 100644 --- a/platform/linux-generic/test/stash-custom.conf +++ b/platform/linux-generic/test/stash-custom.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.27" +config_file_version = "0.1.28" # Test overflow safe stash variant stash: { From 58b89a4f01e25ed210ebee38ef3c8521fc68df92 Mon Sep 17 00:00:00 2001 From: Lifang Zhang <lifang.zhang@nokia.com> Date: Sat, 2 Dec 2023 07:05:58 +0200 Subject: [PATCH 118/147] linux-gen: example: ml: add ML examples based on onnx models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added examples for the new ML API. The examples are based on onnx models. Signed-off-by: Lifang Zhang <lifang.zhang@nokia.com> Signed-off-by: Jere Leppänen <jere.leppanen@nokia.com> Reviewed-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- Makefile.am | 2 + platform/linux-generic/example/Makefile.am | 5 + platform/linux-generic/example/ml/.gitignore | 5 + platform/linux-generic/example/ml/Makefile.am | 46 +++ platform/linux-generic/example/ml/README.md | 94 ++++++ .../example/ml/example_digit.csv | 1 + .../linux-generic/example/ml/mnist-12.onnx | Bin 0 -> 26143 bytes platform/linux-generic/example/ml/mnist.c | 300 ++++++++++++++++++ .../linux-generic/example/ml/model_explorer.c | 88 +++++ .../linux-generic/example/ml/model_read.c | 47 +++ .../linux-generic/example/ml/model_read.h | 29 ++ .../example/ml/odp_ml_run_mnist.sh | 9 + .../example/ml/odp_ml_run_model_explorer.sh | 8 + .../example/ml/odp_ml_run_simple_linear.sh | 8 + .../linux-generic/example/ml/simple_linear.c | 274 ++++++++++++++++ .../example/ml/simple_linear.onnx | Bin 0 -> 214 bytes platform/linux-generic/m4/configure.m4 | 2 + 17 files changed, 918 insertions(+) create mode 100644 platform/linux-generic/example/Makefile.am create mode 100644 platform/linux-generic/example/ml/.gitignore create mode 100644 platform/linux-generic/example/ml/Makefile.am create mode 100644 platform/linux-generic/example/ml/README.md create mode 100644 platform/linux-generic/example/ml/example_digit.csv create mode 100644 platform/linux-generic/example/ml/mnist-12.onnx create mode 100644 platform/linux-generic/example/ml/mnist.c create mode 100644 platform/linux-generic/example/ml/model_explorer.c create mode 100644 platform/linux-generic/example/ml/model_read.c create mode 100644 platform/linux-generic/example/ml/model_read.h create mode 100755 platform/linux-generic/example/ml/odp_ml_run_mnist.sh create mode 100755 platform/linux-generic/example/ml/odp_ml_run_model_explorer.sh create mode 100755 platform/linux-generic/example/ml/odp_ml_run_simple_linear.sh create mode 100644 platform/linux-generic/example/ml/simple_linear.c create mode 100644 platform/linux-generic/example/ml/simple_linear.onnx diff --git a/Makefile.am b/Makefile.am index 477d41740..8b65244c1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -6,6 +6,7 @@ if PLATFORM_IS_LINUX_GENERIC PLATFORM_DIR = platform/linux-generic PLATFORM_DUMPCONF_DIR = platform/linux-generic/dumpconfig PLATFORM_TEST_DIR = platform/linux-generic/test +PLATFORM_EXAMPLE_DIR = platform/linux-generic/example endif SUBDIRS = \ @@ -25,6 +26,7 @@ endif if WITH_EXAMPLES SUBDIRS += example +SUBDIRS += $(PLATFORM_EXAMPLE_DIR) endif if WITH_TESTS diff --git a/platform/linux-generic/example/Makefile.am b/platform/linux-generic/example/Makefile.am new file mode 100644 index 000000000..84f337387 --- /dev/null +++ b/platform/linux-generic/example/Makefile.am @@ -0,0 +1,5 @@ +SUBDIRS = + +if WITH_ML +SUBDIRS += ml +endif diff --git a/platform/linux-generic/example/ml/.gitignore b/platform/linux-generic/example/ml/.gitignore new file mode 100644 index 000000000..d845f6bb5 --- /dev/null +++ b/platform/linux-generic/example/ml/.gitignore @@ -0,0 +1,5 @@ +model_explorer +simple_linear +mnist +*.log +*.trs diff --git a/platform/linux-generic/example/ml/Makefile.am b/platform/linux-generic/example/ml/Makefile.am new file mode 100644 index 000000000..3692b704e --- /dev/null +++ b/platform/linux-generic/example/ml/Makefile.am @@ -0,0 +1,46 @@ +include $(top_srcdir)/example/Makefile.inc + +LDADD += -lm + +bin_PROGRAMS = model_explorer simple_linear mnist + +simple_linear_SOURCES = simple_linear.c model_read.c model_read.h +model_explorer_SOURCES = model_explorer.c model_read.c model_read.h +mnist_SOURCES = mnist.c model_read.c model_read.h + +EXTRA_DIST = \ + odp_ml_run_mnist.sh \ + example_digit.csv \ + mnist-12.onnx \ + odp_ml_run_model_explorer.sh \ + odp_ml_run_simple_linear.sh \ + simple_linear.onnx \ + README.md + +if test_example +TESTS = \ + odp_ml_run_mnist.sh \ + odp_ml_run_model_explorer.sh \ + odp_ml_run_simple_linear.sh +endif + +# If building out-of-tree, make check will not copy the scripts and data to the +# $(builddir) assuming that all commands are run locally. However this prevents +# running tests on a remote target using LOG_COMPILER. +# So copy all script and data files explicitly here. +all-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + if [ -e $(srcdir)/$$f ]; then \ + mkdir -p $(builddir)/$$(dirname $$f); \ + cp -f $(srcdir)/$$f $(builddir)/$$f; \ + fi \ + done \ + fi + +clean-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + rm -f $(builddir)/$$f; \ + done \ + fi diff --git a/platform/linux-generic/example/ml/README.md b/platform/linux-generic/example/ml/README.md new file mode 100644 index 000000000..fc6a57c0a --- /dev/null +++ b/platform/linux-generic/example/ml/README.md @@ -0,0 +1,94 @@ +# ML examples + +Machine Learning API examples demonstrate how to use ODP ML API in different tasks: +for example simple linear computation and predicting a handwritten digit in +a given image. + +## Simple Linear + +This example runs on a very simple model of form y = 3 * x + 4 where x is given +as the second argument. + +### Generate model + +```bash +python3 <odp_directory>/platform/linux-generic/test/validation/api/ml/simple_linear_gen.py +``` + +### Run simple linear + +```bash +$ ./simple_linear 3 +. +. +. +y = 3 * 3 + 4: 13 +. +``` + +Or run the program with multiple threads, each thread inferences on one x given in +the input. Thus, the number of threads is the number of numbers in the second argument. + +```bash +$ ./simple_linear [2,4,5] +. +. +. +y = 3 * 2 + 4: 10 +y = 3 * 5 + 4: 19 +y = 3 * 4 + 4: 16 +. +``` + +## MNIST + +This example predicts a handwritten digit in a given image. Refer to +https://github.com/onnx/models/tree/main/validated/vision/classification/mnist +for more information. The model file is from +https://github.com/onnx/models/raw/main/validated/vision/classification/mnist/model/mnist-12.onnx +(SPDX-License-Identifier: MIT). + +### Prepare input data + +The input image is stored in a csv file which contains, comma separated, the +digit label (a number from 0 to 9) and the 784 pixel values (a number from 0 to +255). Pixel order is left to right and then top down. The MNIST dataset is +available in this format at https://www.kaggle.com/oddrationale/mnist-in-csv. + +### Run mnist + +```bash +$ ./mnist mnist-12.onnx example_digit.csv +. +. +. +predicted_digit: 4, expected_digit: 4 +. +``` + +## Model Explorer + +The example prints basic model information. + +### Run model_explorer + +```bash +$ ./model_explorer simple_linear.onnx +. +. +. +Model info +---------- + Model handle: 0x7fe8426ce1d8 + Name: model-explorer + Model version: 1 + Model interface version: 0 + Index: 0 + Number of inputs: 1 + Input[0]: Name: x, Data_type: int32, Shape: static [1], Size: 4 + Number of outputs: 1 + Output[0]: Name: y, Data_type: int32, Shape: static [1], Size: 4 +. +. +. +``` diff --git a/platform/linux-generic/example/ml/example_digit.csv b/platform/linux-generic/example/ml/example_digit.csv new file mode 100644 index 000000000..2ab0f4a0c --- /dev/null +++ b/platform/linux-generic/example/ml/example_digit.csv @@ -0,0 +1 @@ +4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,55,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,36,215,98,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,36,249,144,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,34,246,148,0,0,0,0,0,0,0,7,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,39,255,139,0,0,0,0,0,0,2,95,117,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,51,255,97,0,0,0,0,0,0,8,203,211,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,70,255,58,0,0,0,0,0,0,13,238,167,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,111,255,23,0,0,0,0,0,0,24,255,110,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,209,222,1,0,0,0,0,0,0,62,255,51,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,255,125,0,0,0,0,0,0,0,117,255,19,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,164,255,60,0,0,0,0,0,0,0,171,230,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,24,235,255,178,120,89,74,72,72,72,74,246,241,121,141,153,148,83,1,0,0,0,0,0,0,0,0,0,6,121,231,255,255,255,255,255,255,255,255,255,255,255,255,255,253,173,14,0,0,0,0,0,0,0,0,0,0,1,19,44,63,76,83,83,83,83,100,255,192,66,52,45,46,34,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,39,255,138,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,68,255,113,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,104,255,84,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,147,255,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,190,255,23,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,229,210,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,255,117,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,91,255,34,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,120,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 diff --git a/platform/linux-generic/example/ml/mnist-12.onnx b/platform/linux-generic/example/ml/mnist-12.onnx new file mode 100644 index 0000000000000000000000000000000000000000..6661bfe3c5c1771594fe50f180df38f4416f2c5f GIT binary patch literal 26143 zcmce-cU+I*_dnjAR2tGyG-yx~sb2Rvg~~{ShAk^(rj$*4DN$LGj7UWYQQhZcBvK-K zM@DvJ&%UqsXTCq<`+Yoq|NI{JU-!Asb=~JW=Ne}`FBN5Ng`vKI-iC_(?FQO8m?=*S zvzxJS$vi7r*I)IrY9+F2N+TAAM=fzwQ}dq|F>OKUlF*2O+Fge(S{S`(e$<k%MGN~o zhxkP;k+OvJGwl3d=w=H4c(u_}>ohMkVqxg~kjOdH!b9B^-4%DqD%)tODMv1e2%8xi z=`QOo`wxnNn##1OC5uABr_D4|3mEL<84~30?>T0dth`MpHO1Kxi=x82Ng3sAwAEB+ zhRvV$kH7zW<Nh+PGHPCG{};jjNq8r1P5=2(k&gZTQ);DuC78(%o;mYh`Wc}1-+8n( z$Apdwa_s+4jsBHjrZ6USe$>CT+90d;-?_B4{TD5oAGUC|lhc2;RQ(T{nX=EcW&e2l zA3BnAm-}xz`Zr1!H3g}9BHd;F_4x;r-an~L$=5G1z%x*)vVRe&QdaGMkw8uTKTPR7 zNL&A(Z06wbPioQluOI$@x8_K-|C?z4JOF2Ht$&!$!Qr25qx^4_|Fq~Jwg17Tt@RHJ zIyemYPs=F(8|9yjnk}pLKlrqD{>_38PXDAXwf`jk-|Z^zF8|-``frr~Z+2a(_P>ds zX7-;V{Kud|B0~RV+4cL^)u(Oz|Kc(15*W51G_t=`ng8C*OsbTB>EK@`)O58@|1ozQ zTpZPm{`u|S+c^B&uk`-ckN@)@(p~4D|1m}F|0$!||K6?ke~SJe+mXt@kH$>NXW9~< zsQLdg22ZvB$@*W0HQ-;TRR8}nHU2TBq?G@dQrfEj*fxhnObef5v)x}sOGQEYsA<WX z#rP{0SHX;9-(gHcDfk-vf+V*~l(hRf<xbKjZOijiyGWniE^5HTMP9V2Zzr6bsfT$Z zzEF-z5boJDhgVhK!ML&E@G#JpT^<rozURTOZO*{zC)M!!hmAP1$9!CMpp}n4GnWLD zw|K`|4*VLnvtvoN@cmV-&}N#AJ-XSW!=$aWvTB3C=Wo-g^g{YIGzyO4cNZsz?!2Va zH@LLw0X&a917|vC(&x}?qE$>Yw2dpKfu==p?yVy`4CupI0k^?(ZGYYze}M8ny~9q= zzX_Gr<{TC>3YWDtQpA8i{G$2*nw=krjk~Vl{P5rK$3BA%`8|KsXKwyoA|BF~XR}r> z5W~*l<#h|#_|{c?mscT}Mdw5RjkeWI*(>PpdN<H{tweK+y5Rf0@@#eShM-@gEoPJr z!t&0l<h}BdID5V#It9CuLi0qh`dcQ<U3ISN?ILgTt~6uUSC8PPr5*SGRfZ#0_2*HM z)@V<fJgU`!kG|2PT|+Hk`34VsXK4qw&h+M$Ls~FtR~{VGe#~)2BcWu$Vm{<rM_;#H z;nTmFD^i!EPqJe5F4Hb<oy;}dY@2UVzwyqv@z+bzm}5o(CC|xatWniIb>AxeW!I?x zukFGY&L(;rL$w`8;aO88=$>@O!nOwxc4|1sj+Gbo?u~_^i4j7KkWCIP_h8+-6)f{? zxgeKU!dpivQSnnV*4Uuuc6HrmTCS|%wqMQ?4><eqqX|dg>BbcP(&-yJR9J$G+6^&y z{5JG_G=<$=_QSRnDd?Y-!<*Ko;OV!1d?E8MZCrj%GPybw=QgQh;kM`E<{OzD*jg#v z8k-2O*XglmuO{fVt`lFc4}}*!_ETV|SCWo>0bG2u6{bcxv#aYVoDn#Wf7SezJU%m? zueCMN=?#^z`+XXYP15D5g=Mh0K9huQ>UieT2Y4{zLRF2730>1Y0uB9#z$cxq^cxF7 zcf~=N@KY5Py2et!Qzs;2ym!Lix*GA$kS(-!)iPAm_rbH9<k)QfX`XT>p0fk(@bkMI z>UFBQLL*<3zSzd%<r7=b#d;uyFTTp*>4W(1upg|Z;O1sx+MizD>_vQkHYAyC<nCkA zAop-O+Yi{ni!=A4ZBPSMo;iu7p`Cco(Pflypd8ey|B(60uN-sX8!gzBONSd}-Fj@c zL`@A<=+{3`)b@M=50YKDt!prkY_P-iQ9-z{`34lcu)?6zf%JRjR&1aCg)G+%<On5g zZu7t@^J6{u=hhWe+f~5Mmg#7he2Ts;ZYO2WW?^5S60EJ!;e9RHthO<ZUprr*+Lt<R z-_1AiBzwe}7Z<Xq<ceM&t;KdvV;ZuhoD!2P-D2n4;_N%JsI~qgS@}-H+VEt2UN8w# z77eM6uJz-V@nv|jrW22C&O)!YiMZ@i9hCQr=2uh9vB+r-_uZ5(Sk$-*-)B0a{F+hV zvuP*H+#sR8Q!Wa%<zq+-e)5T$BwFa(2*YNk;>GUfB)8!rG~1X_bJKQeExLDVdW#~I zmFM$@1?7-2Hizvm?k0m#Yq0KJZ?~ylIUEyk6jj>p@d%w5vdgjtBgeD!Sx=iM1na@% z_e#9d*hR3pYD?o!UJ>Qe26A=cFu3CVN}}>plP-7rM%&&PgVX#GJoAweb_`E}r9<xt zS=(LtbfyotXRpS{UfNjs+E-Llm&N_ZbHux4Am+8dgdRtWxd4=Sm#HpamYs?neR5D$ zJqC*{`lH6Js}OA&&j%x)(*ef_N*$;Lx1D-ZgZc@&IU!R#xJDB)Zd@mGyY;Yn#s(ZQ zDg_eO+VGl`WT;<d#pYH`*wjA;QisRUxCtA1NU{W#ZU5k$wk>$uU>(E9pH+S%|MFAu zFutxj3xghHuur8Yj{dTYSG&zb>o3l@(6^NHW9o!HnqTS4g<_m~B9cqy{}TE)6$|US z&8IJ8y+wQ-%ih}#^0QrkarcKt9GY@UApKD8K0!_r60nzz%03GtavZUJ<3zMNn}UD5 zAH$Fj8`$Q2A<odwz>nHz_~6z))tv_3<?&A(Ic3faY~QKmW{n-V=!>L!_Us(m-&2l) zoVvRm{+y15?_AjAT0bGceLcDl9|oV&J+Ps;3$MI>8sv9%;Tt=LQhiKkOp<#ARs~(E z=6>=LhUm-l`PQ*Gp`{L*KYpmXYF<UH>yc~jR)d@6G<<Y<nsB#yi%{Tp8J4BZgN%+c zusV^z8u_E4Z-Oku-OmNtBa=8Z9C&3@DqeZGfVVbBVN&x1)QeN8R`K3VjrYf4)IK{r zFkICw)Mpf{2Uk>uyc)(mW&TpytTZ^h=LeZB41w65xvcY2vvSu?d;DbQ>9+G&8teHk z2Gvj6yu<g5XzEoi4k=m0@A_qM-+)HAYOBIpAJzE0sUZeit7D#D3F%$Z#(@Kxp)Tw( z9s03F^0D6mxII0LbdnXN`?n41UwKa!%M`I-`a7`FKM0YInmpWoeO1`8T)OC(3)<y+ z7=LpUnLM-NZKFMD%J)}bwAq>)=hV|zZwJ=Bc@g{<27})lZJ0eM*(F!0g_Mu)fuQ&> zvA}x}bSkMKjqpTrf3#KL{db__@pCvi%>f4vOQN8+^5C~R8P2v{qk+w?RQ0-omUKM~ z&L7k1){O)BKDaaMe2byWz5AhW%{=lc*I<p*_53JzVb$xjK3wd686CIX26LCM;<5T& zT>b7=)zr(mV$~oLRX+ss+3D}0)>9yV@jFP4e1y@@%W;S5U{-vt%oYPCqkP>)y8I%V zY8TEGx4gIr8-gBE^wB>;Nb3dKQqv%mUXJG7ogWL^HPs~j{60fdQZ2{??Zrj?Y&h_4 zZ?ycqhC>c0fZ3k2(6}ZJwv4XBy15M;>^y{hM=$5g^^A93ALrAdrIP#IoJl8b5gd0* z;(L+^Qt>F~tM3D_>%=tNc3=`J$(-a5I@)5UPXjxYPZd0#-4V35cEJVH{CI28dX(#V zze2;rlP6T0VdW?idvAInJScn*pK`OrL33u4f9qZvZM221_P#-t7wkac%@Ld^SI$Nb z32>qdlf2e@u*jZ>Yo}!6rGQi1>&-6S<8&NW9BSl_<`K|mTq2%maHSZZQu?BO9`#<w z;CG*|g1e3f_nBrZ{BpA5@^x20vCmd$&B=m*!Kz$2&V!3QHcQ^N2M9{TlEwP`{u0|Y zdZJ<V8%P|L4hOGar;M-fU|8NV;qcTa!t~IOV)^}J5INQk@7mv^mx=GlHm@_g8u^K1 z7T$zaPqImDwu3jvioi>AozUk;JleaoP=}#5?@T-n50vNQX=6)j;BBOFXg9mmmWzcA zI-HlBLm@V1ylz5IR(zdInxjtOckL&V(N_&2A~}<j6}Jh;M->SxFQ$W5j4$@HJr5I0 zKGLvdU*Y9~46$INAEXyFL&$hL(tr0tyf|Y9jto#lRqb(9<K6*DXOF;$zazn<$_x7} zF61B>!|uN$S?xjw>mOJSv(FC0zCHFr{6c;1obSu_9lJ<T^N@Hgz|_q{O`f$pD+x{S zP|4*ST>dGPru1C^nkS3M{Ps>kq4f~v>)6nzZjKxR%DCqELUEq?NBS6*4!17a(OK&* z7&I>q!i_z-*5f73^YRjJuD=i3nYp-lcLzmRb)`tv%_uV>iMt+{NO`KIbg%h4v}eeO z%4u8pyG<d;U#P*j%g3Pi_$0V`wJUxbCxdktX3)^vF=%I8S9xfsGk4x<#ph0cAO*=g zsvmQoR`#($gL@}HR(uVXZeDDtX~aJRL!sPJ9W{pi5%ZcyiZSH}z$iQs%5#^{k>lO? zyPN{HsIS1BowhvdRyoX^rVV%BmXp1|P*piXmA9;z&q}{d!Dy&0e|_nM&mGTDdDAwr zG@ujab`<dZn^Vy6XAvGs_{0}1p5uhwd3=7NF%Qmg#p}OM&{HLA%<63egOw#%+9;7! z|K83cG7a%XRvKm1+VF{=o!Hf(4D$9{lKpCZ^73%u&X!iNHR~@_6!xWstKw+#Q#BTs zpMsrpba3zP5x8#scd?hL1kN8zg6g${X>>w4)DE)*pYFjlDZy0gOM75asTz)obr(l@ z#X{H6Ct@$(-GbT39n^PJZ@AN_N@mfSl;Jgs+P-w<+>5?&@%d6wZ~a3k>{Aajo7FkE z-I9tWmDGLwV$A)Vz;CV&q+a%Kg|6~}aQNvOiB(AtT(kHJgoFmNQqE}_kr;;yBu2b5 z=M2B;X@D0FT!VMxLRj)BwDO|HdwQu`2}ZqC+$IbGoIcqE<;wCXRdYLRKNf}Cpp};k zN|Lmj_1vvNkz30fL=TS)_&C{yJXa=x>-SjtcG3+}N`1r~-QH30rG@<GR0dZCgos(? zn^^bZW&Y4y#XR^R-tqM%wy>on!$h93W*fd3`WT&7EP>DjSux809LtWG%HvyOaLq#p zJX+eHch556Hd_~bWOEXeW;@azdq%lqt^B9f4c7`|#ftgbJkwvNx_6)gkB@Rdqqpa9 zm2!8yGRmA6e9{M9z4c=BjLvjP*^%v`8G3Bp1}0KJ>AIXX88@cEIyT|M%X(wpm@r)T z%nfc#_$(U#xdA8);cH*j(EZ{mGHFP{#o|WJd}ZnOa;`f2`6R&I^shqhy$tL!b_QF# zw7^B1qPS;$D2JAYi9K~*p{f2{@lovvHXWMIT~`i7)mPbY`tWBe8zPNu{F}jhsv|dy zFDJNWk7}PIX?x-;*lKT$+Z2y;51}(@hJT>H6IFT67jxEkO~#CV&CqUfl6K5<;DPI= zko|fITp3mfXZ%Vb$EyHhGnTU4Rz@YUk%AWY;NmoYh|d~@A8S@{vxXW9b()fafgR$^ zwD0ipy)38o9Kst9+Th;&AA)U-30{;NN1lUj3)^1LVWVxT{3l@t_L;||=K!3Z-GIMi zr}I+pepq|HU9@+80xLeAqTjXCs>0;E;pWi+pkQo|<JAj=m%Y>Zt1z2?FVp3?+dee! zOgt_eGzfkA-Q=y|X9bP3{n#d7t~$QqH%#2UgXetf20X+Q73JD-^zldZ;OhlR%Sfc@ zlYelGwIw9aKLmpk<=u)sr9Sp+16-P6K^_Ap^SBEu@Y4`AJ|1*S8h1Iny>*o1jkd}B zV!|(ovYI9N>}guva48Oan?49#G!9~W+DEb1@Mbu5Ng9_q`tkJZ%P=Oa8FE+b<MV-C zcuq$=-L4vi5#Lo{(%-@2yelLO>81#-ruAU=L{n&+cbDHO$fMiXqd0wi7I#{bia!np z<MmfVc_J2J$jt4W_S?Fuz(R)41w~?Z`AUrJ?t_ZDFX-5?SaurtlA7EP@Q}e1c&TG9 zy&QcWmxo30nAg|Qx2&CJzp{jB6T=~8?0qc7Vs1IKReT$^jn|l+BKevdG}Ugj#J56* zcfB;G3oDy=_-q!=ISyj2+yd-48por&pTeV|m*MRydBo_E68C6%43xW0?OyXx;axuK zz0yM&hd{VH&jrJK@1XU%@j`{wa(=mLJG%9~$65*NaHE+q?DvWl%I59H>9M7v*7f0_ zbSYSLnVbRFPt-^<U%T?9D<?sc@K(@KFd&DJUyyKV4fKB54990zl1fEiTsnOpkG!kJ zElu(Gw6Fu>e@&z>>W?HNCGUkU3suCHdI?Tc@DmhTA}Qovt3<`i5}gVIz^zD`R`_-0 z(0;a5HR*z|%)*jx&Ce$7ur(C*>=G$`*h%X<I$+Y6Fi_lg3_|88;Fu8}Rn8Z$(&%)1 ziTj>Rns)LW^p5U<o^Nh|o|O*DjrL~6XdnExe+?g9zfo|Wb03VS{YHzpRG#bcoto9< zcvsL^IPgA|KY2bNjqxLK(y}0Y^UWRqY<ojiemi)}&^B7FTgGF372Vd1_{av~3;9;C z3GUO2$2P$QkJgm&qWEHZs(*lnFZRQ`Zua=7bO679<G@K_Y54T!KKlA~Bb;b^!C9Zz z!S?pmv`ak|e~;E6c<2mw3||Vv?@l7a@$1m8>sOFj9!#lwXF{(l*{<uOE>fn&NoedB zjm?EN{AgiscJg#a{exXZ=Uc5}pDK9{Z8nBOs`BCog_~$GK7bb-k>^{vs<>NTscKW@ zdhRqhhd;wFy35*pa@beU<N~pu;V0pPz7kn9m%{6FTs8Pw23Cw@n(?p~gx*ac$%j;! zl75w+C=bNeAwG0f_bE3GP{FiuK2?iLx>R4jcN`;HjQL4UHZ0gvigTvKi2FWm7uHUH zLjJXytnPULOqcA&AMf|z$xgae6>7@hRJ)m0uUNqAs0e6K5h|Q&qm92h^QC1UpvU8O zGPu7SFGbi2J3sg0r1W|kFW*E_N4t>O4R2mI2~h5et?SsA1H?}ONRRcUbJ!?||7zHa z`<FZc^(O_ak}!=L+jFp=w+|N67S8#cP0we0;A8&ASG=<E>RU}*V3~<q_Ue&E<~}&4 zVb2fi|I)aHX`Ez!0Z&go#)-4*$-=G)_dn^4&T-)wzT*I`(_X}lxzT8^90(ncQgD2G zJt-g3=Y4O?cuF4u+)O{g5>u&PeoB)M?heGSzXX0Y%M35rbU@?PyJDrk56s{LLb3G% zICv|bJeRH&$MxxkpJv>m`%7iHmysn~q&*k*TG_G3_MXTEUt#=XE1p(efyOZ@oYkcZ z%X*yyNpURfe|MJ`zGv!Pkp^A&-2lr;Cs2QawcuO0kEV_EBemB)ELN5Z*X9ejVP;?c zyQW+yZGKGG7Hy?*PCC^EJx<^;t#7b+&KMkUM4q;7RmR0u5q#d5Y5xWVF7EaS-p00b zk8F2R@4u24etJvWzDd~P`W83Qe=KiW)ros{JwzYuuHui=eQ@f^S77}1H<-LS!QW?j zV0tw0ala(c)bc^&_(Iwf?7@2#57Cj6rPSHKysBTnV6GT)7~GGHk%YH*g@D_J5c%ah z&HZ~5F7-SuG);U$Q>TSXa|SbLc@N(zf4Mr!ww(>dql+ow<3`F<C}O*x<IugeGnxJz zitE%BF)8aV+)0SwF1C*#X<j}*JJ%P@F6^YuN95eLT7HDt;SwCW+n&FV8jGGUx3YQu zSsb|{P&_<02ZHTVx%Q|pUYj_WcK7%TBmX25avF@7m_&<ygo4&d6)NZp6nXU-`36;2 zW!DW6-n=Ohe-DX4)#S(I`&nL`aX*6VzNA!DR*Xf{uh&Q?tdv%|gplF<{^;^73C~{Z z!LMIU6zi9&@|$1jf~@fq7?=A4nhU%0W0TMDTEPO0eoVkQ`<*!7vJ9L*t!1U5^QcYR z6YLycig(B8fW&(;eV$?tF{8b(<MI)DGW;CmF4{+HCdy*NtY%s?Y%`BPwSh+ddO}y` z?nf2jF&yvThYYm=>wh1ksxPC-xTFf4FKnX0^B&OKTjQ};_7s%dDuwFkr5y8YjW97% z4?DtRu{%^!!#+bS8SF$*q>8eQa@?zEI1h*^#33Ju>kpjb?>Bp+Qq@RqZ=c9NCk5cK zXa!E|<Vj&xpWum-wwQh~6<P`x@x1p*k~@)Q{3JP#o7Tv(M&^2PZ>TNJwQj_qz=0eT zos9nV1EqVLA?%oLPfN!x;9e0CaP*QXKhYX4Oo%uMZ|er|i}TZ<wzwB_#8I$HyNY2) zws6wLwQOdhhY!OP@cy$~9J4+IKF{yYDM_<1D!3Q|eR}a%w+b3;I0SCK{0p@7n0Vk( z1wE0TF~b@z(NApwt);%8Bx4O9I48};ev~J(B`SFC=u!&c?a;l>fepL5NOL?pcvQ^; z*c{p+aq%mINeP)eI6#}Pj+C!1xibp~jFH8?2J0{h?CHrAx9XbFUSPA>9{+|W@cl+l zH}l>NY=7<?=ZEUL>D0t=Q~N--gZpFn^!EL9$6gNGa+cGc)`hrL>9t_sb%w@Ic|#fd zRH56Ep?L3Q29?ekK}mk;^lMlk4NMroGYpV}j80I&b2F@6s1J9QilAtt6SOS9El#8j zB*r`heN!c2xo$tMEV97Qm-djETQDp=9EP>?cW^-C0~pvk6&|`?L%q7uLeH7NZn2GE z_3e|eJ9;J;?6^lK4rOs`Lw`xrp>1MO$TMNSL8~YX9f&Trn<|q-`||U4SBMxa%X1rR zY2~|nv}0);EZI1M5=n)X3*<OvXpL}daGBtjm=EWt*l>JsC`OEP$JIl>({lYNXi}8r z?B5G{+1E`pC?L+YLVdaTZHPA)R1D?`QR;YS|2QnEJT7+0PZejjtKqB6%TRLlC0^Ut zn++$-<uQ{oh17%z{H*K(eZS_59(xk8X7MBp`=o<|9vzmPne!7yt^Z5n=QyG4mK={3 zO5xGSP|h1x3Xbyz(bn@;w7F*|ab)%b%4~Z@@)yl`L;soRUh<YwKlkIf;BBI7e4OYj z<12Kvy(p~sp^vH+Hkh?~9P6A47pfL}@s!ENoOj9&A1ZEx=;}`t=U7j(6ZAlTXg&&S zYbZ~piITNn;L%B5d^%<c&aqSAYV`-gi|O$=DAAiY9FpgGI<n+gdY6J_jA@X!Jh;qT zOAcN4k?}MYoEBXK8v+kN=Iy`2@U`|r*-B}>Xw6$r=^|i{Sx1F)zxL8E)pf%9;RDHP z)OOfg-5Is27vZSMd&TUQR9Jj-J!P%%g+KR=V4&kq64#eeWNaM0zS<Yv?OW*C?Zsdy z^9GaW>2Onx5!Tu0V(r)+G}AJdW!8)%vCDkCniI-C9*;=9XfU1e?u@?uV|m$@?zqFC zklxoh;iYYhaYlkOce^<YX63o!>=VnVv_=a~A1@(&xgMfH#T|OmtC4gaw}5(RFKk}i z3l$r4VBUumel;i+QiF!$Wb^)bM!A?@t{l%l)Q<>_rIysMv<SLKcvGuWUkceWkuLg$ z<1_gTTr=DN>i0#WY+kr<d&gCJ6nhBLlSfnX^{Ybr`382=JA&HQTWRmWIv7;c9UBMF z;}=%|Yj;eCXSyw7!2Z+pd3*~!Pbj8C8?A9!geRW+V9BFTZRDWH8CB80n@Q_sGNztU zrt0q1^vG6&6Rb8uQ>R3{zqlV3X2tO_*Euw(aTAz_MA5_qk#63e#k;Ib_=e`TN}B^I zn5-|0!tVl9TB*Pj`dWc)x-s9F)rJ|~F}#1lJ`y{3;~3v9aHDoV>y1sL2}!>tv5hgJ zl2HWSHhPDBYS(l4oo>9+DUWJKFUBP{0pyx3<F?LQv3j^sH^H%@pJ<p<gPNs1d9=@c zAwMjJL*A*N>A2a#(9L1en%i0YQgr~{qc8Qf%;beR1~|@0fsfZ+h4>@oq}{TYrkXvG ze3`7vM}&5IDsvRB7yKsG*B5YL@(9V<3U8XFF_o+8jB%TRDlS<07FMp8r?ZB)>Fdd0 zoM!b_EZ#kvU-YYm0cWOh>Ahc|8?M2Vl>5Q3qDz#0BoUq}eh^09Jq(L9NAm8>i}3K% zLveL@H*VN?MVxwX365VmS@g7Npkrl~6n89MSokRehmE>THVbNKby8o5xA&vsbK}{` z^dl?|84rpp^x53!zNqa|4ZrSRp<b!AqQZ3_Fmo{x`^(;iE(HZ(+QWd8in5{8^aId0 zm#Oe4!wa=c&b93!E?C(M6AfI!Q$?;yu1f}84o8;k*(`bWJAmJo%>nlpy|8jlPky3s znF3_f;pf2;l2Kj*T4{-_HAuquCz$ZVis|SvZ!7vQ?SeMF9tyoR2eV)BO?uViIgEAk z#J|noAT4x_L>6tyzP~&#KYc~=GWIVSxJ;7zYmxLoRD=FK)6p{dVbwd?Itr2fB~FRF z0*<RQVC2+Nv2SAlS6f-&8#iek?Mer%{Pc!4eVfH5Z|8}@`zxXSy9tk#`cJwe&(X4y z-8^hZPaOWZ6Nf6dz-osPu1V337;cq~dTy3DInIssyh<rdS|_W%d>Nu|ouKj~$-=Ub zL{bg!gw~Q+etB~#zDNwjI!#BKG9v*S4-G`4XR7!jbuV`r+93A!Tg|;+hH+<~&8RNT zgJ$*5sq&4@hW^J2ut6gWs@HX`s_S!~-bwwyq_(KaU(3>@YxF)j|4iix!>dWr@dVrG z=W=4uaWuJ=Of5Nk_@QGMpRk=y8IM2V=#jhe6SYZhWS!@su8lORvjO(1`zx3{D}}k= zPJ+SeE8^Wl7pbv05j$NUf#LY8%I(*0^hlQaqa9BHpPr&2MYh=E!9rG6n##+HM)Hdu zKj5>%IvCMY7n>(cW7FwKO_@dF<=us@iD_MMa#RdkE36i-h(|Ch_$dGGS0dDY%)=nZ zaFDr*;Cy_#GzR-iC(ZMv=fgJI+IpW%6Sh+R9z$BI8V^}+FJbSsPHeZt249R*go=d^ zAjGRTP3dA*)nb%~`ZM})?fi}W;!8Qc>tl*?e_ip^5j||%>&)&?mb3QXRJ!s`4;A%f zsHNi!FBU6tZt-!5*_=cAHJPL_;jaX8?z1%6M13+YV&CL^4qp6*v`wS%rKvR6^GbwO zQwr$Ymo)mmwSfO*jKk-7D*Wt3I8RC*#5cz+huFjRls#bz77YsEVJ$v5Zu&k@TpNlD z&2+05$#|0W%n0@>If(IE+E}c%6_5N)Wtl&Iu&QsASo%bbD`v+C%O=kfdi$xlEj@pi zT`7S(hAn{Jf0HmRw*ch!E#oNpZ_r|N3apcV;-tty_<R3SiY~LO_N`Ci&q<v*G%OR( zk5Wh9ebOSAoi-oOtK=O`c0w>By{~*uM=a)J;=N2V*%gD!<UT=%Qx3`%?PAq{7cg(2 zLUk9#llc3FCoj7-fOn$@nfx)qQ>Uu1VNV@>NOGeKMZ>YWOohje3?b34lbCXFG%tNV zn!Q##@a}mkAbTR5*68I|)p`%%D5p<g)U}d+Xq*A7ds?{PY&v%HZV?{Y&7yOQ%y9Ta z9UeB*3is?cW<@PqKD)~nZcP(u{@-(;Qg>VI^mqW7Wc26zu^Vx3>~|iJznMqOOG1sb z<8)}r1TH?`f@9PgNw2LAeY<w9HdI;yiNB3`N}&oC+P7n&o&@BbhOneX-L15tjAycp zo3d#NABfyT9X99L_x29tHXDpza|xP@x{8@sXTo`@UwMDN7Hm%lArHGfRjL`v{Ql`x zdh+zI@X_hJ=%jauqNf{C;<X!8?NLjTtyjpcdLKHcrgFj+ZwTIb7u>oROWGe<^5OoC zkQX|cj?Eg&f6sgdYvm;D)>)m$e~=Th_Eh4Cb2;pE*AT7k{s?-*Dq;GMx19TR3ZHZ; ztn%uvhe{hJ;)5S!ux6(gD}CyTW|uX@>Ngij_Us-wEU(E9_g2Ay`KhqlFHLNieF;WC zok>eJt`r`%s-cE~3_G4zpi!CYK(>1TyjXmaob<fdc(w&}>Me`2$CQYbkBy<r+*FRQ zuw~PkDmd%IFf_dRPz;<uRqR*N4GZg@kcE6IA1koKnrAzCR=6T>eLn)?-kEWqPisZ> z%cEh9xh5K%yG_T(>GOkzJ}lA6qcPGLP2+|~)#9N}ylK*KAt!Mjxf-|7m-W{~$Lws# zcqxyuQy$Q}lZI5hMVkNnI9Kef`GPD?GiLt@;w6O|_~7XyIuaIuw^^N4PZWsu<0rur zcmwlPBk9TMEIy|;l1(ZTaDuZl#p@`brn@pN=yVfiy}LzGoh!KD##Q#6tKv4kdkar@ zX~)Br^KpRWAfMN{g1PT{@~xg1Xm9Oo8h+*=8Te<>*fvuf`P2m`+uHKq;xp7b;~O>j z=L+A_dg1Z5G%|g+2ts`N(dIW@&_lKt2X{kWLQBB9M=ClWdny_nRpN{wM_d(P$U?Cq z=9k*Ba&A6cQOV+5kDu`J7}3GlPWa@$AM3fylI%`#$MPAk1ivMQV*k$-!gMd``f7FN zvgK`fada>4=+_AIOm~TKu#u+4^`V3P+9hJ<IFfaKM+!q_*<g7TcpuQ_Lv1%{XJ>2Z z_NEQizWhRw?gMatyd#@^jmL}DeQ=XYp7`sB2OIe6b8i)WJU2UnuWx<`xnAR7@RwL# z^{5V4Mt-W2@%Tmo3L5;i{Q}7-3<Fo6e*E~}MCfdl3ck4~gh5~OD+YPUlY-I`ni!V? zJ1vbsJ0Y8Xl`Rmz9=Sv&OBYhxiuE+(`C78@cudFY)zR?PU09}W$NqI59O0eM)3&FO z@5of#l!Lr4w;pr4R7>mb^I_xaDL8dYA`YIK0z7EH<kL^N>Z19&)Hqxbmrq`g_Xa7z zV~uLIw(}KESmm&OP%>728OnDg@v!?wGtN#pEv*}Uq(fiQ@Mq>OII{FEC2X1tKe8^1 zPQi*4Al<vZ?X#yQtyI|dF;5t|K>>|+DzU!uRcN!ggzvgeW51zAy!TE5>ps3FX4<=R zSgr*WB`CPX4ppF~o0Q?lm1J7(L>Sk01Z4B3@Whjaykv+!re)ed&5dqkJS-mL4kWUC z@Ky|GWoccz442((;Lv&JM9U0g{1A7H6*peSUal2YJAcKo?w4PpOTu@0b3==d>qbEM zwlZ?htbl<xbp`v*Hznz*XC>kT74d<Y6K|OH0xSNQaBJOIo-Nf;lv^)+F(i?Ko|e0Q z^8#-F{es}zWg4eY2XlM+aBq`xiKA~N7A|@L6_eg^?;R_lcThF2y{_Q)^RYZG)orIf z<1{$GF%Uz1)^V&|8Xo8~gX1_Hb<XM0&ek6=<iB7fZE|Ny(Sa%|iYki#KnKoLd+J4# zLM!R;pxadrtK?Zvxf!k`9faWv`*8f5H1@f_T}-)~0Xv$^!0^4h=nxi5$}0`|*YFqQ zW{?S~$s=4-PZopcOc%B})=r-5Cre_BW6)rz7k&!}p@D<s`JwS$_R37-^jits^V>dn zXMG$pWlRO3Z7!P|#nZB?GGX5^Z`z;I7Zp4f<FA4FqJfthdyKOu9Fk0O+Yy6qyWo#W zy8Ly=SHX3FCNC;_2@A|`Ls_>mAYV2gX4jc;vx6$XjoS_Wo8Ci2h8k~I=%9AvK+d?@ zg@0G9qz%UoQPi)&JooK1u0J=KyCol`f_DjU=xhS7yMKdJR~B>r6IHg&m!)N8t-`x` zCqbp~5y^aTV_CaUzBBqd%>Fxs40P{`g6<K~xBDAmnz9G19bI3ECvJe(sN=-Hr&796 z9`v7mN09#&&o4?gvf1t+KD;4IFnO2;!l_R5^YeCocTkq+G}?#>D?WhMx4vjKHkAz4 zOoP8Fj;PQli5863;LCD5`J3V`Qd>@7ma`3S%~}DOLqqxQw<#=aIZE-%I>WBGA8_}9 zHXp0h;;AR@@C^AbM3=_HoZhv<=E8W<WZPE?itWYuX0;Teqz#lcS!nRp!H`wEg++S~ za!1@uc6ejSTi4oRb9gI$H%&!8)FG86gfCM!ZmHT%{<q`t+>vXr)-MsV;%;-;l6=0B zphL<fZZIP8HMnR_px_1S{AY3sPq3T7ZPkCo`?*u#%1%4BpP0k>h1N7Ma}{?EsfRJ| zhe=P)I?7s|#`=>DxN`7evfm^tNlBRuD^|F{nU5BH?>GzNHzvZm8!M$d$ubCc9V2-E zI78OQ?@-;Hx!5*fG+sA4Mb;}<^QI~t^t%=a2JSUt$mIq|`5s0*Zaa+`HIIZ1qv7)y zDf&@&Ef^0p;q6Z~dCAUwf?-&TU}1ihVy3(iZs^K$PQ)Bul$XlB;VRhCk`KRb{3e5^ zx~#3?zzt#BV94W%SaWZyXgChVfQ+7eqrC@deVr>F=&4C6S39BH^B5f3+Yyq~>-cd- zK80o*;IN(nINx83(=T{(e69mUUOWZEhEG9F)q~Wm>&H7QlsG=Ef$WAosr=Ym<fIEb zd4~OU{&HA>Dywz);i6*7f1g07%u+by<zfn2*F?_mbm8ROPIxxifQFb=6YI4|w%pq% zq%53=H>!2S>%;BYN_1xJ1`-6ZJ9~5s790O4)0Ko$I_Nfom+8my87ttY-m|E6+c{_) zcY_@7_vP*8JsIx4cXji+C~42GscO;)#sv@KKzEWg4&Rf3vsTZeP^U7;_R>eFR7dmS z#$tVo9NxXML2OCRr?Ed3>7a)ZXB}2Y1Fx^b*6CTaRN5zI?xkGw3&O4Kw<V@B8a&AV zCFom3N-Hd>blhK#wU^h5RnMm2v60bOUK+-#OFD_>Plxk`XXWH_(uudd^ya9Dot$*6 zg=*)1CdC>Xur(Bf=gAe6ILVTe)>!iLGu3p{;|jd&HXqHkO7LK+8Hag27jM2d!m1m} zpz+NgmyLe_JDlY(A?=Rf`vcfz(j-pSHezd)VLVv4N;daW=>8`ce%L&VOCGJE@X)dB zvbc;^+)?67A3jlttpWc%)Bsm+e1=JH!?4Qkwb)@X5Zp)g#QA+ikZGNRCHu<6ZzDX= z_04A3G;26K*w%wKR=Sf+*jx@$(WR17YZ&!oC$%0;<W(0csY1trC0a}PX6a2S!nH%R zej8Vn6f}xDr22gvIR%4P`%u#dTk?4mOPM`0sHsg|P+YPc?Oi+L8*&oWCuwv2)Ms$z z!X~h|bc|N_9RWreZuC*roqx};;ip>u(rEDzWt_b)*q9%s^v}q127Qz)DpY6F*U?Nm z8Juo>mx`>%Rzk*3D$IKU_gXJPvX>c|^_0g)cnr2INy8QCemFBG5I-xJ!-EIiSm8i5 zMLu~zbEXgD1^d(3KfDKD)4L~*I#mmsCuoS%#w-=DwGU!r&2XA8zlTzOOyfkU{^LF5 zs+Szx$&t6Z@Ezru!t~)Pl(1n1{fY=<$1i<&Sm*~DG@+6{C?&!{uf^Qplqw#a+9u3C z5sZBz?b+=3E-aLy3U3-jvdSYnPInFA$Ai65{Z}9K?Yj!}Cfd{KuvE@!NQT`}oA6qL z0c;;{i!-k5<FRQ8JmF*x?bI30(MB6#<Lo?Pc+@6JTQH3_c;6AdV%*uronhy#!Q!if z0>jJiV!Fb9y5Zdp6*}thMT+<cb?G3j!KZLj)^=>&VUC-Ze}HxJ{n2{K6!bb#CvI3B zNLNb=ME{Pv@H6@?X?$n`yRLBr+uu`XQJT?xlSAW-cR`8bZozKVT{s<B1nN2_xV_63 z&a3ob|I$rldVVPAAM(Xv*QT>?P!^A=tK-!U3T!(*oowym@OtJ22->`e?&WvlSMjm@ z`d%{Lf9?XWzpnxFhl5aSsUtf1o`BQiNVrIO^y9tMYpp<%Nk2#v#wL(oiVS}>J4o)Q z<_N1ay1=t7R(vmJ1xw6{FWhwIPgNS2c;*^Fbpx%^H|6T}m-(n}0nGM)508EigHA00 zxM9sLh}7I#6>GCpVl`nf4_R6TaeGgYsx&Gs$g|{YzTK(VaTw{O1k#e<(kfQNF=5qS zIUX>=iw_M?<0+%gK=HXbcxTi(P`})ZdhU&6w|SZz=4t`oKTgD#_Upm!_jYWamqwe- zmy*Za8DdW1Oq@~l6)wxi&?Sd_TGx~*p?~-VD!FW^rNPEd(>bKSB}a^3$4xUgaz*1L z2DL5JxPLgyJYUBSbCNi?2AS{ALW_h$!pOIQt|p#?g!INF2>78&y3qlwyKW3CEDq!$ z(R;*;k9|QSEmu;kD@EC~EXG}*;z?e9B24F0eE#(a{+Kx$7tjsv*?ASKcibf1<Uw?H z*8`#4zYLZ(gtF3_xturjrbI<QNBrV^5PM1j;Mm?KO20Lq{u)?t_Dd6SgIh3*Z_D7| z&`T6>MV+6I@Fzn*bDlP9KUlRd0>i_z$wG=S)(~Z+=VW&*e5)k|^B0QoMb>;ivWT~D z4P|SMO+x;RFcxLk!`Ac;sGsl`4({}!pziEC=kx`rf0Br^zxD;=cU3gdDFxjFJp~8l zIQTJn6VAI6%vSgJNzrXae4{E&NL9Wqbob9@#VMxzVaz+ix7OUGH%y8UtD;w<4|B}> z19a|{9&f9bMygryRC4q-SSKh#X|x61n`=v_-{~<eMG*UWa4+o(v<>fz)#Vyu!W{|U z(^7=wdsAUx)NF_dsDNafO;p~|3%)CF6waKNg7)8*!W^y5;N!OiP9%@Owk|5T)p$QG zju*)|<B%Aqk_gsQQ-p=5M)9uli+pD91eo+h9}maGiPtW;kyBrH?tjOGde_uY*np+{ z=+6_<*xNxyx4rm|<2XJgdy=|WJg2t!k^J<;E1F@rgcAE+f-y&iRo!ws3;DA$I3YZP z^FH2(l2<mUu3{=hi(RHLz1eg$sxQkVw!qbM@@#Ap#;b($w2ZD&)8{6#T~tZ?{V&n^ z1G4<BFq5t%tJ21M{(NYk1~0Oi#Wg)jC}PPuVP&NSem~HYPw7PP933~l@TdrCp8S?5 zFCD^V54w_bRu?XQBaz+;(TBR`4LoVJHLv*!9Jp~Fm#lZ93)X>bV6v9#Ywn7BzgEz( zw;M!-R}(l^*^ZrF`%=;p3!Jgy7UhWok6&0woAb{LcVF$s?r#r>3+nZ0tPm#_K3EU2 zudbq3lLwx!G~%-F1*EuU2$tNB!@CD6BuOrD)MM9aaSy!`ntMGY`LtbB(b^&jjQ%2y zit$G^+3V06vJ!+hcc@V}5Bk1JaUFEc16{@}f}MSX`CwfO4fi#|amL-~Z0lt5j-N%Y zlD@b>#Q~r6%_g;U6OQn_0Iv=mA;SVg@GSpMTlNm%D^m=p$@V*``y^6T8;B1gkSCva z#H5rk9I@a9D9M+={HtH-<|ip4WbXu&bDIiRg5vSVb7$JqBac?ee-tfM&S4sUfZVEB zK__Prdz)<#muuXHSN^S%BO{||-}bDkG4+~!qCSPH?1%A8KM(3sF-r(eA5W$^7a*k4 z6)#1tL#LbiSa07<cTDT5rn)T@l=a#KnS*_4r=uQk7n4~o(vQ!EE?_ea4^*m9LGxoq zoU~&aw~X1$33s!pbzv|K_1^#s4Q<h-trtd@yU^iaYZeCfWXbj;bbGU=xcp2a`P^=! zX!#tLoa@4cl1w`8yO|odxkG5p2H1ESg?;NTx}LUA#q5bUu<C6Qc8?wpZLcaRWpyPj zep5ifO*?7lO(V8(UQa(Wq{ud-4Z@UpsluuFn^Zs7kYx_3@g3c1qK0j=c<<YHxcGM@ zf0%evOj5ii{JFfCc6H2>T=P9c6K*PULcwA#eEfjUbxvi~VVU$};v+%HWHYYou?&@F zzl8kSv#`o9r>Z_C1002Rq1zWFnmN9NZguTJ-aT|-mRmIW|2YhU*JRM)(}vKsq!pU_ zY^svU3nbYt$y9o~JIz$f61QKT%)`@?IqJ$Helo2ywJy5|^L!-us+$E4aa@TpLmc6& z{z&Y9cQXvz6pGmoqR9RJI`NEKEvfD}M^;_VOPum#_>D^~T$*Enj&Wz{Z(%aqKlSIT zsA+<Umoa>7sU{oco_wia7B#xvrw6Snq-@#^zCS!g!#5nH#gAn8Qmr8yE_^0A92x}o zZWRk!_yV4eks`zT7{GPkWJo*xNHBJoE-3bSN@xGf=e}np@HFKB{JJ-b@5@}HZRX7+ zH|ru5$Ndub?Hi0{j|Re-b-AKniW)n*+w&>~7Yqu12)!HQM474TJg0jIN8M|uAXBLf z(pbZ^DT*8mn5u4A@xiqj6!>Qr_{AK6%2ru^{W}i|T?^4j)g1GRE$MWN43kR|R)5}$ zy(8N|p0{B7jce$6J&wZgJouiyN|{<Y)Z_3*LF3mCD*loV7k(9RT9@b2EXa7?73)ZY zV#n}I2VcH;el|~-yPr4Q22%QI%<hi|@(1s7lJD5hx8C1@7Q@YW;kpPOau3LFU>k)S zKZn587TCXJD9T??6SK4{sa#1zNc1{RwsWWBjoW+h!>7(1cz7vn*<m4M&7B25<U{dL zes8E8kx*sybcCzYicHjw{48u4rA|i|jo~VX$-JuZ8ZX6>JZke~mRDC|%g-~|cVR4t z2BgyaHhFfv_+0RkW_LQ37l|v@946nMHrQ209nYEsfiy47=Nlbaz2|6jmZGGOKQrMI zqhonPxf9&-x5oI*`KXiND(o8^4FyY#*jMhc@Y@b3AiAEwu>$l?{G@>oXOXwpB^tZ@ zg}9}mmI5}usZw!rmZEp!$v63t7<eL$-Bvhrmh`ru@0XQAru;Ontj-pfg?8sDu{J!h zTAtf8(pW*$ib7_ZVf&2dFz<5#42T=SM><Y$=^1q#JtYK>1bq{ZX1j`kALnqiBvu?g zq%XEk>y8=EW594g4{87YLTH{QoLqig99nr^6t}yG2R~cFl#&jK-Hp5Oyv2rf9v<ce z=l76GuN1PMccx0q+=iz-X0A>MVPEB+pjeUxyE@)e^qfR)_k2b>T1wDa@f<vIX3$(z zR+(HUKx4!sDGDx~n!04coO7m-vm#y`T;79Zx0<kZK?j`74aUZys$!W-qA=sLGMbsD z!geXD|B{v`=sh&U=CQUozQ;#Wj?&`fB^lHZ_Zu=-TqYHNWm;;SMZqIRQ{RwD?7g}t z&$!qIl9SdhMfbbnyPfkzqYHVUFQ+7$-#kl+Z%f3MU%$b2k|M?g?}P`Ea+q8^3FPud zg0Uokc9mFw#@1*MObTFX^(b8YTfk0xV<}+qGV$@rOR(J756fPkr6IBwe6-JeE?TjU zj@{Z$UGHv(N8kIy(}-wFv7XE+hTXYjpf?O^oF)vo+L<rUm%-svtALgrBIn3$v@DYa z-Dg?&ZFLUbyIDu(J?hEF+Y>LWStagkO^1(ePW)?hsIc>QH}UknMh-QuWJkF=Va)zE zI=Ep3FWk9|x5#ec!HMT6;?Gw8Wf#oX`X$iD)M^@7qsZCac9N;_8qs9B3I~3<3`u(h zj(3&n#aRRGt(AD@@?znA+EC%(xfJ2rs^2s&*?}!@+=B)P;Jbr+R}J$iptYfK;tcl? z{B~d@51neuwdV1pFi?&MPEy8eP6q7PdWX(Rk-YWiYvAXQC%8W1E#3SulrOEc<}rN~ zIrdxxS(%UF-q*IV%VSMm(>zA_nrn%>LubPa|H&+vN@GVqFOt_wgo#V*#noGa`OJ4^ zj&cdYw?$iVRmmk@G=I0N<0=dMUEd9tY4_vk^`U(J$WK}?h31+4Dy7VqQM|ZsAa{jU z>gtnERtdj&)rv%Z8GRZK4_``FI{jElI7^quEJpVapM{n$D!ig^8}*2cgTWbI_{wXO zuzv0f`m^X1C7oVUHMf%ug^Mjhk$)<EmfrSJ>$(X}KCE$h%bhT2$X?1h;RC;pZ>2rE zbYbiHRQMq767H3~sG8orXVv>|T`_O-Df;^{U7YY}NY#r?y`U;u1r^*q`T6PY5SqIk z3fG#_v334fwK;@Zekt?J2X_3emlWZIIuK%_jE~!9gX}7OZYmr@6SW%zr`WS#Qg94n z+XT)zP{<3H_LHK+)?&xcr?7eYPwEzz!UdOS(0Q*zI7X(Q=ypksUF@qx`A{?7VyJ?= zMY>kIc7>y|m3Y_hP}Tdt<&<{cmX%K3KsBv{;PCQTRp!fneDLc)emx+LM?`q=A?-=L z@A7vcwRawGRMh7$dcFD0z6$YTg&7~-bA*RqN)~(@;&4vWEu1Phkkudd;s^}`X`#~> z+|Kp|=vv5y!XEMD<s&fXhO=<1#)0a0-4s4Q7s2MLn&_NUB_2_FObzRY!ruBLu&-ws z6@9)=@2*8~q~&rU=GJz6@=h7dYHYd3rw6b&KpVUCG{cQLSLw3JY|Ibd0b~5TK-cLy zVoL1_G*9Xe22YhSaN&FUI>8;NK!!VQR^%t!5^&pDJv{hmrtq$>3r}_ZMRT7`M$Zk3 zxa-AguD!mKj1S*|jZH_Sw*!svSh+Lbblm}4>MiL_&Td>daxqVo?$MII#iHeDdF)u& zAsMbYlZ+1aBA;P7!oy?Rq5OkBWz6;BeD`#=b`FQdyY@&<cfLSZXO58Gpw6Rit<%9# zLywLQj^|_xZ5)4X9ax_3MBd*gNvrxkJY!{ll#lBv{-4IaJg%m%?OXGpc~U8)K@z1w z!`W*!p-G68N=4=nl`*7QlavOMG@+TOP-kCPiOge)Nam?bNmLZ?{yq2eyr27bzxVxo z-apRg{Bf?e_S$=$z4l($^&ME7>Dc!piaEGX1|F(rlLI%$&{1NmX^%k@X{o=#xMV(H z9t7KxlNZdGHS6EQN^221N1%^+>sLbBcbj2uXEg3sxJBmby#qh<wW#PFgVMq-RAAg8 zx_>9f`175H*}jC!frgTso6AUa@K2~8-Ak%v!YMqgqC3y)ux~3Cppf(nP!Wt_XJa*3 zsV+t9w?`U^H(v(vtq1Vo>?qu#a1bx2x?=Iz-FW6xES3vBf=?l1@pFe5t{*v#t2$~i zw)zRXK-3E8F9}*VEuK_5^^%{N49yRAr)?hN(QAb?HkT5}OXe7H#D!Q)w!&_i?ck6W zguV%{z;#0sW+s+HN81cM(V4>K|EHr}%{uUS8I3oWRHKl1Ic_~Dh7R89c=RC;ULCZ= zJiP+;t2C0x(4$PzTQ4-8#b;)@Nq{Pck^a6T6~eg+)<P9wD)OP3w9cK%pPnj%h6}I2 z_e-jjT+Am^1EazC`X~@0Zdmjyg1GI{#O_71v^G_NNT{Y`%EJP(e1#|}@62KBvW7uc z{}4<)VL&^J&FKb3b?WYEfn~<oq$*6FuGy*0>Ynw%aXC&PUS>zl!-X1NRc^;dEoC%X zA`VYac90oyg_vHfj&cGEc~_5*p%A?i$L))P+<I@^?mCSec-=sh?G15$&pST<T@K#& z3q^llJKU(a8zOhdk!8X8m~T1-mxr#zDTz(6eBA|b99~FmmWbPYz12$UCd#lS&zIBP z-YryWPZTNpZbO3SZ9(f|RhkeVOTuFbRpi!{8-n{-eS8EL=PJ`p-!tg?p=U&==qbw@ zRI)=M2@u&6Os$61so-l{^hil%Yd>_50f!`9ZFU^inRO8BBQxMkR2*vFw#J7So)I=h z7t>allj9s3W`fow607aXP9J^=JKe(Jd}=P!AA1SNW<*pSAWH(YXvHB)B_1xsfZREl zvU464(d;B?T+Z8?mxO6~JQP2Z3y^Rh*p?hzT(kq1?bXDg(@W4RdmN)$<3K9!X|TFK z#$Z972|m5-4*0!?xhp%C{rt-m4?CQP4QFlu96H7d8pJ^9Wk3FDIR|tS`pUZ^w1^aL zwxv16O;l={D6Q9CPeapFXpr1YTK4q_{Qh0aMt8T87u!1dhdGsnn^GAxNmd2&gjUfh z&DQk&d2LKqizgrLAAm|M#d*nDQ08L<#|kQl$jReaAKgMuCyC(FFVjfLO*y<-u@suV zCO~qL2}tTNY&H8FUT2rEFa8{9SSo*=br;Nm%asLGwWXEaHg-Bq)SgNcO#;XsmyI+j zb^%c*CGaq%ke_m+9YnePJ}*`TCz@Vo<Q0*e$vX$Tll}2pT_5OLRghQ0N+=VQgJPRP zKtRKTTFD5oK?}yw6nP0|iDnx2AMFIaBaXlX#Idr|KM;@awq$he8Q?4L#&;rx7}o8L z-uDw&;Y;nfOt2hBN%!G_Q7>@KkxcB<ug3F)#cL~%<1D{c&>j7i{4jIHgF*F>FmWy0 zlXe2SmdnEz-HYHFvI<5SoF=;Ba_Fqp2KNHSQmU#)ms;G10@G^f7IDS<Zr53nZ}ITP z+>L}u-6yRU<LP;x1CDRK55t8j^vKWa<hy%2GtqW4?g_1B2R5c+l9xIrZk$Bhh3ko) z>N2t;#~amenv#9%1E}vaS-Mo&mS^DI0j#Sq?Ta*H6h3UB#=7Mgviv-_pH{|k#sOHd zFPkai>K-xKeEd|}j!qt*A<e#m^`HBVe5y62C+sJX3mR59n!U_hX36EgQ~Jo+(ED`! zre{DqTZz;cEo9fqQ^muFsMVSC%yC%_eA`#R&v6i;?a$uQ_X8QEY>x<%C<*jgx13kz z;6XS1P-dPh-X-Fh>FkNPR;XLJ7m__rL)_31S6QA#700x(LwB=T=W`rVW^5&tdVFQI z8Ap_#I09~0>+#E-9w_#`2Wk=s`=_YUQPvu$?YRaAbXD-HrW*Q%-z2@evKX<PO|)%B z7fIPF1t&Kz0<Q!6VAO~t-CLngqqojRr{b?9qN@h3Mc9zfAsiYmcK{-K3YoSe`nYz2 z5UEW%fX5_aF!0U^##8nK>}*{K=FW@o_We1q-y#u=dGW~4&><^jEwDl85=3=h28-*l zFil`SsXlU?R29l$I30tM<PmZF@Ci;`Y=+eMBrLuAh25=ZOufIIg?HaR!_yEI3>veW zZFr2d*!2-;U>2OY<;RLRUS$0D%TR}9X9;ho7ynhxQFJ~x7hie`VA!f2;-1pS44eDF zDydXFzQGBGbZbE){~>ff%Yvxmp4e2RhR(Nam_idVEaS`Kvp!k;ZS{%BpB`lUPg~-i z@%P|QaSS<YLaB-Kd{i7+jwz37aha|;-jxrhGv8kZpHIH*&%Q<QH8u)}T^4=)V=Syu z*QVCC227`V8kuNxjBXlrkElG(q>`!1w3PepVjrKX9<rrs)6&R`^H<26O_h*(C!Ogy z*i0m(Rsd{oAZ;K{j;=E$W8EAee6&CPv}Y$vXDQ-5(;UVzagemlHz8xz#qv`^BKc>B zY_L7)2+^4rfHPG7KyHXB?NYZy#V8TTI(r;1ID4a|i2;g<#q;dbx05*i0`gRB40sf* zW}{WQVBLx*L~THga+O~^VR{KVBlg2u5hv)-6vR-&>rCn8r;K0309#e!g`L$MjM5DW zI_uI$(!u3ho$jcTF5N_GZg-b_)EGz0lFDh2-Vx?m=05OiJqun(xIK5JD?EcF;>ICV ze)^Q~U%!rGlHAR}X+;}3(7hh_e7geYG!MbibD<>9%983WC}EF8EFvp~^GKv<F0|SI zB6oZl65afU967E}Uv8bt|FQ5CIHnrmvbR>)d|?i@9)H0Ws6T)RBQ5ONQ2>#nmO#E` z8e<S-2|w!ZvW6RXLdu4dOl3qoS)wI`$wo&B37$;Vz9|u(Q|s{S0(pA4axHD1T1=;= z+2D1%tC)D@P<^qS8#qt?0Eq*G;QVHU2)xPTEf<{#*{`NSxpf-3u+@~UV>S}Oz7{rX zTn(`^`pgJjZ6p%EPC(}D%S_d2Usx)h4u6coz&>Rgt}&9vsy9*iGNTP*1M7*n;d6fb zhz%|=2}HZ!d>DQn%ukng!&g&<aisGCK;0x>kD4ZPL)e)Jjf<nU5u?bRf+V<Qunq!u z*i(xo?o7V91a^FL!^Er?(Cw*>GD>bx7_}Q8n@++m8cMEM&V&~0d{#7UD%yqD^G<&{ z49j&5aYbY;jEEWF&ety>c=A}7U~fn#{aH*8kKaHRuap6Yl2(}b@G)8Wi>sT9&4h#J z9q_1MFUWp#z#&r+{NSLAd)K7kn62GVcTR=ha#aGZ#u&1Ke!95*tpxZTb!I#x1&Kt? zJ95bH5bV>HgUZV<+1tWW^h04Zzvk3;=(#tSr#Z%z_%TvoydwsDhHt<LQUnKf--H=C z4eZZ-qNL3|54;aY5y2{e2Ahr8ByWZbLI>fYY#%WTxNRwGIu&mhD4?_bS-9`j2s(lF zpx2ZG9S4s=a?d5UH{u<;ZCXBacfA9Ro90e$t;n`c*!BggriV~d&p`Slu!iip8-?k1 z#Zc+L1w%(<Xn8;otqe0qVXqiY-N_gQnrDMZ{V0?hmj|-H5~0d;2wF?F!Rn1UMAWAO zj3sq&FLRkJuzSD^>wjWLlHZbKvns|lG=#S7m8Ma)#<;{ShW2)=QHyh$cu~2OzpCQ_ zWHkzqZVw^!%!^@`4&8*V*^|hxcPGh-)lv98_%L%+V-vq8r;J^wst)<=IZ~?m12zvP z!|l{u=oNX)Yx4O+?zjuUqO4SG`&vtGjcO!1nF~pMi#t8^@&NTu*C#tmrlR+yI_68W zG(2CDPqK`DFizikVTXnYp8hUNG!8|O*uGJ8wQn0d$_zuTKcb9m#%8jeIMa>=o0+2K z1@QHU7_+DB5)(5)6F;`D!55EA@U!a~$Zjuzs9U!f?VGMRRW2VN`N!iP%SAZjpaAaq z*~DfZ7>`YS6?}aD9#lu%B^!*iN$VedYMK@Z6aBQ{#LicwW93Hj!tgrXAg@d&MBil; z7K+kkm+w*E*S&Q0;CKF@@-yaw_hsg|RwrAtM-X!_cJUmGFOu{h+evfV891%@hFPZe z3~~dG6M+$5#*NF@-xb(J_mniAo6%hfp9j3bc0&ZzS~ie`9#!UY+EkLVRtGO^G-PL3 zctYRF>9lHx8;MZ74f8bTu%peK$)q>q@Z^f+#B%dBqBKDaqqH9|ktU_gj^S*w(`1;u z{HzCyrVNvN$I}>XrzZF$YfRp(Q6~##KZcHTn^=jG9_Cl}(FT(PqmfJdW3#6$l?gro zXRT+_1xprFle};IO_OVg&tqBQqLjjTA6mm$&C@3vOG82Iupe6|a1;6;a}RKyFO=W( zB2}CUQ*_1<^lJ&?_5lH${roA}Tvt!tkJ$t|d0d{zwH=O&cvIoqi|Nbq6r!ybjaNVF z;A`GKCSk;g^qlJ=vUl`wAb1bT>^Nq9-~%5vEKP@^s5>yk_r@D8>&W!;%b5#(UJd0B zL@~F(4|5hgW(qcFgVQEYEKd3be?;#xPu#{5|I@w1rsg{=<P^hNcU^%iBlGFJk9O41 z!2tH(wjq~RF6EGchoL~C7)I7Vg|s{o4mWdz_1U=yUrP$$O*;>8BoZ8+AO<fF+riI- zab#biF7E$ihf1r$&{4XHEaVh$HpUs#vJdL?c6|cj8KpA1n<8-Z{(a>3qcWm$QJE_g ztJ0PAMi}jX0fO$h@s2*RC6hO+lJz^!LXC<jaosf?vj_ix(Yx=2tW#u*&H&G&bRnKB zO(Va3viSkKDgm`;lJT3o8LM7>vQ|DH;x3sJpO<;0Ek26>XoUr18MvPE0_4cd(F!>G z$TfcSwwFYxNSg*&)WB;&SKMgl3Zecj4R3d7g4z2CP_`nCIq0&8H&4z6OK!zd&%htN zjHO{5&V@T}kuy2y(M@#3)$pW11~zdBy57`>tmJ}K=wUB|p#}@FcBu>|uP6pp2Yal0 zCV>w|D`Vl>)$p`P1qYVSBzGc$No7SPJGmf+i3q2()oKZuWShh+4}1cmEn|5(js~bJ z`ihNQkpXZ~87<GaVqdxzvoyP&NS9PQV_BJCkY?u82aEeU76ho+OlU)Nz&St6Md zCWUp51)#lU2NCY=V|=oX6X*L&Y5Dv=yz$@7P(pMpn>)jom6Q#Jqu~Kqo@)-NV@E?( zh&d+OCh$T&M?t62MKV!pGbqJ#i1Z#I@JQaz?iv@sOxtKrUin&pbhZTM$r^z}s~DUQ z(T8PTz08~Rtu$ukTnrA0<@dI0(&a~IV7a0#Uw+CKcK@o`)GAws?9qyc8M>b2es?6t zo*Pd^C%gu0cMfO5<+sZ#k|6P0Ab$2f1dZ}P8H>Jo=<;Pbo;&gpzScj3kQo*<U1c^E z=*}jlp{r=VPb|D!eV3_LIzc{fh^068O45tbf~;4L6C=9)4w16jM0qw>;Bi+x+GcS$ zh_GE`!jg+%_|p~_AN>RuR*l8xOWZRl@El}iWHLHWuCl*tZo|wfPmGjrB*FIM!DqDx z+8Q`<`%p79HRuJ=PrCGZg#`}kE=R9bLF6+YfLEPuJY^wwnw~CavvI*3x;H@#=VfNn z$)6LcS6Km_SNoYX{c0oPP)K%I$I<Qny0osw2@0R_L3`3_V7C2+$s6MFN{cK?8H$tp zJHkOIO^a!q5=s*SUGcP6BDuCrh&&lSPD`ZKY5$ruR&2{0s`APdmDWz6V_tj$lr_T2 z$pJix74PA`fdJ`QFOJ{4Jn`^R5vE2mn_YC2!vbAWL9LpV9P)XHbgXY8;~K}3twkBo zub7J$w3>(m$I36(=!a=v=i@%}TVOK3oNP?W!o6lE@cAn#T$^%~Tn=4Bo`laLn|>Hz zqqiZ}m&V{$`&wpk&3Tf2@G^-Kt0$7a!l-;YiPxo~1O|JO7@npPDys_NXfG|^S|K6S zdAk>XORmM?(7CW~u_HNA=mcS{;#4iDgzVj(0r&3$ncB6OEWUk@bbp%8q<s4VsW06b z^Dh!qXY)rwE?MJuyBz2lR;1^GW|Hn#HSj=P7kg#pA*bOvZ(_AO`Wsb2#Ig(Qgvdr% zb8jDr=*wcL+yl^e`Apiht4OnF7_(VP9M8q-gTy%rdR6)nQ>HtGiOU=!p-w|Y=wuQM zFPu$YFYh6px3h^#l{4|QE(fbUJ>c0tpKTZz0$fxJ4zDKD0et~<oS;c>rr!m-=jr&m z*BPykDpCIfU%+<JM6RwAg1p%Zc&ojX*iXH|Q|KOH4sdLsR}GGMc)2oc*{_9$b}sm! z{1`e2g`uMKPiF90KDro`qw=z|823DjoVq=q7Edz7&D{H-&m})x-kV9=$KRymehxyC zk}2swT|(7O>#3?n6qa>9Ch7*y*~$=Se5o-PC7q&irok2R_|$r8crb$qH;j-ah2My& zgBIO3dVr}rRtt|GPop;@Cen;}0c4+ep#BOs`0d<AWTwmE!+FZ|6<Y{ZdI(GJkFYb| zbNJQ6-n3J8E7{glM3m;thW7lAY`^?C$cnqp<_Ug;r}o+e9|^+AfiOIEOd9%Cu7KY2 z1mdKV%<i3fn<Vx<0G=+lM#jqH5Pu&I78lUj-cum;qz?u)=!5(#VU$nZ3j361k<l~H zl5@8nllr(|`p#$)UQ}6+`Re0vS>6#?=8(g$76>GXOb$F<a*IuOoJu9sDv0sE8PxiR zFNqTxWLC*2QVrJ|#MyH%9a|_t%pE0}6`rY7SjnII%N~YVeG+uewIVXO-ix-*Sw_j= zRrqZk1^xVqY?PE0p3ibZR);~aP0zvpQ!XR@L!N$dFrceb6~QI*9yq&(0R?kt6$D<= z)=|K!$so@5#4odxQAyJpZ;7_R6jKA@w=LDmO`;y^FQ0|x6IJB3;92$%y$mDPy*$Sj zBVu+y4(}&f;KOb^bmJ7GGQtLlavaANeRl+&I_}3Ai(4T1$`1O{m`7*b&SAe#j6}y< z!K_DPFL}|Kjgf*GER$>p;cEIgb7ulL#%GZ|@e~Hlp1}c)Xx^8SljK!|A(7OwrACc& zaGAU+TDo6jGp~B#1MV6x2?`)ub+2II2QxaDa2T9l%p$c<eX(cCb+~blPyr!=*UZb{ zUd3bf#+@hd`qynZNWF>P6)n{H6b-}u*NF4Ce&*4{E7mSs;;5qDDsp3NFgAo7<>~%0 z!TcZ2)FddG99|n|J=$Xr`Q|bL#)XQgQ>}^*gBVohl;zr2+Mxe_6BJvvm-qgh45pa+ z!ze#K(|kDq_9vP#J9vLUzfl*GReb1F=WAqXZw0v}8_Df)9>Sv$Yip%9nbdo2Bf0a3 z0r}aLjDJ%x#}aR(5>q^h$qGZ%i7FvwU*3~j$24e2=O&twVMEuaNl^##2~0i-pvM<K z9OE(x!y|)G@s0y-C0F5qvH@hgT>v{(KeMYvuS0l3J;*e=;FavXP$T&f4qYfAFFMqT zUi%JA|2`gT+vGv^%U*O+YiFkziIbQ+$|zqwo&<Ll;jDT&y5p8VP2JtdKHK4f>S7`o zwAGld?w(Gcl|E+2)NO{+5Ouo2$DCeuT@8x*j<`PgEoi>GOD1^55DYoTsCqg=e|{o7 zv(Z6I!5m7%n#jYw9vt3b0ZNZfg3{O~Y89VOcFjKzz31ydg;V_LoSjd%kJSb>jv<|1 zRttKLrKH)#oQ~KZU=NSPVY0s-SWTCqK5rvIK6xW5m~gci&3KqoRR%el3z>0ielaDJ zEYWGy9;Ri68`*KG3tG-uV+G>^(#QG<|7AE@b84U^_hQgWR1ovzpOc4NO{7zphfZ&P zg1*8}=zQ&f6$MAIG?))t7p({FFcC)9upiRzl;Yr4JMuj+iZn*JbL*1?zOyZdR%U?M z-@i+mPi{v2msjDFtpL!ZT<VlmOeV!_!IHT%@QhwM+rK@Erd#f|)?Tn5`=Tr0M0Pf< zNDXCDDjq?*0k{5G#jzDCXYlalb{q(kg7M>ju-Afg(PDHOo)Q{^KQhBfL&8ViWAH{3 zxyA5)niwhUoljJ~&w_Wu5xBQk0p&R5!f8GeiB+8t8t**HKJcuCI8zZat@sUO#4Ka> z&micn5rjX+6{7LFB3wNA8f10NWL_++hO>|M;1rAd?CVA?bXAzd^dF97vTRI<bon}V zUekRhHA|LiNSk2MVp&#U<!ji9?@0%@H;mzNjP3pL;Gy;g<euJv7l#$F^7Ie*c6cqy zS#QI_yc$?uA%SLRn|bR}^H9UXhplTDA^k-c*l^t|WZ<bgHW4FYK2?Fn$*?5Q!3N9c zs?gfK26P#SQjK~oW=pv))(1Aw?wI-1TInK*U{>&DrmLVp!w;SwSEr3ywVAZ8Od;#O z=Ha&d2<&?t%(M@Q(YIfm>Ft0nbWl$vAA{rIOo$v_8Ek@S3iqJIxtTAx?@dGV$3<w< z+zdMpEW$N?3&>FoIqq#KfKj#U2K#?jUpJ^1<)|S7|4j}l5LM}xXS)uDqPD$Y<A7x> zOt6rrP03>#a~FR@CoLC_LO9c=o0I2Qcl8%~-dRq9H$>a03dq<z(0YmGe%dI>m$7lp z7-O^eeKz_GX*3R2Ym(fM)zs=%0N!2Dk3K$GSm~<`g&hHOlJ_4P{h@<OJ)TYcE$@I5 zC*fD8o5pYM7qzKLzD;ensN=htC8&REBF?>ij*hvG)PA9SW5KQ#oY^%U^P|^NCI52T z{=FG%hJ)}_>0Yo;3uga3OQ$Jf&Fovz2V_HM6di9}Pd~ggwwV@QL5CyTDI?xOH<gys z?vCwL!k`xyrR9R}Z$(_$Fda9Y-i5^iI~r?DEE?xsm1uN+quIDA_YLYbyJCJ-IZkdk zgR7I2G07$xbS)3!3KNck|G5Ogq6mgAE$1esA4gieFtd0a7*tH6xleTIoV2Ux#{uoc z!iLGQimBvG+*%rX^a44t6RAweQo13^f+g`*w5Ym-#<$ea>FS@E0P0CYLr+qvYG0)N zLoi=ZgFi)oH9Csx;!Xdv#QcprWz{$!w)|=?87AA9aDFSO>$K9rB^%hqlfPl_O1Z|K zijVLxN}isr-bZbAO(SQl2rJz+j#(hGi%!>{U=!Fg&1MH1LO*Z`Ea)!B?N=SJYl1Hs z9+abL_B<*#PpZ-FJg@Q0A(cj<`b#J=U(zPk;xx@Oj6%^3r%~^x7f#-|jGj8FZ}a<7 z15FJVrnvMu#8q!4`(<<K_R3INZPrV_oLE5}k6$G#tYvLfGJUbkqX%EVUyL6!8tAO0 zL)4Rh7uPm*qQI`*m=e-P7w6_sfsh`&dpH-pm!1PNjz-qy(?iCD8KbUjErwK$#lhG! zG{KHirK#y8W2FRT3<&6bs%4!F#c;5NPdp7a(CWS_dO>h94QXzopX(gyPrpJs(OBNb ztMV&^pK_w<tQZ|>bRhR0iPNDi@0rxyGC1r#0Iz4c;M1&8jUjvma1FFa^|y~P;d(l% zeXvLSe{!KPnDJk?jsVB1noz+lYIWnE;<l4~QYM|mW*NqT>swK*voB$ei4Vd*wVPq! z#qqHO#Q$n%s}NcWhkiVTQ|sl>>Y6)xZnVG`xl7PQSOP`XJE7j<L|kDOi)RONq4Bsk zv+P(pzk6{TiYZ%Q@uG2myO}Kwriu&7iCGH?L<-vfe@O8g==~!+Qc%+Vf78)7pxkKx zQkK{6Z?J&r?tge8+mp%0y6?>15CM$hmeBQ~fpBWJ3rL=>BrPig;BN#pGoK0n<Kdy8 zBB>-UpdcqHC@%2d9}VtXBXHtB>=FK1@m@hjQi_We|L-`-ApQR&ye}&$%>_zG3W;-P zA}B5yWc;6OIR54MfA#%e#}AtP{~B)meW-s6{fG0$|8+R{kG&fHJM)Ww6D`AyLndhY zf3%zUmnMH*zJD|^_&2T+e;@bXV*k}|;y?B~_21e%{F`uLZYaV*M*q8Eg2Mky&|kCu W*CYjL{~HH{f1Pi8VaZW)QvU}WMo+{5 literal 0 HcmV?d00001 diff --git a/platform/linux-generic/example/ml/mnist.c b/platform/linux-generic/example/ml/mnist.c new file mode 100644 index 000000000..4c1066302 --- /dev/null +++ b/platform/linux-generic/example/ml/mnist.c @@ -0,0 +1,300 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp_api.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <math.h> +#include <inttypes.h> + +#include "model_read.h" + +/** + * About MNIST model used in this example. + * + * The model predicts handwritten digits. It has one input and one output whose + * detailed information is as follows: + * + * Input: + * Name: Input3, type: float32, shape: [1, 1, 28, 28] + * + * Output: + * Name: Plus214_Output_0, type: float32, shape: [1, 10] + * + * Refer https://github.com/onnx/models/tree/main/validated/vision/classification/mnist + * for more information about the model. + * + * The model outputs the likelihood of each number before softmax, so we need to + * map the output to probabilities across the 10 classes with softmax function. + * + * In this example, the input image is stored in example_digit.csv file, which + * contains, comma separated, the digit label (a number from 0 to 9) and the 784 + * pixel values (a number from 0 to 255). Pixel order is first left to right and + * then top down. The MNIST dataset is available in this format at + * https://www.kaggle.com/oddrationale/mnist-in-csv. + */ + +#define MAX_MODEL_SIZE 30000 +#define INPUT_NUM_ELEMS 784 /* Total shape for input: 1 * 1 * 28 * 28 */ +#define OUTPUT_NUM_ELEMS 10 /* Total shape for output: 1 * 10 */ + +static int read_digit_csv(const char *file_name, uint8_t *expected_digit, float *pixels) +{ + char *tmp; + char *token; + char *end; + FILE *digit_file; + size_t size, num_elem; + const char *delim = ","; /* Delimiter */ + size_t num_pixel = 0; + + /* Get the model file size in bytes */ + digit_file = fopen(file_name, "rb"); + fseek(digit_file, 0, SEEK_END); + size = ftell(digit_file); + rewind(digit_file); + + tmp = malloc(size); + memset(tmp, 0, size); + num_elem = fread(tmp, size, 1, digit_file); + + fclose(digit_file); + if (num_elem != 1) { + printf("Read digit file failed\n"); + free(tmp); + return -1; + } + + /* Get the first token which is the expected digit */ + token = strtok(tmp, delim); + *expected_digit = (uint8_t)strtol(token, &end, 10); + if ((*expected_digit > 9) || (end == token)/*No numeric character*/) { + printf("Invalid digit %u or no numeric character available\n", + *expected_digit); + free(tmp); + return -1; + } + + /* The rest 784 numbers are pixel values */ + token = strtok(NULL, delim); + while (token != NULL) { + pixels[num_pixel] = strtof(token, NULL); + num_pixel++; + token = strtok(NULL, delim); + } + + if (num_pixel != INPUT_NUM_ELEMS) { + printf("Wrong number of pixels: %zu (expected:784)\n", num_pixel); + free(tmp); + return -1; + } + + free(tmp); + return 0; +} + +static int prepare_run_params(const char *file_name, uint8_t *expected_digit, + odp_ml_data_seg_t *input, odp_ml_data_seg_t *output) +{ + input->size = INPUT_NUM_ELEMS * sizeof(float); + input->addr = malloc(input->size); + memset(input->addr, 0, input->size); + + if (read_digit_csv(file_name, expected_digit, input->addr)) { + free(input->addr); + return -1; + } + + output->size = OUTPUT_NUM_ELEMS * sizeof(float); + output->addr = malloc(output->size); + memset(output->addr, 0, output->size); + + return 0; +} + +static float array_max(float *arr, uint8_t arr_len) +{ + float max = arr[0]; + + for (size_t i = 1; i < arr_len; i++) { + if (arr[i] > max) + max = arr[i]; + } + + return max; +} + +static void softmax(float *input, uint8_t input_len) +{ + float rowmax = array_max(input, input_len); + + float input_exp[input_len]; + float sum = 0.0f; + + for (size_t i = 0; i != input_len; ++i) { + input_exp[i] = exp(input[i] - rowmax); + sum += input_exp[i]; + } + + for (size_t i = 0; i != input_len; ++i) + input[i] = input_exp[i] / sum; +} + +static uint8_t index_of_max(float *arr, uint8_t arr_len) +{ + uint8_t i = 0; + uint8_t max_index = 0; + float max = arr[0]; + + for (i = 1; i < arr_len; i++) { + if (arr[i] > max) { + max = arr[i]; + max_index = i; + } + } + + return max_index; +} + +int main(int argc, char *argv[]) +{ + const char *model_file; + const char *input_file; + float *probabilities; + uint8_t expected_digit; + uint8_t predicted_digit; + odp_instance_t inst; + odp_ml_data_t data; + odp_ml_model_t ml_model; + odp_ml_data_seg_t input; + odp_ml_data_seg_t output; + odp_ml_capability_t capa; + odp_ml_config_t ml_config; + odp_ml_model_param_t model_param; + int ret = 0; + + if (argc != 3) { + printf("Please provide an input image file for classification.\n" + "\nUsage:\n" + " %s model_file input_image\n" + "\nThis example classifies digit written on the input image.\n\n", + argv[0]); + return -1; + } + + model_file = argv[1]; + input_file = argv[2]; + + if (odp_init_global(&inst, NULL, NULL)) { + printf("Global init failed.\n"); + return -1; + } + + if (odp_init_local(inst, ODP_THREAD_CONTROL)) { + printf("Local init failed.\n"); + return -1; + } + + if (odp_ml_capability(&capa)) { + printf("odp_ml_capability() failed\n"); + ret = -1; + goto odp_term; + } + + if (MAX_MODEL_SIZE > capa.max_model_size) { + printf("Configured max model size %d exceeds max mode size %" PRIu64 " in capa\n", + MAX_MODEL_SIZE, capa.max_model_size); + ret = -1; + goto odp_term; + } + + odp_ml_config_init(&ml_config); + ml_config.max_model_size = MAX_MODEL_SIZE; + ml_config.load_mode_mask = ODP_ML_COMPL_MODE_SYNC; + ml_config.run_mode_mask = ODP_ML_COMPL_MODE_SYNC; + + if (odp_ml_config(&ml_config)) { + printf("odp_ml_config() failed\n"); + ret = -1; + goto odp_term; + } + + odp_ml_model_param_init(&model_param); + if (read_model_from_file(model_file, &model_param)) { + printf("Read model file failed\n"); + ret = -1; + goto odp_term; + } + + ml_model = odp_ml_model_create("mnist", &model_param); + free(model_param.model); + if (ml_model == ODP_ML_MODEL_INVALID) { + printf("odp_ml_model_create() failed\n"); + ret = -1; + goto odp_term; + } + + odp_ml_model_print(ml_model); + + if (odp_ml_model_load(ml_model, NULL)) { + printf("odp_ml_model_load() failed\n"); + ret = -1; + goto destroy_model; + } + + data.num_input_seg = 1; + data.num_output_seg = 1; + data.input_seg = &input; + data.output_seg = &output; + if (prepare_run_params(input_file, &expected_digit, &input, &output)) { + printf("prepare_run_params() failed\n"); + ret = -1; + goto unload; + } + + if (odp_ml_run(ml_model, &data, NULL) != 1) { + printf("odp_ml_model_run() failed\n"); + ret = -1; + goto free_model_io; + } + + probabilities = output.addr; + + /* Post-process the model output */ + softmax(probabilities, OUTPUT_NUM_ELEMS); + predicted_digit = index_of_max(probabilities, OUTPUT_NUM_ELEMS); + printf("predicted_digit: %u, expected_digit: %u\n", predicted_digit, expected_digit); + +free_model_io: + free(input.addr); + free(output.addr); + +unload: + if (odp_ml_model_unload(ml_model, NULL)) { + printf("odp_ml_model_unload() failed\n"); + ret = -1; + goto odp_term; + } + +destroy_model: + /* Destroy the model */ + if (odp_ml_model_destroy(ml_model)) { + printf("odp_ml_model_destroy() failed\n"); + ret = -1; + } + +odp_term: + if (odp_term_local()) { + printf("Local term failed.\n"); + return -1; + } + + if (odp_term_global(inst)) { + printf("Global term failed.\n"); + return -1; + } + + return ret; +} diff --git a/platform/linux-generic/example/ml/model_explorer.c b/platform/linux-generic/example/ml/model_explorer.c new file mode 100644 index 000000000..bd449b032 --- /dev/null +++ b/platform/linux-generic/example/ml/model_explorer.c @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp_api.h> +#include <stdio.h> +#include <stdlib.h> + +#include "model_read.h" + +/** + * Read basic model information, e.g. inputs/outputs. + */ + +int main(int argc, char *argv[]) +{ + odp_instance_t inst; + odp_ml_model_t ml_model; + odp_ml_capability_t capa; + odp_ml_config_t ml_config; + odp_ml_model_param_t model_param; + int ret = 0; + + if (argc != 2) { + printf("Please specify model path\n" + "\nUsage:\n" + " %s model_path\n" + "\nThis example prints model information\n\n", + argv[0]); + return -1; + } + + if (odp_init_global(&inst, NULL, NULL)) { + printf("Global init failed.\n"); + return -1; + } + + if (odp_init_local(inst, ODP_THREAD_CONTROL)) { + printf("Local init failed.\n"); + return -1; + } + + if (odp_ml_capability(&capa)) { + printf("odp_ml_capability() failed\n"); + ret = -1; + goto odp_term; + } + + odp_ml_config_init(&ml_config); + ml_config.max_model_size = capa.max_model_size; + ml_config.load_mode_mask = ODP_ML_COMPL_MODE_SYNC; + ml_config.run_mode_mask = ODP_ML_COMPL_MODE_SYNC; + + if (odp_ml_config(&ml_config)) { + printf("odp_ml_config() failed\n"); + ret = -1; + goto odp_term; + } + + odp_ml_model_param_init(&model_param); + if (read_model_from_file(argv[1], &model_param)) { + ret = -1; + goto odp_term; + } + + ml_model = odp_ml_model_create("model-explorer", &model_param); + free(model_param.model); + if (ml_model == ODP_ML_MODEL_INVALID) { + printf("odp_ml_model_create failed.\n"); + ret = -1; + goto odp_term; + } + + odp_ml_model_print(ml_model); + +odp_term: + if (odp_term_local()) { + printf("Local term failed.\n"); + return -1; + } + + if (odp_term_global(inst)) { + printf("Global term failed.\n"); + return -1; + } + + return ret; +} diff --git a/platform/linux-generic/example/ml/model_read.c b/platform/linux-generic/example/ml/model_read.c new file mode 100644 index 000000000..7aa20bf35 --- /dev/null +++ b/platform/linux-generic/example/ml/model_read.c @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <odp_api.h> + +#include "model_read.h" + +int read_model_from_file(const char *file_name, odp_ml_model_param_t *model_param) +{ + FILE *model_file; + /* Number of elements successfully read */ + size_t num_elem; + + /* Get the model file size in bytes */ + model_file = fopen(file_name, "rb"); + if (model_file == NULL) { + perror("Failed to open model file"); + return -1; + } + + fseek(model_file, 0, SEEK_END); + model_param->size = ftell(model_file); + rewind(model_file); + + /* Allocate memory for model buffer */ + model_param->model = malloc(model_param->size); + memset(model_param->model, 0, model_param->size); + if (!model_param->model) { + printf("Allocating memory for model buffer failed\n"); + return -1; + } + + /* Read the model file */ + num_elem = fread(model_param->model, model_param->size, 1, model_file); + fclose(model_file); + if (num_elem != 1) { + printf("Read model file failed\n"); + free(model_param->model); + return -1; + } + + return 0; +} diff --git a/platform/linux-generic/example/ml/model_read.h b/platform/linux-generic/example/ml/model_read.h new file mode 100644 index 000000000..df2062d5f --- /dev/null +++ b/platform/linux-generic/example/ml/model_read.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_MODEL_READ_H_ +#define ODP_MODEL_READ_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp_api.h> + +/** + * Read model binaries from model file + * + * @param file_name The name of model file + * @param model_param Model parameter where model content and size are read to + * + * @retval 0 on success + * @retval < 0 on failure + */ +int read_model_from_file(const char *file_name, odp_ml_model_param_t *model_param); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/example/ml/odp_ml_run_mnist.sh b/platform/linux-generic/example/ml/odp_ml_run_mnist.sh new file mode 100755 index 000000000..f83d6f60d --- /dev/null +++ b/platform/linux-generic/example/ml/odp_ml_run_mnist.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Nokia +# +set -e + +# wget https://github.com/onnx/models/raw/main/validated/vision/classification/mnist/model/mnist-12.onnx +./mnist${EXEEXT} mnist-12.onnx example_digit.csv diff --git a/platform/linux-generic/example/ml/odp_ml_run_model_explorer.sh b/platform/linux-generic/example/ml/odp_ml_run_model_explorer.sh new file mode 100755 index 000000000..7f9fed5a6 --- /dev/null +++ b/platform/linux-generic/example/ml/odp_ml_run_model_explorer.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Nokia +# +set -e + +./model_explorer${EXEEXT} simple_linear.onnx diff --git a/platform/linux-generic/example/ml/odp_ml_run_simple_linear.sh b/platform/linux-generic/example/ml/odp_ml_run_simple_linear.sh new file mode 100755 index 000000000..b394b61a8 --- /dev/null +++ b/platform/linux-generic/example/ml/odp_ml_run_simple_linear.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Nokia +# +set -e + +./simple_linear${EXEEXT} [2,4,5] diff --git a/platform/linux-generic/example/ml/simple_linear.c b/platform/linux-generic/example/ml/simple_linear.c new file mode 100644 index 000000000..142dbe718 --- /dev/null +++ b/platform/linux-generic/example/ml/simple_linear.c @@ -0,0 +1,274 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp_api.h> +#include <odp/helper/odph_api.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <inttypes.h> + +#include "model_read.h" + +/** + * About model simple_linear.onnx used in this example. + * + * Model info: + * Inputs: name: x, type: int32, shape: [1] + * Outputs: name: y, type: int32, shape: [1] + * + * The model is of form y = 3 * x + 4 where x is given as the second argument. + * Thus when x = 5, the output y should be 19. + */ + +#define NUM_INPUTS 1 +#define NUM_OUTPUTS 1 +#define MAX_NUM_WORKERS 10 +#define MAX_MODEL_SIZE 500 + +typedef struct infer_param_t { + int32_t x; + odp_ml_model_t ml_model; +} infer_param_t; + +typedef struct { + odp_shm_t shm; + /* Thread specific arguments */ + infer_param_t infer_param[MAX_NUM_WORKERS]; +} thread_args_t; + +/* Global pointer to thread_args */ +static thread_args_t *thread_args; + +static int run_inference(void *infer_param) +{ + int32_t y; + odp_ml_data_t data; + odp_ml_data_seg_t input; + odp_ml_data_seg_t output; + infer_param_t *param = (infer_param_t *)infer_param; + + data.num_input_seg = NUM_INPUTS; + data.input_seg = &input; + input.addr = ¶m->x; + input.size = sizeof(int32_t); + + data.num_output_seg = NUM_OUTPUTS; + data.output_seg = &output; + output.addr = &y; + output.size = sizeof(int32_t); + + if (odp_ml_run(param->ml_model, &data, NULL) != 1) { + ODPH_ERR("odp_ml_model_run() failed\n"); + return -1; + } + + printf("y = 3 * %d + 4: %d\n", param->x, y); + + return 0; +} + +static int parse_argv1(char *argv1, uint32_t *num, int32_t *x) +{ + char *token; + int i; + + if (!strstr(argv1, "[")) { + *num = 1; + *x = strtol(argv1, NULL, 10); + return 0; + } + + token = strtok(argv1, "[,]"); + if (token == NULL) { + ODPH_ERR("Invalid argv[1]\n"); + return -1; + } + x[0] = strtol(token, NULL, 10); + + for (i = 0; i < MAX_NUM_WORKERS; i++) { + token = strtok(NULL, "[,]"); + if (token == NULL) + break; + + x[i + 1] = strtol(token, NULL, 10); + } + + if (i == MAX_NUM_WORKERS) { + ODPH_ERR("Too much xs, maximum number is: %d\n", MAX_NUM_WORKERS); + return -1; + } + + *num = i + 1; + return 0; +} + +int main(int argc, char *argv[]) +{ + odp_shm_t shm; + int num_workers; + odp_instance_t inst; + odp_cpumask_t cpumask; + odp_ml_model_t ml_model; + odp_ml_capability_t capa; + odp_ml_config_t ml_config; + int32_t x[MAX_NUM_WORKERS]; + odp_ml_model_param_t model_param; + odph_thread_t thread_tbl[MAX_NUM_WORKERS]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param[MAX_NUM_WORKERS]; + char cpumaskstr[ODP_CPUMASK_STR_SIZE]; + int ret = 0; + uint32_t num = 0; + + if (argc != 2) { + ODPH_ERR("Please specify x\n" + "\nUsage:\n" + " %s x\n" + "\nThis example runs inference on model y = 3x + 4\n\n", + argv[0]); + return -1; + } + + if (parse_argv1(argv[1], &num, x)) + return -1; + + if (odp_init_global(&inst, NULL, NULL)) { + ODPH_ERR("Global init failed.\n"); + return -1; + } + + if (odp_init_local(inst, ODP_THREAD_CONTROL)) { + ODPH_ERR("Local init failed.\n"); + return -1; + } + + if (odp_ml_capability(&capa)) { + ODPH_ERR("odp_ml_capability() failed\n"); + ret = -1; + goto odp_term; + } + + if (MAX_MODEL_SIZE > capa.max_model_size) { + ODPH_ERR("Configured max model size %d exceeds max mode size %" PRIu64 " in capa\n", + MAX_MODEL_SIZE, capa.max_model_size); + ret = -1; + goto odp_term; + } + + /* Set ML configuration parameter */ + odp_ml_config_init(&ml_config); + ml_config.max_model_size = MAX_MODEL_SIZE; + ml_config.load_mode_mask = ODP_ML_COMPL_MODE_SYNC; + ml_config.run_mode_mask = ODP_ML_COMPL_MODE_SYNC; + + if (odp_ml_config(&ml_config)) { + ODPH_ERR("odp_ml_config() failed\n"); + ret = -1; + goto odp_term; + } + + odp_ml_model_param_init(&model_param); + if (read_model_from_file("simple_linear.onnx", &model_param)) { + ret = -1; + goto odp_term; + } + + ml_model = odp_ml_model_create("simple linear", &model_param); + free(model_param.model); + if (ml_model == ODP_ML_MODEL_INVALID) { + ODPH_ERR("odp_ml_model_create() failed\n"); + ret = -1; + goto odp_term; + } + + odp_ml_model_print(ml_model); + odp_ml_print(); + + if (odp_ml_model_load(ml_model, NULL)) { + ODPH_ERR("odp_ml_model_load() failed\n"); + ret = -1; + goto destroy_model; + } + + /* Reserve memory for args from shared mem */ + shm = odp_shm_reserve("_thread_args", sizeof(thread_args_t), + ODP_CACHE_LINE_SIZE, 0); + if (shm == ODP_SHM_INVALID) { + ODPH_ERR("Error: shared mem reserve failed.\n"); + ret = -1; + goto unload; + } + + thread_args = odp_shm_addr(shm); + if (thread_args == NULL) { + ODPH_ERR("Error: shared mem alloc failed.\n"); + ret = -1; + goto free_shm; + } + thread_args->shm = shm; + memset(thread_args, 0, sizeof(thread_args_t)); + + /* Prepare inference parameter */ + for (uint32_t i = 0; i < num; i++) { + thread_args->infer_param[i].x = x[i]; + thread_args->infer_param[i].ml_model = ml_model; + } + + num_workers = odp_cpumask_default_worker(&cpumask, num); + (void)odp_cpumask_to_str(&cpumask, cpumaskstr, sizeof(cpumaskstr)); + + printf("num worker threads: %i\n", num_workers); + printf("first CPU: %i\n", odp_cpumask_first(&cpumask)); + printf("cpu mask: %s\n", cpumaskstr); + + /* Create and init worker threads */ + memset(thread_tbl, 0, sizeof(thread_tbl)); + odph_thread_common_param_init(&thr_common); + thr_common.instance = inst; + thr_common.cpumask = &cpumask; + + for (int i = 0; i < num_workers; ++i) { + odph_thread_param_init(&thr_param[i]); + thr_param[i].start = run_inference; + thr_param[i].arg = &thread_args->infer_param[i]; + thr_param[i].thr_type = ODP_THREAD_WORKER; + } + + odph_thread_create(thread_tbl, &thr_common, thr_param, num_workers); + + odph_thread_join(thread_tbl, num_workers); + +free_shm: + if (odp_shm_free(shm)) { + ODPH_ERR("Error: shm free global data\n"); + return -1; + } + +unload: + /* Unload a model */ + if (odp_ml_model_unload(ml_model, NULL)) { + ODPH_ERR("odp_ml_model_load() failed\n"); + ret = -1; + } + +destroy_model: + if (odp_ml_model_destroy(ml_model)) { + ODPH_ERR("odp_ml_model_destroy() failed\n"); + ret = -1; + } + +odp_term: + if (odp_term_local()) { + ODPH_ERR("Local term failed.\n"); + return -1; + } + + if (odp_term_global(inst)) { + ODPH_ERR("Global term failed.\n"); + return -1; + } + + return ret; +} diff --git a/platform/linux-generic/example/ml/simple_linear.onnx b/platform/linux-generic/example/ml/simple_linear.onnx new file mode 100644 index 0000000000000000000000000000000000000000..45c4b95b9491f3817b1c63bca1a5e529d61f7df8 GIT binary patch literal 214 zcmd<!5aRU9&(2I#@OKH&U^L>YRIpVru29fcFtOUi$fYa9ST4m@o|>7SQKH1*oS#=* zl9*SbCC0^CmY7qTs>Q{@D8Q!0$n3;ezJQU1ON5KDf{U?Sh^f3nirKd`M+ro5X$vtX zNwFklCKlr{g2jn331S3LGZ$l$5M!kjvtvq%5{M9D%gM}3O)QGy2U;M+#l^wFCd9(U T#KD*Z6sSZNaAM(N5a0y>4~#0v literal 0 HcmV?d00001 diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 161bf48c5..fa9f77914 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -61,6 +61,8 @@ AM_CONDITIONAL([PLATFORM_IS_LINUX_GENERIC], AC_CONFIG_FILES([platform/linux-generic/Makefile platform/linux-generic/libodp-linux.pc platform/linux-generic/dumpconfig/Makefile + platform/linux-generic/example/Makefile + platform/linux-generic/example/ml/Makefile platform/linux-generic/test/Makefile platform/linux-generic/test/example/Makefile platform/linux-generic/test/example/classifier/Makefile From 003231b8e6b754cd390490ee1b676704412509ed Mon Sep 17 00:00:00 2001 From: Lifang Zhang <lifang.zhang@nokia.com> Date: Sat, 2 Dec 2023 07:06:17 +0200 Subject: [PATCH 119/147] validation: ml: add tests for the new ML API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added validation for the new ML API. Signed-off-by: Lifang Zhang <lifang.zhang@nokia.com> Signed-off-by: Jere Leppänen <jere.leppanen@nokia.com> Reviewed-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- test/m4/configure.m4 | 1 + test/validation/api/Makefile.am | 2 + test/validation/api/ml/.gitignore | 1 + test/validation/api/ml/Makefile.am | 4 + test/validation/api/ml/ml.c | 572 +++++++++++++++++++++++++++++ test/validation/api/pool/pool.c | 3 +- 6 files changed, 582 insertions(+), 1 deletion(-) create mode 100644 test/validation/api/ml/.gitignore create mode 100644 test/validation/api/ml/Makefile.am create mode 100644 test/validation/api/ml/ml.c diff --git a/test/m4/configure.m4 b/test/m4/configure.m4 index 7dbaa7fcd..ea05e954f 100644 --- a/test/m4/configure.m4 +++ b/test/m4/configure.m4 @@ -36,6 +36,7 @@ AC_CONFIG_FILES([test/common/Makefile test/validation/api/ipsec/Makefile test/validation/api/lock/Makefile test/validation/api/Makefile + test/validation/api/ml/Makefile test/validation/api/packet/Makefile test/validation/api/pktio/Makefile test/validation/api/pool/Makefile diff --git a/test/validation/api/Makefile.am b/test/validation/api/Makefile.am index d9e7aa86e..5a3c0216b 100644 --- a/test/validation/api/Makefile.am +++ b/test/validation/api/Makefile.am @@ -16,6 +16,7 @@ ODP_MODULES = align \ init \ ipsec \ lock \ + ml \ queue \ packet \ pktio \ @@ -66,6 +67,7 @@ TESTS = \ ipsec/ipsec_inline_in.sh \ ipsec/ipsec_inline_out.sh \ lock/lock_main$(EXEEXT) \ + ml/ml_main$(EXEEXT) \ packet/packet_main$(EXEEXT) \ pktio/pktio_main$(EXEEXT) \ pool/pool_main$(EXEEXT) \ diff --git a/test/validation/api/ml/.gitignore b/test/validation/api/ml/.gitignore new file mode 100644 index 000000000..be2347720 --- /dev/null +++ b/test/validation/api/ml/.gitignore @@ -0,0 +1 @@ +ml_main diff --git a/test/validation/api/ml/Makefile.am b/test/validation/api/ml/Makefile.am new file mode 100644 index 000000000..b7946a963 --- /dev/null +++ b/test/validation/api/ml/Makefile.am @@ -0,0 +1,4 @@ +include ../Makefile.inc + +test_PROGRAMS = ml_main +ml_main_SOURCES = ml.c diff --git a/test/validation/api/ml/ml.c b/test/validation/api/ml/ml.c new file mode 100644 index 000000000..5f8be1b64 --- /dev/null +++ b/test/validation/api/ml/ml.c @@ -0,0 +1,572 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp_api.h> +#include <odp/helper/odph_api.h> +#include "odp_cunit_common.h" + +#define UAREA 0xaa +#define NUM_COMPL 10u +#define COMPL_POOL_NAME "ML compl pool" + +typedef struct global_t { + int disabled; + uint32_t num_compl; + odp_ml_capability_t ml_capa; +} global_t; + +typedef struct { + uint32_t count; + uint8_t mark[NUM_COMPL]; +} uarea_init_t; + +static global_t global; + +static int ml_suite_init(void) +{ + memset(&global, 0, sizeof(global_t)); + + if (odp_ml_capability(&global.ml_capa)) { + ODPH_ERR("ML capability failed\n"); + return -1; + } + + if (global.ml_capa.max_models == 0) { + global.disabled = 1; + ODPH_DBG("ML test disabled\n"); + return 0; + } + + global.num_compl = ODPH_MIN(NUM_COMPL, global.ml_capa.pool.max_num); + + return 0; +} + +static int check_ml_support(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static void test_ml_capability(void) +{ + odp_ml_capability_t ml_capa; + + memset(&ml_capa, 0, sizeof(odp_ml_capability_t)); + CU_ASSERT(odp_ml_capability(&ml_capa) == 0); + + if (ml_capa.max_models == 0) + return; + + CU_ASSERT(ml_capa.max_model_size > 0); + CU_ASSERT(ml_capa.max_models_loaded > 0); + CU_ASSERT(ml_capa.max_inputs > 0); + CU_ASSERT(ml_capa.max_outputs > 0); + CU_ASSERT(ml_capa.max_segs_per_input > 0); + CU_ASSERT(ml_capa.max_segs_per_output > 0); + CU_ASSERT(ml_capa.min_input_align > 0); + CU_ASSERT(ml_capa.min_output_align > 0); + + if ((ml_capa.load.compl_mode_mask | ml_capa.run.compl_mode_mask) & + ODP_ML_COMPL_MODE_EVENT) { + odp_pool_capability_t pool_capa; + + CU_ASSERT_FATAL(odp_pool_capability(&pool_capa) == 0); + + CU_ASSERT(ml_capa.pool.max_pools > 0); + CU_ASSERT(ml_capa.pool.max_pools <= pool_capa.max_pools); + CU_ASSERT(ml_capa.pool.max_num > 0); + CU_ASSERT(ml_capa.pool.max_cache_size >= ml_capa.pool.min_cache_size); + } + + CU_ASSERT(ml_capa.load.compl_mode_mask); + CU_ASSERT(ml_capa.run.compl_mode_mask); + + if (ml_capa.load.compl_mode_mask & ODP_ML_COMPL_MODE_EVENT) + CU_ASSERT(ml_capa.load.compl_queue_plain || ml_capa.load.compl_queue_sched); + + if (ml_capa.run.compl_mode_mask & ODP_ML_COMPL_MODE_EVENT) + CU_ASSERT(ml_capa.run.compl_queue_plain || ml_capa.run.compl_queue_sched); +} + +static void test_ml_param(uint8_t fill) +{ + odp_ml_config_t config; + odp_ml_model_param_t model_param; + odp_ml_compl_pool_param_t pool_param; + odp_ml_compl_param_t compl_param; + odp_ml_run_param_t run_param; + + memset(&config, fill, sizeof(config)); + odp_ml_config_init(&config); + CU_ASSERT(config.max_models_created == 1); + CU_ASSERT(config.max_models_loaded == 1); + CU_ASSERT(config.load_mode_mask == 0); + CU_ASSERT(config.run_mode_mask == 0); + + memset(&model_param, fill, sizeof(model_param)); + odp_ml_model_param_init(&model_param); + CU_ASSERT(model_param.max_compl_id == 0); + CU_ASSERT(!model_param.extra_stat_enable); + CU_ASSERT(model_param.extra_param == NULL); + CU_ASSERT(model_param.extra_info.num_inputs == 0); + CU_ASSERT(model_param.extra_info.num_outputs == 0); + CU_ASSERT(model_param.extra_info.input_format == NULL); + CU_ASSERT(model_param.extra_info.output_format == NULL); + + memset(&pool_param, fill, sizeof(pool_param)); + odp_ml_compl_pool_param_init(&pool_param); + CU_ASSERT(pool_param.uarea_size == 0); + CU_ASSERT(pool_param.uarea_init.args == NULL); + CU_ASSERT(pool_param.uarea_init.init_fn == NULL); + CU_ASSERT(pool_param.cache_size <= global.ml_capa.pool.max_cache_size); + CU_ASSERT(pool_param.cache_size >= global.ml_capa.pool.min_cache_size); + + memset(&compl_param, fill, sizeof(compl_param)); + odp_ml_compl_param_init(&compl_param); + CU_ASSERT(compl_param.user_ptr == NULL); + + memset(&run_param, fill, sizeof(run_param)); + odp_ml_run_param_init(&run_param); + CU_ASSERT(run_param.batch_size == 0); + CU_ASSERT(run_param.result == NULL); +} + +static void test_ml_param_init(void) +{ + test_ml_param(0x00); + test_ml_param(0xff); +} + +static void test_ml_debug(void) +{ + odp_ml_print(); +} + +static void ml_compl_pool_create_max_pools(void) +{ + int ret; + uint32_t i, j; + odp_ml_compl_pool_param_t ml_pool_param; + uint32_t max_pools = global.ml_capa.pool.max_pools; + odp_pool_t compl_pools[max_pools]; + + odp_ml_compl_pool_param_init(&ml_pool_param); + ml_pool_param.num = global.num_compl; + for (i = 0; i < max_pools; i++) { + compl_pools[i] = odp_ml_compl_pool_create(NULL, &ml_pool_param); + + if (compl_pools[i] == ODP_POOL_INVALID) + break; + } + + CU_ASSERT(i == max_pools); + + /* Destroy the created valid pools */ + for (j = 0; j < i; j++) { + ret = odp_pool_destroy(compl_pools[j]); + CU_ASSERT(ret == 0); + + if (ret == -1) + ODPH_ERR("ML completion pool destroy failed: %u / %u\n", j, i); + } +} + +static void compl_pool_info(void) +{ + odp_pool_t pool; + odp_pool_t compl_pool; + odp_pool_info_t pool_info; + odp_ml_compl_pool_param_t pool_param; + + /* Create an ML job completion pool */ + odp_ml_compl_pool_param_init(&pool_param); + pool_param.num = global.num_compl; + + compl_pool = odp_ml_compl_pool_create(COMPL_POOL_NAME, &pool_param); + CU_ASSERT_FATAL(compl_pool != ODP_POOL_INVALID); + + /* Verify info about the created ML completion pool compl_pool */ + pool = odp_pool_lookup(COMPL_POOL_NAME); + CU_ASSERT(pool == compl_pool); + + memset(&pool_info, 0x66, sizeof(odp_pool_info_t)); + CU_ASSERT_FATAL(odp_pool_info(compl_pool, &pool_info) == 0); + + CU_ASSERT(!strcmp(pool_info.name, COMPL_POOL_NAME)); + CU_ASSERT(pool_info.pool_ext == 0); + CU_ASSERT(pool_info.type == ODP_POOL_ML_COMPL); + CU_ASSERT(pool_info.ml_pool_param.num == NUM_COMPL); + CU_ASSERT(pool_info.ml_pool_param.uarea_size == 0); + CU_ASSERT(pool_info.ml_pool_param.cache_size == pool_param.cache_size); + + CU_ASSERT_FATAL(odp_pool_destroy(compl_pool) == 0); +} + +static void compl_alloc_max(void) +{ + uint64_t u64; + odp_event_t event; + odp_pool_t compl_pool; + odp_ml_compl_pool_param_t pool_param; + const int num = ODPH_MIN(global.ml_capa.pool.max_num, 1000000u); + odp_ml_compl_t *compl = malloc(num * sizeof(odp_ml_compl_t)); + + CU_ASSERT_FATAL(compl != NULL); + + /* Create an ML job completion pool */ + odp_ml_compl_pool_param_init(&pool_param); + pool_param.num = num; + + compl_pool = odp_ml_compl_pool_create(COMPL_POOL_NAME, &pool_param); + CU_ASSERT_FATAL(compl_pool != ODP_POOL_INVALID); + + for (int i = 0; i < num; i++) { + compl[i] = odp_ml_compl_alloc(compl_pool); + CU_ASSERT_FATAL(compl[i] != ODP_ML_COMPL_INVALID); + + u64 = odp_ml_compl_to_u64(compl[i]); + CU_ASSERT(u64 != odp_ml_compl_to_u64(ODP_ML_COMPL_INVALID)); + + event = odp_ml_compl_to_event(compl[i]); + CU_ASSERT(odp_event_type(event) == ODP_EVENT_ML_COMPL); + } + + for (int i = 0; i < num; i++) + odp_ml_compl_free(compl[i]); + + free(compl); + + CU_ASSERT_FATAL(odp_pool_destroy(compl_pool) == 0); +} + +static void compl_pool_lookup(void) +{ + odp_pool_t pool, pool_a, pool_b; + odp_ml_compl_pool_param_t pool_param; + + /* Create an ML job completion pool */ + odp_ml_compl_pool_param_init(&pool_param); + pool_param.num = global.num_compl; + + pool_a = odp_ml_compl_pool_create(COMPL_POOL_NAME, &pool_param); + CU_ASSERT_FATAL(pool_a != ODP_POOL_INVALID); + + pool = odp_pool_lookup(COMPL_POOL_NAME); + CU_ASSERT(pool == pool_a); + + /* Second pool with the same name */ + pool_b = odp_ml_compl_pool_create(COMPL_POOL_NAME, &pool_param); + CU_ASSERT_FATAL(pool_b != ODP_POOL_INVALID); + + pool = odp_pool_lookup(COMPL_POOL_NAME); + CU_ASSERT(pool == pool_a || pool == pool_b); + + CU_ASSERT(odp_pool_destroy(pool_a) == 0); + CU_ASSERT(odp_pool_destroy(pool_b) == 0); +} + +static int check_event_user_area(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + if (((global.ml_capa.load.compl_mode_mask | global.ml_capa.run.compl_mode_mask) & + ODP_ML_COMPL_MODE_EVENT) && + (global.ml_capa.pool.max_uarea_size > 0)) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static void test_ml_compl_user_area(void) +{ + uint32_t i; + void *addr; + void *prev; + odp_pool_t pool; + odp_ml_compl_pool_param_t pool_param; + uint32_t size = global.ml_capa.pool.max_uarea_size; + uint32_t num = global.num_compl; + odp_ml_compl_t compl_evs[num]; + + odp_ml_compl_pool_param_init(&pool_param); + pool_param.num = num; + pool_param.uarea_size = size; + pool = odp_ml_compl_pool_create(NULL, &pool_param); + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + prev = NULL; + for (i = 0; i < num; i++) { + odp_event_t ev; + int flag = 0; + + compl_evs[i] = odp_ml_compl_alloc(pool); + + if (compl_evs[i] == ODP_ML_COMPL_INVALID) + break; + + addr = odp_ml_compl_user_area(compl_evs[i]); + + CU_ASSERT_FATAL(addr != NULL); + CU_ASSERT(prev != addr); + + memset(addr, 0, size); + + ev = odp_ml_compl_to_event(compl_evs[i]); + CU_ASSERT(odp_event_user_area(ev) == addr); + CU_ASSERT(odp_event_user_area_and_flag(ev, &flag) == addr); + CU_ASSERT(flag < 0); + + prev = addr; + } + CU_ASSERT(i == num); + + for (uint32_t j = 0; j < i; j++) + odp_ml_compl_free(compl_evs[j]); + + CU_ASSERT(odp_pool_destroy(pool) == 0); +} + +static int check_event_user_area_init(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + if (global.ml_capa.pool.max_uarea_size > 0 && global.ml_capa.pool.uarea_persistence) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static void init_event_uarea(void *uarea, uint32_t size, void *args, uint32_t index) +{ + uarea_init_t *data = args; + + data->count++; + data->mark[index] = 1; + memset(uarea, UAREA, size); +} + +static void test_ml_compl_user_area_init(void) +{ + odp_ml_compl_pool_param_t pool_param; + uint32_t num = global.num_compl, i; + odp_pool_t pool; + uarea_init_t data; + odp_ml_compl_t compl_evs[num]; + uint8_t *uarea; + + memset(&data, 0, sizeof(uarea_init_t)); + odp_ml_compl_pool_param_init(&pool_param); + pool_param.uarea_init.init_fn = init_event_uarea; + pool_param.uarea_init.args = &data; + pool_param.num = num; + pool_param.uarea_size = 1; + pool = odp_ml_compl_pool_create(NULL, &pool_param); + + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + CU_ASSERT(data.count == num); + + for (i = 0; i < num; i++) { + CU_ASSERT(data.mark[i] == 1); + + compl_evs[i] = odp_ml_compl_alloc(pool); + + CU_ASSERT(compl_evs[i] != ODP_ML_COMPL_INVALID); + + if (compl_evs[i] == ODP_ML_COMPL_INVALID) + break; + + uarea = odp_ml_compl_user_area(compl_evs[i]); + + CU_ASSERT(*uarea == UAREA); + } + + for (uint32_t j = 0; j < i; j++) + odp_ml_compl_free(compl_evs[j]); + + odp_pool_destroy(pool); +} + +static void test_ml_fp32_to_uint8(void) +{ + uint8_t u8[8]; + float fp[8] = {-20.f, -16.4f, -14.6f, -12.5f, 0, 31.4f, 80.f, 96.3f}; + uint8_t expected[8] = {0, 0, 4, 10, 43, 127, 255, 255}; + float scale = 0.3746f; + uint8_t zero_point = 43; + + odp_ml_fp32_to_uint8(u8, fp, 8, scale, zero_point); + for (uint32_t i = 0; i < 8; i++) + CU_ASSERT(u8[i] == expected[i]); +} + +static void test_ml_fp32_from_uint8(void) +{ + float fp[4]; + float scale = 0.4f; + uint8_t zero_point = 43; + uint8_t u8[4] = {0, 43, 145, 255}; + float expected[4] = {-17.2f, 0.0f, 40.8f, 84.8f}; + + odp_ml_fp32_from_uint8(fp, u8, 4, scale, zero_point); + for (uint32_t i = 0; i < 4; i++) + CU_ASSERT(fp[i] == expected[i]); +} + +static void test_ml_fp32_to_int8(void) +{ + int8_t i8[5]; + float scale = 0.0223f; + int8_t zero_point = 0; + float fp32[5] = {-3.4f, -2.5f, 0, 1.4f, 2.9f}; + int8_t i8_expected[5] = {-127, -112, 0, 63, 127}; + + odp_ml_fp32_to_int8(i8, fp32, 5, scale, zero_point); + + for (uint32_t i = 0; i < 5; i++) + CU_ASSERT(i8[i] == i8_expected[i]); +} + +static void test_ml_fp32_to_int8_positive_zp(void) +{ + int8_t i8[6]; + float scale = 0.0223f; + int8_t zero_point = 56; + float fp32[6] = {-4.1f, -3.4f, -2.5f, 0, 1.4f, 2.9f}; + int8_t i8_expected[6] = {-127, -96, -56, 56, 119, 127}; + + odp_ml_fp32_to_int8(i8, fp32, 6, scale, zero_point); + + for (uint32_t i = 0; i < 6; i++) + CU_ASSERT(i8[i] == i8_expected[i]); +} + +static void test_ml_fp32_to_int8_negative_zp(void) +{ + int8_t i8[6]; + float scale = 0.0223f; + int8_t zero_point = -56; + float fp32[6] = {-3.4f, -2.5f, 0, 1.4f, 2.9f, 4.1f}; + int8_t i8_expected[6] = {-127, -127, -56, 7, 74, 127}; + + odp_ml_fp32_to_int8(i8, fp32, 6, scale, zero_point); + + for (uint32_t i = 0; i < 6; i++) + CU_ASSERT(i8[i] == i8_expected[i]); +} + +static void test_ml_fp32_from_int8(void) +{ + float fp32[6]; + float scale = 0.05f; + int8_t zero_point = 56; + int8_t i8[6] = {-128, 46, 0, 56, 85, 127}; + float fp32_expected[6] = {-9.2f, -0.5f, -2.8f, 0.0f, 1.45f, 3.55f}; + + odp_ml_fp32_from_int8(fp32, i8, 6, scale, zero_point); + + for (uint32_t i = 0; i < 6; i++) + CU_ASSERT(fp32[i] == fp32_expected[i]); +} + +static int approx_equal(double a, double b) +{ + const double tolerance = .01; + + if (a < 0 && b < 0) { + a = -a; + b = -b; + } + + if (a < b) { + double tmp = a; + + a = b; + b = tmp; + } + + return (a * (1 - tolerance) < b && a * (1 + tolerance) > b); +} + +static void test_ml_fp32_fp16(void) +{ + float fp32[4]; + uint16_t fp16[4]; + float big = 1, small = 1; + const float phi = 1.618033988749; + + fp32[0] = 0.0; + fp32[1] = -0.0; + memset(fp16, 1, sizeof(fp16)); + odp_ml_fp32_to_fp16(fp16, fp32, 2); + memset(fp32, 1, sizeof(fp32)); + odp_ml_fp32_from_fp16(fp32, fp16, 2); + CU_ASSERT(fp32[0] == 0); + CU_ASSERT(fp32[1] == 0); + + /* + * 65504 is the largest normal number for fp16 with 5 exponent bits (IEEE 754-2008). + */ + while (big < 65504 / 2) { + fp32[0] = big; + fp32[1] = -big; + fp32[2] = small; + fp32[3] = -small; + memset(fp16, 0, sizeof(fp16)); + odp_ml_fp32_to_fp16(fp16, fp32, 4); + memset(fp32, 0, sizeof(fp32)); + odp_ml_fp32_from_fp16(fp32, fp16, 4); + CU_ASSERT(approx_equal(fp32[0], big)); + CU_ASSERT(approx_equal(fp32[1], -big)); + CU_ASSERT(approx_equal(fp32[2], small)); + CU_ASSERT(approx_equal(fp32[3], -small)); + big *= phi; + small /= phi; + } +} + +odp_testinfo_t ml_suite[] = { + ODP_TEST_INFO(test_ml_capability), + ODP_TEST_INFO_CONDITIONAL(test_ml_param_init, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_debug, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(ml_compl_pool_create_max_pools, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(compl_pool_info, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(compl_alloc_max, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(compl_pool_lookup, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_compl_user_area, check_event_user_area), + ODP_TEST_INFO_CONDITIONAL(test_ml_compl_user_area_init, check_event_user_area_init), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_to_uint8, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_from_uint8, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_to_int8, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_to_int8_positive_zp, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_to_int8_negative_zp, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_from_int8, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_fp16, check_ml_support), + ODP_TEST_INFO_NULL +}; + +odp_suiteinfo_t ml_suites[] = { + {"ML", ml_suite_init, NULL, ml_suite}, + ODP_SUITE_INFO_NULL +}; + +int main(int argc, char *argv[]) +{ + int ret; + + /* parse common options: */ + if (odp_cunit_parse_options(&argc, argv)) + return -1; + + ret = odp_cunit_register(ml_suites); + + if (ret == 0) + ret = odp_cunit_run(); + + return ret; +} diff --git a/test/validation/api/pool/pool.c b/test/validation/api/pool/pool.c index c78931d5c..86a47230a 100644 --- a/test/validation/api/pool/pool.c +++ b/test/validation/api/pool/pool.c @@ -1718,7 +1718,8 @@ static void test_packet_pool_ext_capa(void) odp_pool_ext_capability_t capa; odp_pool_type_t type; const odp_pool_type_t unsupported_types[] = {ODP_POOL_BUFFER, ODP_POOL_TIMEOUT, - ODP_POOL_VECTOR, ODP_POOL_DMA_COMPL}; + ODP_POOL_VECTOR, ODP_POOL_DMA_COMPL, + ODP_POOL_ML_COMPL}; const int num_types = ODPH_ARRAY_SIZE(unsupported_types); /* Verify operation for unsupported pool types */ From da534723de652505ed8587a9f625b0ab3c5eb6bf Mon Sep 17 00:00:00 2001 From: Lifang Zhang <lifang.zhang@nokia.com> Date: Sat, 2 Dec 2023 07:06:28 +0200 Subject: [PATCH 120/147] linux-gen: validation: ml: add tests based on onnx models for the new ML API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added validation tests based on onnx models for the new ML API. Signed-off-by: Lifang Zhang <lifang.zhang@nokia.com> Signed-off-by: Jere Leppänen <jere.leppanen@nokia.com> Reviewed-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-generic/m4/configure.m4 | 1 + platform/linux-generic/test/Makefile.am | 5 + .../test/validation/api/ml/.gitignore | 1 + .../test/validation/api/ml/Makefile.am | 34 + .../test/validation/api/ml/README.md | 23 + .../test/validation/api/ml/batch_add.onnx | Bin 0 -> 144 bytes .../test/validation/api/ml/batch_add_gen.py | 32 + .../test/validation/api/ml/gen_models.sh | 14 + .../test/validation/api/ml/ml_linux.c | 1167 +++++++++++++++++ .../test/validation/api/ml/requirements.txt | 2 + .../test/validation/api/ml/simple_linear.onnx | Bin 0 -> 214 bytes .../validation/api/ml/simple_linear_gen.py | 34 + 12 files changed, 1313 insertions(+) create mode 100644 platform/linux-generic/test/validation/api/ml/.gitignore create mode 100644 platform/linux-generic/test/validation/api/ml/Makefile.am create mode 100644 platform/linux-generic/test/validation/api/ml/README.md create mode 100644 platform/linux-generic/test/validation/api/ml/batch_add.onnx create mode 100644 platform/linux-generic/test/validation/api/ml/batch_add_gen.py create mode 100755 platform/linux-generic/test/validation/api/ml/gen_models.sh create mode 100644 platform/linux-generic/test/validation/api/ml/ml_linux.c create mode 100644 platform/linux-generic/test/validation/api/ml/requirements.txt create mode 100644 platform/linux-generic/test/validation/api/ml/simple_linear.onnx create mode 100644 platform/linux-generic/test/validation/api/ml/simple_linear_gen.py diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index fa9f77914..3306849d2 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -77,6 +77,7 @@ AC_CONFIG_FILES([platform/linux-generic/Makefile platform/linux-generic/test/example/switch/Makefile platform/linux-generic/test/validation/api/shmem/Makefile platform/linux-generic/test/validation/api/pktio/Makefile + platform/linux-generic/test/validation/api/ml/Makefile platform/linux-generic/test/performance/Makefile platform/linux-generic/test/performance/dmafwd/Makefile platform/linux-generic/test/pktio_ipc/Makefile]) diff --git a/platform/linux-generic/test/Makefile.am b/platform/linux-generic/test/Makefile.am index 30ef26078..7aca5fd3f 100644 --- a/platform/linux-generic/test/Makefile.am +++ b/platform/linux-generic/test/Makefile.am @@ -21,6 +21,11 @@ SUBDIRS += validation/api/pktio \ example \ performance +if WITH_ML +TESTS += validation/api/ml/ml_linux$(EXEEXT) +SUBDIRS += validation/api/ml +endif + if ODP_PKTIO_PCAP TESTS += validation/api/pktio/pktio_run_pcap.sh endif diff --git a/platform/linux-generic/test/validation/api/ml/.gitignore b/platform/linux-generic/test/validation/api/ml/.gitignore new file mode 100644 index 000000000..e31f902c4 --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/.gitignore @@ -0,0 +1 @@ +ml_linux diff --git a/platform/linux-generic/test/validation/api/ml/Makefile.am b/platform/linux-generic/test/validation/api/ml/Makefile.am new file mode 100644 index 000000000..f4b9e9755 --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/Makefile.am @@ -0,0 +1,34 @@ +include ../Makefile.inc + +test_PROGRAMS = ml_linux +ml_linux_SOURCES = ml_linux.c + +EXTRA_DIST = \ + batch_add_gen.py \ + batch_add.onnx \ + gen_models.sh \ + README.md \ + requirements.txt \ + simple_linear_gen.py \ + simple_linear.onnx + +# If building out-of-tree, make check will not copy the scripts and data to the +# $(builddir) assuming that all commands are run locally. However this prevents +# running tests on a remote target using LOG_COMPILER. +# So copy all script and data files explicitly here. +all-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + if [ -e $(srcdir)/$$f ]; then \ + mkdir -p $(builddir)/$$(dirname $$f); \ + cp -f $(srcdir)/$$f $(builddir)/$$f; \ + fi \ + done \ + fi + +clean-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + rm -f $(builddir)/$$f; \ + done \ + fi diff --git a/platform/linux-generic/test/validation/api/ml/README.md b/platform/linux-generic/test/validation/api/ml/README.md new file mode 100644 index 000000000..80ad30e96 --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/README.md @@ -0,0 +1,23 @@ +# How to run ML validation test + +Simple onnx models are used to test ML API. + +## Generate models + +### Install python requirements + +```bash +python3 -m pip install -r <this directory>/requirements.txt +``` + +### Generate models for validation tests + +```bash +<this directory>/gen_models.sh +``` + +## Run ML validation tests + +```bash +<this directory>/ml_linux +``` diff --git a/platform/linux-generic/test/validation/api/ml/batch_add.onnx b/platform/linux-generic/test/validation/api/ml/batch_add.onnx new file mode 100644 index 0000000000000000000000000000000000000000..43485f463feb6d1e5680616f225cb93f35dabae8 GIT binary patch literal 144 zcmd<!5aRU9&(2I#@OKH&U^L>cRIpX3FjUZ1s4%k1;gaTJsxSm%BO%60DNd)vlH?2p z$CMN$W*{Mi;75sq)Cuu(@p5nradI&WF(z{{aWI30jWC3hM7S6$F+`nMxEKWZ0Qfl< AFaQ7m literal 0 HcmV?d00001 diff --git a/platform/linux-generic/test/validation/api/ml/batch_add_gen.py b/platform/linux-generic/test/validation/api/ml/batch_add_gen.py new file mode 100644 index 000000000..33515bd2f --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/batch_add_gen.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Nokia +# + +import onnx +from onnx import helper +from onnx import TensorProto + +graph = helper.make_graph( + [ # nodes + helper.make_node("Add", ["x1", "x2"], ["y"], "Batch Add"), + ], + "Batch Add", # name + [ # inputs + helper.make_tensor_value_info('x1', TensorProto.DOUBLE, ["c", 3]), + helper.make_tensor_value_info('x2', TensorProto.DOUBLE, ["c", 3]), + ], + [ # outputs + helper.make_tensor_value_info('y', TensorProto.DOUBLE, ["c", 3]), + ] +) + +model = helper.make_model( + graph, + opset_imports=[helper.make_opsetid("", 14)], + producer_name='ODP validation tests', + model_version=1, + doc_string="y = x1 + x2", + ir_version = 8 +) + +onnx.save(model, 'batch_add.onnx') diff --git a/platform/linux-generic/test/validation/api/ml/gen_models.sh b/platform/linux-generic/test/validation/api/ml/gen_models.sh new file mode 100755 index 000000000..d88f3c432 --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/gen_models.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Nokia +# + +set -e + +# cd to the directory where this script is in +cd "$( dirname "${BASH_SOURCE[0]}" )" + +python3 simple_linear_gen.py + +python3 batch_add_gen.py diff --git a/platform/linux-generic/test/validation/api/ml/ml_linux.c b/platform/linux-generic/test/validation/api/ml/ml_linux.c new file mode 100644 index 000000000..28e18fbb5 --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/ml_linux.c @@ -0,0 +1,1167 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include <unistd.h> +#include <string.h> +#include <libgen.h> +#include <odp_api.h> +#include <odp/helper/odph_api.h> +#include "odp_cunit_common.h" + +#define TIMEOUT 5 +#define MODEL_NAME "Test" +#define NUM_INPUTS 1 +#define NUM_OUTPUTS 1 +#define RUN_NUM 2 +#define BUF_LEN 256 +#define CONFIG_MAX_MODEL_SIZE 500 + +#define COMPL_POOL_NAME "ML compl pool" +#define NUM_COMPL 10 + +/** + * About model simple_linear.onnx being tested in this suite + * + * Model info: + * Version: 1 + * Inputs: name: x, type: int32, shape: [1] + * Outputs: name: y, type: int32, shape: [1] + * + * The model is of form y = 3 * x + 4 + * Thus when x = 5, the output y should be 19. + */ +typedef struct global_t { + int disabled; + odp_ml_capability_t ml_capa; + odp_ml_config_t ml_config; + odp_ml_model_param_t model_param; + odp_ml_model_t ml_model; + odp_pool_t compl_pool; + odp_queue_t queue; + odp_ml_data_t data; + odp_ml_data_seg_t input_seg; + odp_ml_data_seg_t output_seg; + odp_ml_run_param_t run_param; + uint64_t wait_ns; + int32_t x; + int32_t y; + int32_t y_expected; + +} global_t; + +static global_t global; + +static int fill_model_param(const char *model_name, odp_ml_model_param_t *model_param) +{ + size_t size; + char *pos; + char *exe_dir; + size_t exe_dir_len; + FILE *model_file; + char exe_path[BUF_LEN]; + ssize_t exe_path_len; + char model_path[BUF_LEN]; + + /* Model file is placed in the same directory as the executable ml_linux */ + exe_path_len = readlink("/proc/self/exe", exe_path, BUF_LEN - 1); + if (exe_path_len != -1) { + exe_path[exe_path_len] = '\0'; + + pos = strstr(exe_path, ".libs"); + if (pos) + *(pos + 5) = '\0'; + + exe_dir = dirname(exe_path); + exe_dir_len = strlen(exe_dir); + + memcpy(model_path, exe_dir, exe_dir_len); + model_path[exe_dir_len] = '/'; + model_path[exe_dir_len + 1] = '\0'; + + strncat(model_path, model_name, BUF_LEN - strlen(model_path) - 1); + ODPH_DBG("model_path: %s\n", model_path); + model_file = fopen(model_path, "rb"); + } else { /* Can't get executable path, try to find model file at current dir*/ + model_file = fopen(model_name, "rb"); + } + + if (model_file == NULL) { + perror("Failed to open model file"); + return -1; + } + + /* Get the model file size in bytes */ + fseek(model_file, 0, SEEK_END); + model_param->size = ftell(model_file); + rewind(model_file); + + model_param->model = malloc(model_param->size); + if (!model_param->model) { + ODPH_ERR("\n\nMemory allocation failed\n"); + fclose(model_file); + return -1; + } + size = fread(model_param->model, model_param->size, 1, model_file); + + fclose(model_file); + if (size != 1) { + ODPH_ERR("\n\nRead model file failed\n"); + return -1; + } + + model_param->max_compl_id = 0; + + return 0; +} + +static int ml_suite_init(void) +{ + odp_ml_capability_t *ml_capa = &global.ml_capa; + odp_queue_param_t queue_param; + odp_ml_compl_pool_param_t ml_pool_param; + + memset(&global, 0, sizeof(global_t)); + global.queue = ODP_QUEUE_INVALID; + global.compl_pool = ODP_POOL_INVALID; + + if (odp_ml_capability(ml_capa)) { + ODPH_ERR("ML capability failed\n"); + return -1; + } + + if (ml_capa->max_models == 0) { + global.disabled = 1; + ODPH_DBG("ML test disabled\n"); + return 0; + } + + /* Configure ML */ + odp_ml_config_init(&global.ml_config); + global.ml_config.max_models_created = ml_capa->max_models; + global.ml_config.max_models_loaded = ml_capa->max_models_loaded; + global.ml_config.max_model_size = CONFIG_MAX_MODEL_SIZE; + + if (ml_capa->load.compl_mode_mask & ODP_ML_COMPL_MODE_SYNC) + global.ml_config.load_mode_mask |= ODP_ML_COMPL_MODE_SYNC; + + if (ml_capa->load.compl_mode_mask & ODP_ML_COMPL_MODE_POLL) + global.ml_config.load_mode_mask |= ODP_ML_COMPL_MODE_POLL; + + if (ml_capa->load.compl_mode_mask & ODP_ML_COMPL_MODE_EVENT) + global.ml_config.load_mode_mask |= ODP_ML_COMPL_MODE_EVENT; + + if (ml_capa->run.compl_mode_mask & ODP_ML_COMPL_MODE_SYNC) + global.ml_config.run_mode_mask |= ODP_ML_COMPL_MODE_SYNC; + + if (ml_capa->run.compl_mode_mask & ODP_ML_COMPL_MODE_POLL) + global.ml_config.run_mode_mask |= ODP_ML_COMPL_MODE_POLL; + + if (ml_capa->run.compl_mode_mask & ODP_ML_COMPL_MODE_EVENT) + global.ml_config.run_mode_mask |= ODP_ML_COMPL_MODE_EVENT; + + if (odp_ml_config(&global.ml_config)) { + ODPH_ERR("\n\nConfiguring ML failed\n"); + return -1; + } + + global.x = 5; + global.wait_ns = 500 * ODP_TIME_MSEC_IN_NS; + global.y_expected = 19; /* y = 3 * x + 4 = 3 * 5 + 4 = 19 */ + + /* Prepare data for running model inference */ + odp_ml_run_param_init(&global.run_param); + + global.data.num_input_seg = NUM_INPUTS; + global.data.input_seg = &global.input_seg; + global.input_seg.size = sizeof(int32_t); + global.input_seg.addr = &global.x; + + global.data.num_output_seg = NUM_OUTPUTS; + global.data.output_seg = &global.output_seg; + global.output_seg.size = sizeof(int32_t); + global.output_seg.addr = &global.y; + + if (fill_model_param("simple_linear.onnx", &global.model_param)) + return -1; + + /* Create ML model */ + global.ml_model = odp_ml_model_create(MODEL_NAME, &global.model_param); + if (global.ml_model == ODP_ML_MODEL_INVALID) { + ODPH_ERR("Create ML model failed\n"); + goto error; + } + + /* Asynchronous mode with event completion is not supported */ + if (!((ml_capa->load.compl_mode_mask & ODP_ML_COMPL_MODE_EVENT) || + (ml_capa->run.compl_mode_mask & ODP_ML_COMPL_MODE_EVENT))) + return 0; + + /* Create a queue for sending ML completion event to */ + odp_queue_param_init(&queue_param); + queue_param.type = ODP_QUEUE_TYPE_SCHED; + queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL; + queue_param.sched.prio = odp_schedule_default_prio(); + queue_param.sched.group = ODP_SCHED_GROUP_ALL; + + global.queue = odp_queue_create("ML compl queue", &queue_param); + if (global.queue == ODP_QUEUE_INVALID) { + ODPH_ERR("Queue create failed\n"); + goto error; + } + + /* Create an ML job completion pool */ + if (ml_capa->pool.max_num < NUM_COMPL) { + ODPH_ERR("Too small ML compl pool %u\n", ml_capa->pool.max_num); + goto error; + } + + odp_ml_compl_pool_param_init(&ml_pool_param); + ml_pool_param.num = NUM_COMPL; + + global.compl_pool = odp_ml_compl_pool_create(COMPL_POOL_NAME, &ml_pool_param); + if (global.compl_pool == ODP_POOL_INVALID) { + ODPH_ERR("Create ML completion pool failed\n"); + goto error; + } + + return 0; + +error: + free(global.model_param.model); + return -1; +} + +static int ml_suite_term(void) +{ + if (global.compl_pool != ODP_POOL_INVALID && + odp_pool_destroy(global.compl_pool)) { + ODPH_ERR("Completion pool destroy failed\n"); + return -1; + } + + if (global.ml_model && odp_ml_model_destroy(global.ml_model)) { + ODPH_ERR("Destroy ML model failed\n"); + return -1; + } + + if (global.queue != ODP_QUEUE_INVALID && + odp_queue_destroy(global.queue)) { + ODPH_ERR("Destroy ML queue failed\n"); + return -1; + } + + free(global.model_param.model); + + return 0; +} + +static int check_ml_support(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static int check_load_sync(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + if (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_SYNC) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static int check_load_poll(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + if (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_POLL) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static int check_load_event(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + if (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_EVENT) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static int check_run_sync(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + /* Model run test uses synchronous load */ + if ((global.ml_config.run_mode_mask & ODP_ML_COMPL_MODE_SYNC) && + (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_SYNC)) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static int check_run_poll(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + /* Poll mode model run test uses synchronous load */ + if ((global.ml_config.run_mode_mask & ODP_ML_COMPL_MODE_POLL) && + (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_SYNC)) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static int check_run_event(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + /* Poll mode model run test uses synchronous load */ + if ((global.ml_config.run_mode_mask & ODP_ML_COMPL_MODE_EVENT) && + (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_SYNC)) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static int check_run_poll_event(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + /* test_ml_run_start_multi uses synchronous load, poll mode and event mode run */ + if ((global.ml_config.run_mode_mask & ODP_ML_COMPL_MODE_EVENT) && + (global.ml_config.run_mode_mask & ODP_ML_COMPL_MODE_POLL) && + (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_SYNC)) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static void test_ml_debug(void) +{ + uint64_t u64; + + u64 = odp_ml_model_to_u64(global.ml_model); + CU_ASSERT(u64 != odp_ml_model_to_u64(ODP_ML_MODEL_INVALID)); + printf("\n ML model handle: 0x%" PRIx64 "\n", u64); + + odp_ml_model_print(global.ml_model); +} + +static void test_ml_model_create(void) +{ + uint32_t i; + /* One for global.ml_model */ + uint32_t max_models = global.ml_config.max_models_created - 1; + odp_ml_model_t models[max_models]; + + for (i = 0; i < max_models; i++) { + models[i] = odp_ml_model_create(NULL, &global.model_param); + + if (models[i] == ODP_ML_MODEL_INVALID) { + ODPH_ERR("ML model create failed: %u / %u\n", i, max_models); + break; + } + } + + CU_ASSERT(i == max_models); + max_models = i; + + /* Destroy valid models */ + for (i = 0; i < max_models; i++) + CU_ASSERT_FATAL(odp_ml_model_destroy(models[i]) == 0); +} + +static void test_ml_model_lookup(void) +{ + odp_ml_model_t model2; + odp_ml_model_t model_lookup; + + /* Look up model with the same name, should find one with equal handle */ + model_lookup = odp_ml_model_lookup(MODEL_NAME); + CU_ASSERT_FATAL(model_lookup != ODP_ML_MODEL_INVALID); + CU_ASSERT(odp_ml_model_to_u64(global.ml_model) == odp_ml_model_to_u64(model_lookup)); + + /* Look up model with a different name, should return invalid handle */ + model_lookup = odp_ml_model_lookup("diff"); + CU_ASSERT_FATAL(model_lookup == ODP_ML_MODEL_INVALID); + + model2 = odp_ml_model_create(MODEL_NAME, &global.model_param); + CU_ASSERT_FATAL(model2 != ODP_ML_MODEL_INVALID); + CU_ASSERT(odp_ml_model_to_u64(global.ml_model) != odp_ml_model_to_u64(model2)); + + model_lookup = odp_ml_model_lookup(MODEL_NAME); + CU_ASSERT(odp_ml_model_to_u64(model_lookup) == odp_ml_model_to_u64(global.ml_model) || + odp_ml_model_to_u64(model_lookup) == odp_ml_model_to_u64(model2)); + + CU_ASSERT(odp_ml_model_destroy(model2) == 0); +} + +static void test_ml_model_info(void) +{ + int ret; + uint32_t num_ret; + odp_ml_model_info_t ml_info; + odp_ml_input_info_t input_info[2]; + odp_ml_output_info_t output_info[2]; + + /* Verify model info about global.ml_model, namely, simple_linear.onnx */ + memset(&ml_info, 0x88, sizeof(odp_ml_model_info_t)); + ret = odp_ml_model_info(global.ml_model, &ml_info); + CU_ASSERT(ret == 0); + CU_ASSERT(!strcmp(ml_info.name, MODEL_NAME)); + CU_ASSERT(ml_info.model_version == 1); + CU_ASSERT(ml_info.num_inputs == NUM_INPUTS); + CU_ASSERT(ml_info.num_outputs == NUM_OUTPUTS); + + num_ret = odp_ml_model_input_info(global.ml_model, input_info, NUM_INPUTS); + CU_ASSERT(num_ret == NUM_INPUTS); + CU_ASSERT(!strcmp(input_info[0].name, "x")); + CU_ASSERT(input_info[0].shape.num_dim == 1); + CU_ASSERT(input_info[0].shape.dim[0] == 1); + CU_ASSERT((int)input_info[0].data_type == ODP_ML_DATA_TYPE_INT32); + + /* When num is 0, return normally, and input_info is ignored */ + num_ret = odp_ml_model_input_info(global.ml_model, input_info, 0); + CU_ASSERT(num_ret == NUM_INPUTS); + + /* When num is bigger than actual number of inputs, extra input_info is left untouched */ + input_info[1].data_type = (odp_ml_data_type_t)-1; + num_ret = odp_ml_model_input_info(global.ml_model, input_info, NUM_INPUTS + 1); + CU_ASSERT(num_ret == NUM_INPUTS); + CU_ASSERT(!strcmp(input_info[0].name, "x")); + CU_ASSERT(input_info[0].shape.num_dim == 1); + CU_ASSERT(input_info[0].shape.dim[0] == 1); + CU_ASSERT((int)input_info[0].data_type == ODP_ML_DATA_TYPE_INT32); + /* input_info[1] is left untouched */ + CU_ASSERT(input_info[1].data_type == (odp_ml_data_type_t)-1); + + num_ret = odp_ml_model_output_info(global.ml_model, output_info, NUM_OUTPUTS); + CU_ASSERT(num_ret == NUM_OUTPUTS); + CU_ASSERT(!strcmp(output_info[0].name, "y")); + CU_ASSERT(output_info[0].shape.num_dim == 1); + CU_ASSERT(output_info[0].shape.dim[0] == 1); + CU_ASSERT((int)output_info[0].data_type == ODP_ML_DATA_TYPE_INT32); + + /* When num is 0, return normally, and input_info is ignored */ + num_ret = odp_ml_model_output_info(global.ml_model, output_info, 0); + CU_ASSERT(num_ret == NUM_OUTPUTS); + + /* When num is bigger than actual number of inputs, extra output_info is left untouched */ + num_ret = odp_ml_model_output_info(global.ml_model, output_info, NUM_OUTPUTS + 1); + output_info[1].shape.num_dim = 98876; + CU_ASSERT(num_ret == NUM_OUTPUTS); + CU_ASSERT(!strcmp(output_info[0].name, "y")); + CU_ASSERT(output_info[0].shape.num_dim == 1); + CU_ASSERT(output_info[0].shape.dim[0] == 1); + CU_ASSERT((int)output_info[0].data_type == ODP_ML_DATA_TYPE_INT32); + /* output_info[1] is left untouched */ + CU_ASSERT(output_info[1].shape.num_dim == 98876); +} + +static void test_ml_model_load(void) +{ + int ret; + odp_ml_model_t test_model; + odp_ml_load_result_t result; + + test_model = odp_ml_model_create(NULL, &global.model_param); + CU_ASSERT_FATAL(test_model != ODP_ML_MODEL_INVALID); + + ret = odp_ml_model_load(test_model, &result); + CU_ASSERT(ret == 0); + CU_ASSERT(result.error_code == 0); + + ret = odp_ml_model_unload(test_model, NULL); + CU_ASSERT(ret == 0); + + CU_ASSERT(odp_ml_model_destroy(test_model) == 0); +} + +/* Test asynchronous model loading in ODP_ML_COMPL_MODE_POLL mode */ +static void test_ml_model_load_async_poll(void) +{ + int ret; + odp_ml_load_result_t result; + odp_ml_compl_param_t compl_param; + int dummy = 6; + void *user_ptr = &dummy; + uint64_t wait_ns = 500 * ODP_TIME_MSEC_IN_NS; + + memset(&result, 0, sizeof(result)); + odp_ml_compl_param_init(&compl_param); + compl_param.mode = ODP_ML_COMPL_MODE_POLL; + compl_param.compl_id = 0; + compl_param.user_ptr = user_ptr; + + ret = odp_ml_model_load_start(global.ml_model, &compl_param); + CU_ASSERT_FATAL(ret == 0); + + /* When odp_ml_model_load_start() succeeded, continue to check completion status */ + for (int i = 0; i < TIMEOUT; i++) { + ret = odp_ml_model_load_status(global.ml_model, 0, &result); + if (ret) + break; + + /* ret = 0 meaning run has not finished, continue to check status */ + odp_time_wait_ns(wait_ns); + } + + CU_ASSERT(ret > 0); + CU_ASSERT(result.error_code == 0); + CU_ASSERT(result.user_ptr == user_ptr); + /* odp_ml_model_load does not modify data in user_ptr */ + if (result.user_ptr) + CU_ASSERT(*(int *)result.user_ptr == dummy); + + ret = odp_ml_model_unload_start(global.ml_model, &compl_param); + CU_ASSERT_FATAL(ret == 0); + + /* When odp_ml_model_unload_start() succeeded, continue to check completion + * status */ + for (int i = 0; i < TIMEOUT; i++) { + ret = odp_ml_model_unload_status(global.ml_model, 0, &result); + if (ret) + break; + + /* ret = 0 meaning run has not finished, continue to check status */ + odp_time_wait_ns(wait_ns); + } + + CU_ASSERT_FATAL(ret > 0); + CU_ASSERT(result.error_code == 0); + CU_ASSERT(result.user_ptr == user_ptr); + + /* odp_ml_model_unload does not modify data in user_ptr */ + if (result.user_ptr) + CU_ASSERT(*(int *)result.user_ptr == dummy); +} + +static int +get_result_from_ml_compl_event(odp_ml_load_result_t *load_result, odp_ml_run_result_t *run_result) +{ + int ret; + odp_event_t ev; + odp_ml_compl_t compl; + odp_event_type_t ev_type; + odp_queue_t from_queue = ODP_QUEUE_INVALID; + uint64_t sched_wait = odp_schedule_wait_time(global.wait_ns); + + /* Run event scheduler to find the ml completion event */ + for (int i = 0; i < TIMEOUT; i++) { + ev = odp_schedule(&from_queue, sched_wait); + if (ev != ODP_EVENT_INVALID) + break; + } + + CU_ASSERT(ev != ODP_EVENT_INVALID); + if (ev == ODP_EVENT_INVALID) { + ODPH_ERR("Timeout while waiting for completion event\n"); + return -1; + } + + ev_type = odp_event_type(ev); + CU_ASSERT(from_queue == global.queue); + CU_ASSERT(ev_type == ODP_EVENT_ML_COMPL); + if (from_queue != global.queue || ev_type != ODP_EVENT_ML_COMPL) { + odp_event_free(ev); + ODPH_ERR("Received unexpected event while waiting for completion\n"); + return -1; + } + + compl = odp_ml_compl_from_event(ev); + CU_ASSERT(compl != ODP_ML_COMPL_INVALID); + + if (load_result) { + CU_ASSERT(odp_ml_compl_load_result(compl, NULL) == 0); + ret = odp_ml_compl_load_result(compl, load_result); + } else { + CU_ASSERT(odp_ml_compl_run_result(compl, NULL) == 0); + ret = odp_ml_compl_run_result(compl, run_result); + } + + CU_ASSERT(ret == 0); + odp_ml_compl_free(compl); + + return ret; +} + +/* Test asynchronous model loading in ODP_ML_COMPL_MODE_EVENT mode */ +static void test_ml_model_load_async_event(void) +{ + int ret; + odp_ml_compl_t compl; + odp_ml_load_result_t result; + odp_ml_compl_param_t compl_param; + int dummy = 6; + void *user_ptr = &dummy; + + compl = odp_ml_compl_alloc(global.compl_pool); + CU_ASSERT_FATAL(compl != ODP_ML_COMPL_INVALID); + + odp_ml_compl_param_init(&compl_param); + compl_param.mode = ODP_ML_COMPL_MODE_EVENT; + compl_param.event = odp_ml_compl_to_event(compl); + compl_param.queue = global.queue; + compl_param.user_ptr = user_ptr; + + ret = odp_ml_model_load_start(global.ml_model, &compl_param); + CU_ASSERT(ret == 0); + + /* Return when odp_ml_model_load_start() failed */ + if (ret) { + odp_ml_compl_free(compl); + ODPH_ERR("ML model odp_ml_model_load_start() failed\n"); + return; + } + + /* Run event scheduler to find the ml completion event and verify it */ + if (get_result_from_ml_compl_event(&result, NULL)) + return; + + CU_ASSERT(result.error_code == 0); + CU_ASSERT(result.user_ptr == user_ptr); + + /* Model load does not modify data in user_ptr */ + if (result.user_ptr) + CU_ASSERT(*(int *)result.user_ptr == dummy); + + compl = odp_ml_compl_alloc(global.compl_pool); + CU_ASSERT(compl != ODP_ML_COMPL_INVALID); + + if (compl == ODP_ML_COMPL_INVALID) + return; + + compl_param.event = odp_ml_compl_to_event(compl); + ret = odp_ml_model_unload_start(global.ml_model, &compl_param); + CU_ASSERT_FATAL(ret == 0); + + /* Run event scheduler to find the ml completion event and verify it */ + if (get_result_from_ml_compl_event(&result, NULL)) + return; + + CU_ASSERT(result.error_code == 0); + CU_ASSERT(result.user_ptr == user_ptr); + + /* odp_ml_model_unload does not modify data in user_ptr */ + if (result.user_ptr) + CU_ASSERT(*(int *)result.user_ptr == dummy); +} + +/* About model batch_add.onnx being tested in this function + * + * Model info: + * Version: 1 + * Inputs: + * inputs[0]: name: x1, type: double, shape: [c, 3] + * inputs[1]: name: x2, type: double, shape: [c, 3] + * Outputs: + * Outputs[0]: name: y, type: double, shape: [c, 3] + * + * The model computes element-wise sum of input tensors x1 and x2 and stores them + * in y. The first dimension of input and output tensors represent batch size, + * thus it must be the same for all tensors here. The dynamic dimension size + * in the output tensor here can be deduced from the given batch size, thus no + * need for the implementation to fill it. + */ +#define NUM_COLUMN 3 +#define MAX_BATCH_SIZE 4 +#define SIZE (NUM_COLUMN * MAX_BATCH_SIZE * sizeof(double)) +static void run_model_batch_add(void) +{ + int ret; + odp_ml_data_t data; + odp_ml_model_t model; + odp_ml_data_seg_t input_segs[SIZE * 2]; + odp_ml_data_seg_t output_segs[SIZE]; + odp_ml_run_result_t result; + odp_ml_run_param_t run_param; + odp_ml_model_param_t model_param; + + double y[12]; + double y_expected[12]; + uint32_t batch_size = MAX_BATCH_SIZE; + double x1[12] = {97, 47, 62, 19, 93, 59, 67, 42, 28, 55, 46, 31}; + double x2[12] = {81, 56, 27, 4, 69, 12, 91, 98, 23, 90, 52, 64}; + + for (int i = 0; i < 12; i++) + y_expected[i] = x1[i] + x2[i]; + + odp_ml_model_param_init(&model_param); + + odp_ml_data_format_t input_format[2] = { + { + .data_type = ODP_ML_DATA_TYPE_FP64, + .data_type_size = 8, + .shape.type = ODP_ML_SHAPE_BATCH, + .shape.num_dim = 2, + .shape.dim = {ODP_ML_DIM_DYNAMIC, NUM_COLUMN}, + .shape.dim_max = {MAX_BATCH_SIZE, NUM_COLUMN} + }, + { + .data_type = ODP_ML_DATA_TYPE_FP64, + .data_type_size = 8, + .shape.type = ODP_ML_SHAPE_BATCH, + .shape.num_dim = 2, + .shape.dim = {ODP_ML_DIM_DYNAMIC, NUM_COLUMN}, + .shape.dim_max = {MAX_BATCH_SIZE, NUM_COLUMN} + } + }; + + model_param.extra_info.num_inputs = 2; + model_param.extra_info.input_format = input_format; + + /* Verify model info about matrix_mul.onnx */ + if (fill_model_param("batch_add.onnx", &model_param)) + return; + + model = odp_ml_model_create("batch_add", &model_param); + free(model_param.model); + CU_ASSERT(model != ODP_ML_MODEL_INVALID); + if (!model) + return; + + if (odp_ml_model_load(model, NULL)) { + CU_ASSERT(odp_ml_model_destroy(model) == 0); + return; + } + + odp_ml_model_print(model); + + /* Prepare parameters for running inference */ + odp_ml_run_param_init(&run_param); + run_param.result = &result; + + data.num_input_seg = 2; + data.input_seg = input_segs; + input_segs[0].addr = x1; + input_segs[1].addr = x2; + + data.num_output_seg = 1; + data.output_seg = output_segs; + output_segs[0].size = sizeof(y); + output_segs[0].addr = y; + + /* Test different batch sizes */ + for (int i = 0; i < MAX_BATCH_SIZE; i++) { + run_param.batch_size = batch_size; + input_segs[0].size = sizeof(double) * NUM_COLUMN * batch_size; + input_segs[1].size = sizeof(double) * NUM_COLUMN * batch_size; + ret = odp_ml_run(model, &data, &run_param); + CU_ASSERT(ret == 1); + if (ret != 1) + goto fail; + + for (uint32_t j = 0; j < batch_size * NUM_COLUMN; j++) + CU_ASSERT(y[j] == y_expected[j]); + + batch_size--; + } + + /* Test also without run results */ + run_param.result = NULL; + ret = odp_ml_run(model, &data, &run_param); + CU_ASSERT(ret == 1); + + /* Test different segment sizes */ + batch_size = MAX_BATCH_SIZE; + odp_ml_run_param_init(&run_param); + run_param.result = &result; + run_param.batch_size = batch_size; + data.input_seg = input_segs; + data.output_seg = output_segs; + + for (int seg_size = SIZE; seg_size > 0; seg_size--) { + int num_seg = (SIZE + seg_size - 1) / seg_size; + + if ((uint32_t)num_seg > global.ml_capa.max_segs_per_input || + (uint32_t)num_seg > global.ml_capa.max_segs_per_output) + break; + + data.num_input_seg = num_seg * 2; + data.num_output_seg = num_seg; + + for (int seg = 0; seg < num_seg; seg++) { + int size = seg_size; + + if (seg == num_seg - 1) + size = SIZE - seg * seg_size; + + input_segs[seg].addr = (char *)x1 + seg * seg_size; + input_segs[seg].size = size; + input_segs[seg + num_seg].addr = (char *)x2 + seg * seg_size; + input_segs[seg + num_seg].size = size; + output_segs[seg].addr = (char *)y + seg * seg_size; + output_segs[seg].size = size; + } + + memset(y, 0, sizeof(y)); + ret = odp_ml_run(model, &data, &run_param); + CU_ASSERT(ret == 1); + if (ret != 1) + goto fail; + + for (uint32_t j = 0; j < batch_size * NUM_COLUMN; j++) + CU_ASSERT(y[j] == y_expected[j]); + } + +fail: + CU_ASSERT_FATAL(odp_ml_model_unload(model, NULL) == 0); + CU_ASSERT(odp_ml_model_destroy(model) == 0); +} + +static void run_global_ml_model(void) +{ + int ret = 0; + odp_ml_run_result_t result; + + ret = odp_ml_model_load(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); + + global.run_param.result = &result; + + ret = odp_ml_run(global.ml_model, &global.data, &global.run_param); + CU_ASSERT(ret == 1); + CU_ASSERT(!result.error_code); + CU_ASSERT(*(int32_t *)global.output_seg.addr == global.y_expected); + + ret = odp_ml_model_unload(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); + global.run_param.result = NULL; +} + +static void test_ml_run(void) +{ + run_global_ml_model(); + run_model_batch_add(); +} + +static void test_ml_run_multi(void) +{ + int ret; + int32_t y; + int32_t x = 8; + int32_t y_expected = 28; + odp_ml_data_t data[RUN_NUM]; + odp_ml_data_seg_t input_seg; + odp_ml_data_seg_t output_seg; + odp_ml_run_param_t param[RUN_NUM]; + odp_ml_run_result_t result[RUN_NUM]; + uint64_t wait_ns = 500 * ODP_TIME_MSEC_IN_NS; + + ret = odp_ml_model_load(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); + + param[0] = global.run_param; + param[0].result = &result[0]; + odp_ml_run_param_init(¶m[1]); + param[1].result = &result[1]; + + /* Prepare data for running model inference */ + data[0] = global.data; + data[1].num_input_seg = NUM_INPUTS; + data[1].input_seg = &input_seg; + input_seg.size = sizeof(int32_t); + input_seg.addr = &x; + + data[1].num_output_seg = NUM_OUTPUTS; + data[1].output_seg = &output_seg; + output_seg.size = sizeof(int32_t); + output_seg.addr = &y; + + int num_completed = 0; + + for (int i = 0; i < TIMEOUT; i++) { + ret = odp_ml_run_multi(global.ml_model, data + num_completed, param + num_completed, + RUN_NUM - num_completed); + CU_ASSERT(ret >= 0); + if (ret < 0) + break; + + num_completed += ret; + + if (num_completed >= RUN_NUM) + break; + + odp_time_wait_ns(wait_ns); + } + + CU_ASSERT(num_completed == RUN_NUM); + CU_ASSERT(!result[0].error_code); + CU_ASSERT(!result[1].error_code); + CU_ASSERT(*(int32_t *)global.output_seg.addr == global.y_expected); + CU_ASSERT(*(int32_t *)output_seg.addr == y_expected); + + ret = odp_ml_model_unload(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); +} + +/* Test asynchronous inference running in ODP_ML_COMPL_MODE_EVENT mode */ +static void test_ml_model_run_async_event(void) +{ + int ret; + void *user_ptr; + odp_ml_compl_t compl; + odp_ml_run_result_t result; + odp_ml_data_seg_t *outputs; + odp_ml_compl_param_t compl_param; + + /* Load model in order to run inference */ + ret = odp_ml_model_load(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); + + compl = odp_ml_compl_alloc(global.compl_pool); + CU_ASSERT_FATAL(compl != ODP_ML_COMPL_INVALID); + + odp_ml_compl_param_init(&compl_param); + compl_param.mode = ODP_ML_COMPL_MODE_EVENT; + compl_param.event = odp_ml_compl_to_event(compl); + compl_param.queue = global.queue; + + /* user_ptr structure maintains the output data pointer for output retrieval */ + user_ptr = &global.output_seg; + compl_param.user_ptr = user_ptr; + + memset(global.output_seg.addr, 0, global.output_seg.size); + ret = odp_ml_run_start(global.ml_model, &global.data, &compl_param, NULL); + CU_ASSERT_FATAL(ret == 1); + + /* Run event scheduler to find the ml completion event and verify it */ + if (get_result_from_ml_compl_event(NULL, &result)) + return; + + CU_ASSERT(!result.error_code); + CU_ASSERT(result.user_ptr == user_ptr); + + outputs = (odp_ml_data_seg_t *)result.user_ptr; + CU_ASSERT(*(int32_t *)outputs[0].addr == global.y_expected); + + /* Unload model */ + ret = odp_ml_model_unload(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); +} + +/* Test asynchronous inference running in ODP_ML_COMPL_MODE_POLL mode */ +static void test_ml_model_run_async_poll(void) +{ + int ret; + void *user_ptr; + odp_ml_run_result_t result; + odp_ml_data_seg_t *outputs; + odp_ml_compl_param_t compl_param; + uint64_t wait_ns = 500 * ODP_TIME_MSEC_IN_NS; + + memset(&result, 0, sizeof(result)); + /* Load model in order to run inference */ + ret = odp_ml_model_load(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); + + odp_ml_compl_param_init(&compl_param); + compl_param.mode = ODP_ML_COMPL_MODE_POLL; + compl_param.compl_id = 0; + + /* user_ptr structure maintains the output data pointer for output retrieval */ + user_ptr = &global.output_seg; + compl_param.user_ptr = user_ptr; + + memset(global.output_seg.addr, 0, global.output_seg.size); + ret = odp_ml_run_start(global.ml_model, &global.data, &compl_param, NULL); + CU_ASSERT_FATAL(ret == 1); + + /* When odp_ml_run_start() succeeded, continue to check completion status */ + for (int i = 0; i < TIMEOUT; i++) { + ret = odp_ml_run_status(global.ml_model, 0, &result); + if (ret) + break; + + /* ret = 0 meaning run has not finished, continue to check status */ + odp_time_wait_ns(wait_ns); + } + + outputs = (odp_ml_data_seg_t *)result.user_ptr; + + CU_ASSERT(ret > 0); + CU_ASSERT(!result.error_code); + CU_ASSERT(result.user_ptr == user_ptr); + CU_ASSERT(*(int32_t *)outputs[0].addr == global.y_expected); + + /* Unload model */ + ret = odp_ml_model_unload(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); +} + +static void test_ml_run_start_multi(void) +{ + int ret; + int32_t y; + odp_ml_compl_t compl; + odp_ml_data_t data[RUN_NUM]; + odp_ml_data_seg_t input_seg; + odp_ml_data_seg_t output_seg; + odp_ml_data_seg_t *outputs[RUN_NUM]; + odp_ml_compl_param_t compl_param[RUN_NUM]; + odp_ml_run_result_t run_result[RUN_NUM]; + int32_t x = 5; + int32_t y_expected = 19; + uint64_t wait_ns = 500 * ODP_TIME_MSEC_IN_NS; + + /* Load model in order to run inference */ + ret = odp_ml_model_load(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); + + compl = odp_ml_compl_alloc(global.compl_pool); + CU_ASSERT_FATAL(compl != ODP_ML_COMPL_INVALID); + + /* Prepare data for running model inference */ + data[0] = global.data; + + data[1].num_input_seg = NUM_INPUTS; + data[1].input_seg = &input_seg; + input_seg.size = sizeof(int32_t); + input_seg.addr = &x; + + data[1].num_output_seg = NUM_OUTPUTS; + data[1].output_seg = &output_seg; + output_seg.size = sizeof(int32_t); + output_seg.addr = &y; + + /* Two completion parameters: one use event mode, another poll mode */ + odp_ml_compl_param_init(&compl_param[0]); + compl_param[0].mode = ODP_ML_COMPL_MODE_EVENT; + compl_param[0].event = odp_ml_compl_to_event(compl); + compl_param[0].queue = global.queue; + /* user_ptr structure maintains the output data pointer for output retrieval */ + compl_param[0].user_ptr = &global.output_seg; + + odp_ml_compl_param_init(&compl_param[1]); + compl_param[1].mode = ODP_ML_COMPL_MODE_POLL; + compl_param[1].compl_id = 0; + /* user_ptr structure maintains the output data pointer for output retrieval */ + compl_param[1].user_ptr = &output_seg; + + memset(global.output_seg.addr, 0, sizeof(int32_t)); + + int num_completed = 0; + + for (int i = 0; i < TIMEOUT; i++) { + ret = odp_ml_run_start_multi(global.ml_model, data + num_completed, + compl_param + num_completed, NULL, + RUN_NUM - num_completed); + CU_ASSERT(ret >= 0); + if (ret < 0) + break; + + num_completed += ret; + + if (num_completed >= RUN_NUM) + break; + + odp_time_wait_ns(wait_ns); + } + + CU_ASSERT(num_completed == RUN_NUM); + + /* Run event scheduler to find the ml completion event and verify it */ + if (get_result_from_ml_compl_event(NULL, &run_result[0])) { + ret = odp_ml_model_unload(global.ml_model, NULL); + return; + } + + CU_ASSERT(!run_result[0].error_code); + CU_ASSERT(run_result[0].user_ptr == &global.output_seg); + outputs[0] = (odp_ml_data_seg_t *)run_result[0].user_ptr; + CU_ASSERT(*(int32_t *)outputs[0][0].addr == global.y_expected); + + /* Check completion status for the poll mode */ + for (int i = 0; i < TIMEOUT; i++) { + ret = odp_ml_run_status(global.ml_model, 0, &run_result[1]); + if (ret) + break; + + /* ret = 0 meaning run has not finished, continue to check status */ + odp_time_wait_ns(wait_ns); + } + + outputs[1] = (odp_ml_data_seg_t *)run_result[1].user_ptr; + CU_ASSERT(ret > 0); + CU_ASSERT(!run_result[1].error_code); + CU_ASSERT(run_result[1].user_ptr == &output_seg); + CU_ASSERT(*(int32_t *)outputs[1][0].addr == y_expected); + + /* Unload model */ + ret = odp_ml_model_unload(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); +} + +static void test_ml_model_extra_stat_info(void) +{ + int ret; + + ret = odp_ml_model_extra_stat_info(global.ml_model, NULL, 0); + CU_ASSERT(ret >= 0); +} + +static void test_ml_model_extra_stats(void) +{ + int ret; + + ret = odp_ml_model_extra_stats(global.ml_model, NULL, 0); + CU_ASSERT(ret >= 0); +} + +odp_testinfo_t ml_suite[] = { + ODP_TEST_INFO_CONDITIONAL(test_ml_debug, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_create, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_lookup, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_info, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_load, check_load_sync), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_load_async_poll, check_load_poll), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_load_async_event, check_load_event), + /* Synchronous load/unload is used load/unload model before/after model run */ + ODP_TEST_INFO_CONDITIONAL(test_ml_run, check_run_sync), + ODP_TEST_INFO_CONDITIONAL(test_ml_run_multi, check_run_sync), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_run_async_event, check_run_event), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_run_async_poll, check_run_poll), + ODP_TEST_INFO_CONDITIONAL(test_ml_run_start_multi, check_run_poll_event), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_extra_stat_info, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_extra_stats, check_ml_support), + ODP_TEST_INFO_NULL +}; + +odp_suiteinfo_t ml_suites[] = { + {"ML", ml_suite_init, ml_suite_term, ml_suite}, + ODP_SUITE_INFO_NULL +}; + +int main(int argc, char *argv[]) +{ + int ret; + + /* parse common options: */ + if (odp_cunit_parse_options(&argc, argv)) + return -1; + + ret = odp_cunit_register(ml_suites); + + if (ret == 0) + ret = odp_cunit_run(); + + return ret; +} diff --git a/platform/linux-generic/test/validation/api/ml/requirements.txt b/platform/linux-generic/test/validation/api/ml/requirements.txt new file mode 100644 index 000000000..2dcba7a3a --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/requirements.txt @@ -0,0 +1,2 @@ +onnx +numpy diff --git a/platform/linux-generic/test/validation/api/ml/simple_linear.onnx b/platform/linux-generic/test/validation/api/ml/simple_linear.onnx new file mode 100644 index 0000000000000000000000000000000000000000..45c4b95b9491f3817b1c63bca1a5e529d61f7df8 GIT binary patch literal 214 zcmd<!5aRU9&(2I#@OKH&U^L>YRIpVru29fcFtOUi$fYa9ST4m@o|>7SQKH1*oS#=* zl9*SbCC0^CmY7qTs>Q{@D8Q!0$n3;ezJQU1ON5KDf{U?Sh^f3nirKd`M+ro5X$vtX zNwFklCKlr{g2jn331S3LGZ$l$5M!kjvtvq%5{M9D%gM}3O)QGy2U;M+#l^wFCd9(U T#KD*Z6sSZNaAM(N5a0y>4~#0v literal 0 HcmV?d00001 diff --git a/platform/linux-generic/test/validation/api/ml/simple_linear_gen.py b/platform/linux-generic/test/validation/api/ml/simple_linear_gen.py new file mode 100644 index 000000000..b3e6124cd --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/simple_linear_gen.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Nokia +# + +import onnx +from onnx import helper +from onnx import TensorProto + +weight = helper.make_tensor(name='w', data_type=TensorProto.INT32, dims=[1], vals=[3]) +w = helper.make_node('Constant', inputs=[], outputs=['w'], name='weight', value=weight) + +bias = helper.make_tensor(name='b', data_type=TensorProto.INT32, dims=[1], vals=[4]) +b = helper.make_node('Constant', inputs=[], outputs=['b'], name='bias', value=bias) + +# The functional nodes: +mul = helper.make_node('Mul', inputs=['x', 'w'], outputs=['wx'], name='Mul') +add = helper.make_node('Add', inputs=['wx', 'b'], outputs=['y'], name='Add') + +# Create the graph +g = helper.make_graph([w, mul, b, add], 'linear', + [helper.make_tensor_value_info('x', TensorProto.INT32, [1])], + [helper.make_tensor_value_info('y', TensorProto.INT32, [1])] +) + +model = helper.make_model( + producer_name='ODP validation tests', + model_version=1, + doc_string="y = 3x + 4", + graph=g, + opset_imports=[helper.make_opsetid("", 13)] +) + +# Save the model +onnx.save(model, 'simple_linear.onnx') From 5d1df18874b2d87e5973e3ecacbb54caf4407bbe Mon Sep 17 00:00:00 2001 From: Lifang Zhang <lifang.zhang@nokia.com> Date: Fri, 22 Dec 2023 16:38:10 +0200 Subject: [PATCH 121/147] example: sysinfo: print ML capabilities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Print ML capabilities in sysinfo example. Signed-off-by: Lifang Zhang <lifang.zhang@nokia.com> Reviewed-by: Jere Leppänen <jere.leppanen@nokia.com> Reviewed-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- example/sysinfo/odp_sysinfo.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/example/sysinfo/odp_sysinfo.c b/example/sysinfo/odp_sysinfo.c index 11b789254..df33f45cf 100644 --- a/example/sysinfo/odp_sysinfo.c +++ b/example/sysinfo/odp_sysinfo.c @@ -746,6 +746,7 @@ int main(int argc, char **argv) odp_ipsec_capability_t ipsec_capa; odp_schedule_capability_t schedule_capa; odp_stash_capability_t stash_capa; + odp_ml_capability_t ml_capa; appl_args_t appl_args; uint64_t huge_page[MAX_HUGE_PAGES]; char ava_mask_str[ODP_CPUMASK_STR_SIZE]; @@ -861,6 +862,11 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } + if (odp_ml_capability(&ml_capa)) { + ODPH_ERR("ml capability failed\n"); + exit(EXIT_FAILURE); + } + crypto_ret = odp_crypto_capability(&crypto_capa); if (crypto_ret < 0) ODPH_ERR("crypto capability failed\n"); @@ -1073,6 +1079,33 @@ int main(int argc, char **argv) printf(" max_put_batch: %u\n", stash_capa.max_put_batch); printf(" stats: 0x%" PRIx64 "\n", stash_capa.stats.all); + printf("\n"); + printf(" ML\n"); + printf(" max_models: %u\n", ml_capa.max_models); + printf(" max_models_loaded: %u\n", ml_capa.max_models_loaded); + printf(" max_model_size: %" PRIu64 "B\n", ml_capa.max_model_size); + printf(" max_compl_id: %u\n", ml_capa.max_compl_id); + printf(" max_inputs: %u\n", ml_capa.max_inputs); + printf(" max_outputs: %u\n", ml_capa.max_outputs); + printf(" max_segs_per_input: %u\n", ml_capa.max_segs_per_input); + printf(" max_segs_per_output: %u\n", ml_capa.max_segs_per_output); + printf(" min_input_align: %u\n", ml_capa.min_input_align); + printf(" min_output_align: %u\n", ml_capa.min_output_align); + printf(" packed_input_data: %u\n", ml_capa.packed_input_data); + printf(" packed_output_data: %u\n", ml_capa.packed_output_data); + printf(" load.compl_mode_mask: 0x%x\n", ml_capa.load.compl_mode_mask); + printf(" load.compl_queue_plain: %i\n", ml_capa.load.compl_queue_plain); + printf(" load.compl_queue_sched: %i\n", ml_capa.load.compl_queue_sched); + printf(" run.compl_mode_mask: 0x%x\n", ml_capa.run.compl_mode_mask); + printf(" run.compl_queue_plain: %i\n", ml_capa.run.compl_queue_plain); + printf(" run.compl_queue_sched: %i\n", ml_capa.run.compl_queue_sched); + printf(" pool.max_pools: %u\n", ml_capa.pool.max_pools); + printf(" pool.max_num: %u\n", ml_capa.pool.max_num); + printf(" pool.max_uarea_size: %u B\n", ml_capa.pool.max_uarea_size); + printf(" pool.uarea_persistence: %u\n", ml_capa.pool.uarea_persistence); + printf(" pool.min_cache_size: %u\n", ml_capa.pool.min_cache_size); + printf(" pool.max_cache_size: %u\n", ml_capa.pool.max_cache_size); + print_timer_capa(&appl_args); if (crypto_ret == 0) { From 472da6b66b3c4cc49bc075cd201b09a3a65e3ef2 Mon Sep 17 00:00:00 2001 From: Lifang Zhang <lifang.zhang@nokia.com> Date: Sun, 3 Dec 2023 04:15:37 +0200 Subject: [PATCH 122/147] checkpatch: skip checking onnxruntime Camelcase use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skipped checking onnxruntime Camelcase use. Signed-off-by: Lifang Zhang <lifang.zhang@nokia.com> Reviewed-by: Jere Leppänen <jere.leppanen@nokia.com> Reviewed-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- scripts/checkpatch.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 9e6d8d750..71af1c682 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -5855,6 +5855,9 @@ sub process { $var !~ /\bSCN[diux]16/ && $var !~ /\bSCN[diux]32/ && $var !~ /\bSCN[diux]64/ && +#Ignore onnxruntime CamelCase usage + $var !~ /\bOrt*/ && + $var !~ /\bort_api->*/ && #Ignore ETHTOOL_LINK_MODE_<foo> variants $var !~ /^ETHTOOL_LINK_MODE_/ && #Ignore SI style variants like nS, mV and dB From 7abbd0d823091bfc52ec0345fb8fc67f68cc8c80 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Wed, 21 Feb 2024 09:36:37 +0200 Subject: [PATCH 123/147] api: increment ODP API version to 1.44.0 Increment API version number to reflect the following changes: Backward compatible: - ml: add new API module for machine learning offload Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> --- CHANGELOG | 37 +++++++++++++++++++++++++++++++++++++ configure.ac | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index dd3d04048..e9d953a92 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,40 @@ +== OpenDataPlane (1.44.0.0) + +=== Backward compatible API changes +==== ML +* Add new API module for Machine Learning offload. + +=== Implementation +==== ML +* Implement the API using ONNX Runtime. A non-standard ONNX install path can be +passed with `--with-ort-path` configure option. See `DEPENDENCIES` for +additional information. + +=== Example Applications +==== debug +* Print packet IO link information. +* Add packet IO interface name option (`-i`). + +==== packet_dump +* Add support for measuring packet input delay. + +=== Performance Tests +==== l2fwd +* Add packet input timestamping option (`-T`). +* Add packet data prefetch option (`-F`). +* Add packet data read option (`-R`). +* Add packet print option (`-V`). + +==== pool_latency +* New pool latency tester application for profiling pool allocation and free +latencies under different load patterns. + +==== sched_perf +* Add options to read (`-u`) and modify (`-U`) event user area contents per +event reception. +* Run application indefinitely when the number of events to schedule option +(`-s`) is set to zero. + == OpenDataPlane (1.43.0.0) === Backward incompatible API changes diff --git a/configure.ac b/configure.ac index da1dbc491..f0b2c9952 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,7 @@ AC_PREREQ([2.5]) # ODP API version ########################################################################## m4_define([odp_version_generation], [1]) -m4_define([odp_version_major], [43]) +m4_define([odp_version_major], [44]) m4_define([odp_version_minor], [0]) m4_define([odp_version_patch], [0]) From 041086966f077b56a162c16c730382e13d7882d1 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Fri, 1 Mar 2024 14:45:41 +0200 Subject: [PATCH 124/147] linux-gen: ml: add missing includes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add missing includes to ML implementation and sort the existing ones. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Jere Leppänen <jere.leppanen@nokia.com> --- platform/linux-generic/include/odp_ml_fp16.h | 2 ++ platform/linux-generic/odp_ml.c | 23 +++++++++++++++----- platform/linux-generic/odp_ml_fp16.c | 4 ++-- platform/linux-generic/odp_ml_null.c | 3 +++ platform/linux-generic/odp_ml_quantize.c | 8 +++---- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/platform/linux-generic/include/odp_ml_fp16.h b/platform/linux-generic/include/odp_ml_fp16.h index 5294a7c0b..476028cb4 100644 --- a/platform/linux-generic/include/odp_ml_fp16.h +++ b/platform/linux-generic/include/odp_ml_fp16.h @@ -9,6 +9,8 @@ extern "C" { #endif +#include <stdint.h> + uint16_t _odp_float32_to_float16(float x); float _odp_float16_to_float32(uint16_t f16); uint16_t _odp_float32_to_bfloat16(float x); diff --git a/platform/linux-generic/odp_ml.c b/platform/linux-generic/odp_ml.c index fda06e7cb..6ab9e7177 100644 --- a/platform/linux-generic/odp_ml.c +++ b/platform/linux-generic/odp_ml.c @@ -2,23 +2,36 @@ * Copyright (c) 2023 Nokia */ +#include <odp/autoheader_external.h> + +#include <odp/api/atomic.h> +#include <odp/api/buffer.h> +#include <odp/api/event.h> +#include <odp/api/hints.h> #include <odp/api/ml.h> +#include <odp/api/pool.h> #include <odp/api/queue.h> +#include <odp/api/shared_memory.h> +#include <odp/api/std_types.h> +#include <odp/api/ticketlock.h> + #include <odp/api/plat/event_inline_types.h> +#include <odp/api/plat/strong_types.h> -#include <odp_global_data.h> +#include <odp_buffer_internal.h> +#include <odp_config_internal.h> #include <odp_debug_internal.h> +#include <odp_global_data.h> #include <odp_init_internal.h> -#include <odp_pool_internal.h> -#include <odp_config_internal.h> -#include <odp_macros_internal.h> #include <odp_libconfig_internal.h> +#include <odp_macros_internal.h> +#include <odp_pool_internal.h> #include <onnxruntime_c_api.h> +#include <inttypes.h> #include <stdint.h> #include <stdio.h> -#include <inttypes.h> #include <string.h> #define ML_MAX_IO_SEGS UINT32_MAX diff --git a/platform/linux-generic/odp_ml_fp16.c b/platform/linux-generic/odp_ml_fp16.c index f135f8b5a..47b10f841 100644 --- a/platform/linux-generic/odp_ml_fp16.c +++ b/platform/linux-generic/odp_ml_fp16.c @@ -8,11 +8,11 @@ * - dpdk/lib/mldev/mldev_utils_scalar_bfloat16.c */ +#include <odp_ml_fp16.h> + #include <errno.h> #include <stdint.h> -#include <odp_ml_fp16.h> - #ifndef BIT #define BIT(nr) (1UL << (nr)) #endif diff --git a/platform/linux-generic/odp_ml_null.c b/platform/linux-generic/odp_ml_null.c index faf431997..718e80d76 100644 --- a/platform/linux-generic/odp_ml_null.c +++ b/platform/linux-generic/odp_ml_null.c @@ -2,9 +2,12 @@ * Copyright (c) 2023 Nokia */ +#include <odp/api/hints.h> #include <odp/api/ml.h> + #include <odp_init_internal.h> +#include <stdint.h> #include <string.h> /* Dummy ML API implementation, no capability and just return error for diff --git a/platform/linux-generic/odp_ml_quantize.c b/platform/linux-generic/odp_ml_quantize.c index 0678f15ef..d3f3601e3 100644 --- a/platform/linux-generic/odp_ml_quantize.c +++ b/platform/linux-generic/odp_ml_quantize.c @@ -2,15 +2,15 @@ * Copyright (c) 2023 Nokia */ -#include <stdint.h> -#include <math.h> - #include <odp/api/ml_quantize.h> -#include <odp_macros_internal.h> #include <odp_debug_internal.h> +#include <odp_macros_internal.h> #include <odp_ml_fp16.h> +#include <math.h> +#include <stdint.h> + void odp_ml_fp32_to_uint8(uint8_t *u8, const float *fp32, uint32_t num, float scale, uint8_t zerop) { From 64afe6531fc98a83aa6583a9c302d7cd2843dc8d Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Mon, 4 Mar 2024 10:01:28 +0200 Subject: [PATCH 125/147] linux-gen: example: simple_linear: fix odp_ml_run() return value check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Handle odp_ml_run() busy return value. Resources may be busy if the application is run with multiple workers. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Jere Leppänen <jere.leppanen@nokia.com> --- platform/linux-generic/example/ml/simple_linear.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/platform/linux-generic/example/ml/simple_linear.c b/platform/linux-generic/example/ml/simple_linear.c index 142dbe718..3417219c7 100644 --- a/platform/linux-generic/example/ml/simple_linear.c +++ b/platform/linux-generic/example/ml/simple_linear.c @@ -59,9 +59,16 @@ static int run_inference(void *infer_param) output.addr = &y; output.size = sizeof(int32_t); - if (odp_ml_run(param->ml_model, &data, NULL) != 1) { - ODPH_ERR("odp_ml_model_run() failed\n"); - return -1; + while (1) { + int ret = odp_ml_run(param->ml_model, &data, NULL); + + if (ret == 1) + break; + + if (ret < 0) { + ODPH_ERR("odp_ml_model_run() failed: %d\n", ret); + return -1; + } } printf("y = 3 * %d + 4: %d\n", param->x, y); From 946d7abef9467c67a1d1fc8bafd4336813624526 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Wed, 6 Mar 2024 14:41:38 +0200 Subject: [PATCH 126/147] github_ci: update checkout github action Update arm64 GitHub checkout action into the latest release version. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com> --- .github/workflows/ci-pipeline-arm64.yml | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml index 5b4f7e13c..780fc00cc 100644 --- a/.github/workflows/ci-pipeline-arm64.yml +++ b/.github/workflows/ci-pipeline-arm64.yml @@ -28,7 +28,7 @@ jobs: '--enable-pcapng-support'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=gcc -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -51,7 +51,7 @@ jobs: '--enable-wfe-locks'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=clang -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -70,7 +70,7 @@ jobs: conf: ['', '--enable-lto'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="gcc-${{matrix.cc_ver}}" -e CXX="g++-${{matrix.cc_ver}}" -e CONF="${CONF} ${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_static.sh - if: ${{ failure() }} @@ -86,7 +86,7 @@ jobs: os: ['rocky_linux_8'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -104,7 +104,7 @@ jobs: conf: ['', '--enable-abi-compat'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="gcc-${{matrix.cc_ver}}" -e CXX="g++-${{matrix.cc_ver}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -115,7 +115,7 @@ jobs: runs-on: [self-hosted, ARM64] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/out_of_tree.sh - if: ${{ failure() }} @@ -130,7 +130,7 @@ jobs: conf: ['--enable-user-guides', '--enable-user-guides --enable-abi-compat'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Ignore distcheck failure (caused by the first 'make check' run unmounting huge pages) - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/distcheck.sh || true @@ -153,7 +153,7 @@ jobs: '--enable-wfe-locks'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=gcc -e ARCH="${ARCH}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -175,7 +175,7 @@ jobs: '--disable-host-optimization --enable-abi-compat'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=clang -e ARCH="${ARCH}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -191,7 +191,7 @@ jobs: cflags: ['-march=armv8.2-a -O2', '-march=armv8.2-a+crypto -O2'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e ARCH="${ARCH}" -e CXX=g++-10 -e CFLAGS="${{matrix.cflags}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -206,7 +206,7 @@ jobs: os: ['ubuntu_22.04'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -217,7 +217,7 @@ jobs: runs-on: [self-hosted, ARM64] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/sched-basic.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -228,7 +228,7 @@ jobs: runs-on: [self-hosted, ARM64] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/stash-custom.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -239,7 +239,7 @@ jobs: runs-on: [self-hosted, ARM64] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_SCHEDULER=sp $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -250,7 +250,7 @@ jobs: runs-on: [self-hosted, ARM64] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/process-mode.conf -e ODPH_PROC_MODE=1 $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh @@ -262,7 +262,7 @@ jobs: runs-on: [self-hosted, ARM64] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native-dpdk_21.11 /odp/scripts/ci/check.sh - if: ${{ failure() }} From d7d6f1154a4514b9cd693845d04a02825a788ccb Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Thu, 29 Feb 2024 10:22:26 +0200 Subject: [PATCH 127/147] Port ac147a12f "linux-gen: fix doxygen documentation grouping" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/include-abi/odp/api/abi/dma_types.h | 2 +- platform/linux-dpdk/include-abi/odp/api/abi/packet_types.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/dma_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/dma_types.h index 3c009f4c4..318c2c385 100644 --- a/platform/linux-dpdk/include-abi/odp/api/abi/dma_types.h +++ b/platform/linux-dpdk/include-abi/odp/api/abi/dma_types.h @@ -11,7 +11,7 @@ extern "C" { #include <odp/api/plat/strong_types.h> -/** @ingroup odp_dma +/** @addtogroup odp_dma * @{ */ diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/packet_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/packet_types.h index 1f5f9e6f7..9ca66db54 100644 --- a/platform/linux-dpdk/include-abi/odp/api/abi/packet_types.h +++ b/platform/linux-dpdk/include-abi/odp/api/abi/packet_types.h @@ -21,7 +21,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/plat/strong_types.h> -/** @ingroup odp_packet +/** @addtogroup odp_packet * @{ */ From df2b373200e1d65d30e2d840e80e4f224a6b09a2 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Thu, 29 Feb 2024 15:27:39 +0200 Subject: [PATCH 128/147] Port dfec7b832 "linux-gen: event: initialize subtype always" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/odp_pool.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index f7726f97b..0d62b8f1d 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -626,6 +626,7 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi event_hdr->hdr.pool = _odp_pool_handle(pool); event_hdr->hdr.type = priv_data->type; event_hdr->hdr.event_type = priv_data->event_type; + event_hdr->hdr.subtype = ODP_EVENT_NO_SUBTYPE; switch (priv_data->type) { case ODP_POOL_BUFFER: @@ -1361,6 +1362,7 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned event_hdr->hdr.pool = _odp_pool_handle(pool); event_hdr->hdr.type = mb_ctor_arg->type; event_hdr->hdr.event_type = mb_ctor_arg->event_type; + event_hdr->hdr.subtype = ODP_EVENT_NO_SUBTYPE; switch (mb_ctor_arg->type) { case ODP_POOL_BUFFER: From 889080035df939b09c447bb296af49baead168e7 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Thu, 29 Feb 2024 16:07:57 +0200 Subject: [PATCH 129/147] Port 187680b44 "linux-gen: event: move subtype into event header" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- .../linux-dpdk/include/odp/api/plat/event_inline_types.h | 1 + .../linux-dpdk/include/odp/api/plat/packet_inline_types.h | 1 - platform/linux-dpdk/include/odp/api/plat/packet_inlines.h | 4 +++- platform/linux-dpdk/include/odp_packet_internal.h | 8 +++++--- platform/linux-dpdk/odp_event.c | 1 + platform/linux-dpdk/odp_packet.c | 1 - 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h index ee5490ff1..94a95a889 100644 --- a/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h +++ b/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h @@ -28,6 +28,7 @@ extern "C" { typedef struct _odp_event_inline_offset_t { uint16_t event_type; uint16_t base_data; + uint16_t subtype; uint16_t flow_id; uint16_t pool; uint16_t buf_len; diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h index 5bbcadd32..255db9d78 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h @@ -41,7 +41,6 @@ typedef struct _odp_packet_inline_offset_t { uint16_t timestamp; uint16_t input_flags; uint16_t flags; - uint16_t subtype; uint16_t cls_mark; uint16_t ipsec_ctx; uint16_t crypto_op; diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h index 7b2764a05..b41a272ef 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h @@ -25,6 +25,7 @@ extern "C" { #include <odp/api/time_types.h> #include <odp/api/plat/debug_inlines.h> +#include <odp/api/plat/event_inline_types.h> #include <odp/api/plat/event_validation_external.h> #include <odp/api/plat/packet_io_inlines.h> #include <odp/api/plat/packet_inline_types.h> @@ -647,7 +648,8 @@ _ODP_INLINE void odp_packet_to_event_multi(const odp_packet_t pkt[], _ODP_INLINE odp_event_subtype_t odp_packet_subtype(odp_packet_t pkt) { - return (odp_event_subtype_t)_odp_pkt_get(pkt, int8_t, subtype); + return (odp_event_subtype_t)_odp_event_hdr_field((odp_event_t)(uintptr_t)pkt, + int8_t, subtype); } _ODP_INLINE odp_packet_tx_compl_t odp_packet_tx_compl_from_event(odp_event_t ev) diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h index c86f0646a..cae77245a 100644 --- a/platform/linux-dpdk/include/odp_packet_internal.h +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -218,9 +218,9 @@ static inline struct rte_mbuf *pkt_to_mbuf(odp_packet_t pkt) return (struct rte_mbuf *)(uintptr_t)pkt; } -static inline void packet_subtype_set(odp_packet_t pkt, int ev) +static inline void packet_subtype_set(odp_packet_t pkt, int subtype) { - packet_hdr(pkt)->event_hdr.subtype = ev; + packet_hdr(pkt)->event_hdr.subtype = subtype; } /** @@ -236,7 +236,9 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, odp_pktio_t input) pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; - pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC; + if (odp_unlikely(pkt_hdr->event_hdr.subtype != ODP_EVENT_PACKET_BASIC)) + pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC; + pkt_hdr->input = input; } diff --git a/platform/linux-dpdk/odp_event.c b/platform/linux-dpdk/odp_event.c index dff3e2ed2..dca743029 100644 --- a/platform/linux-dpdk/odp_event.c +++ b/platform/linux-dpdk/odp_event.c @@ -36,6 +36,7 @@ const _odp_event_inline_offset_t _odp_event_inline_offset ODP_ALIGNED_CACHE = { .event_type = offsetof(_odp_event_hdr_t, hdr.event_type), .base_data = offsetof(_odp_event_hdr_t, mb.buf_addr), + .subtype = offsetof(_odp_event_hdr_t, hdr.subtype), .flow_id = offsetof(_odp_event_hdr_t, hdr.flow_id), .pool = offsetof(_odp_event_hdr_t, hdr.pool), .buf_len = offsetof(_odp_event_hdr_t, mb.buf_len) diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index a61c4c34a..bcd2c2fb3 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -64,7 +64,6 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .timestamp = offsetof(odp_packet_hdr_t, timestamp), .input_flags = offsetof(odp_packet_hdr_t, p.input_flags), .flags = offsetof(odp_packet_hdr_t, p.flags), - .subtype = offsetof(odp_packet_hdr_t, event_hdr.subtype), .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), .ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx), .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result), From d3d7a49a448e1f7ab284237cb8a82201be903362 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Thu, 29 Feb 2024 15:34:43 +0200 Subject: [PATCH 130/147] Port 10753512c "linux-gen: pool: add buffer_subtype_set()" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/include/odp_buffer_internal.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h index dc65fd17d..cb7f50073 100644 --- a/platform/linux-dpdk/include/odp_buffer_internal.h +++ b/platform/linux-dpdk/include/odp_buffer_internal.h @@ -73,6 +73,13 @@ static inline odp_buffer_hdr_t *_odp_buf_hdr(odp_buffer_t buf) return (odp_buffer_hdr_t *)(uintptr_t)buf; } +static inline void _odp_buffer_subtype_set(odp_buffer_t buf, int subtype) +{ + odp_buffer_hdr_t *buf_hdr = _odp_buf_hdr(buf); + + buf_hdr->event_hdr.subtype = subtype; +} + #ifdef __cplusplus } #endif From 426ff7ab4c88cfbd311c367c8bb40de1d90ebb25 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Thu, 29 Feb 2024 15:32:20 +0200 Subject: [PATCH 131/147] Port 45f617f44 "api: ml: add machine learning API" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/Makefile.am | 1 + platform/linux-dpdk/include-abi/odp/api/abi/ml_types.h | 1 + 2 files changed, 2 insertions(+) create mode 120000 platform/linux-dpdk/include-abi/odp/api/abi/ml_types.h diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 886fe8f48..c04979c45 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -88,6 +88,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/init.h \ include-abi/odp/api/abi/ipsec.h \ include-abi/odp/api/abi/ipsec_types.h \ + include-abi/odp/api/abi/ml_types.h \ include-abi/odp/api/abi/packet.h \ include-abi/odp/api/abi/packet_types.h \ include-abi/odp/api/abi/packet_flags.h \ diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/ml_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/ml_types.h new file mode 120000 index 000000000..18b483da1 --- /dev/null +++ b/platform/linux-dpdk/include-abi/odp/api/abi/ml_types.h @@ -0,0 +1 @@ +../../../../../linux-generic/include-abi/odp/api/abi/ml_types.h \ No newline at end of file From e23b1d04fb1e40afc667bec47fdcbe50422bff5b Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Thu, 29 Feb 2024 15:56:00 +0200 Subject: [PATCH 132/147] Port b04851ad2 "linux-gen: ml: implement the new ML API" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- config/odp-linux-dpdk.conf | 33 ++++++++++++++++++- platform/linux-dpdk/Makefile.am | 13 ++++++++ .../linux-dpdk/include/odp_config_internal.h | 9 +++++ platform/linux-dpdk/libodp-dpdk.pc.in | 2 +- platform/linux-dpdk/m4/configure.m4 | 4 ++- platform/linux-dpdk/m4/odp_libconfig.m4 | 2 +- platform/linux-dpdk/m4/odp_ml.m4 | 1 + platform/linux-dpdk/odp_event.c | 6 ++++ platform/linux-dpdk/odp_init.c | 15 +++++++++ platform/linux-dpdk/odp_pool.c | 8 +++++ platform/linux-dpdk/odp_system_info.c | 3 ++ platform/linux-dpdk/test/crypto.conf | 2 +- platform/linux-dpdk/test/default-timer.conf | 2 +- platform/linux-dpdk/test/process-mode.conf | 2 +- platform/linux-dpdk/test/sched-basic.conf | 2 +- platform/linux-dpdk/test/stash-custom.conf | 2 +- 16 files changed, 97 insertions(+), 9 deletions(-) create mode 120000 platform/linux-dpdk/m4/odp_ml.m4 diff --git a/config/odp-linux-dpdk.conf b/config/odp-linux-dpdk.conf index 7e1192fca..e82626bcd 100644 --- a/config/odp-linux-dpdk.conf +++ b/config/odp-linux-dpdk.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.25" +config_file_version = "0.1.26" # System options system: { @@ -335,3 +335,34 @@ dma: { # Maximum number of inflight transfers per session max_inflight = 16 } + +ml: { + # Enable onnxruntime profiling, when enabled, a json file will be + # generated after inference. chrome://tracing/ can be used to check + # the profiling. Use 0 to disable and 1 to enable profiling. + enable_profiling = 0 + + # Choose onnxruntime execution mode, which can be "SEQUENTIAL" or + # "PARALLEL" + execution_mode = "SEQUENTIAL" + + # Set the number of threads used to parallelize the execution of the + # graph across nodes. A value of 0 means onnxruntime will pick a default. + inter_op_num_threads = 0 + + # Set the number of threads used to parallelize the execution within + # a node. A value of 0 means onnxruntime will pick a default. + intra_op_num_threads = 0 + + # Set graph optimization level. Valid values are: + # DISABLE_ALL: disables all optimizations + # ENABLE_BASIC: enables basic optimizations + # ENABLE_EXTENDED: enables basic and extended optimizations + # ENABLE_ALL: enables all available optimizations including layout optimization + graph_optimization_level = "ENABLE_ALL" + + # Serialize the optimized model to disk. When initializing a session + # with the same model, no need to apply optimization anymore, thus + # reducing model startup time. + optimized_model_filepath = "" +} diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index c04979c45..3973b3c3b 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -13,6 +13,7 @@ AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/@ARCH_DIR@ AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/default AM_CPPFLAGS += $(OPENSSL_CPPFLAGS) +AM_CPPFLAGS += $(ORT_CPPFLAGS) AM_CFLAGS += $(DPDK_CFLAGS) AM_CFLAGS += $(LIBCONFIG_CFLAGS) @@ -137,6 +138,7 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_classification_internal.h \ include/odp_eventdev_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_forward_typedefs_internal.h \ + ${top_srcdir}/platform/linux-generic/include/odp_ml_fp16.h \ ${top_srcdir}/platform/linux-generic/include/odp_global_data.h \ ${top_srcdir}/platform/linux-generic/include/odp_init_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_ipsec_internal.h \ @@ -218,6 +220,8 @@ __LIB__libodp_dpdk_la_SOURCES = \ ../linux-generic/odp_ipsec_sad.c \ ../linux-generic/odp_name_table.c \ ../linux-generic/odp_libconfig.c \ + ../linux-generic/odp_ml_fp16.c \ + ../linux-generic/odp_ml_quantize.c \ odp_packet.c \ odp_packet_dpdk.c \ ../linux-generic/odp_packet_vector.c \ @@ -256,6 +260,14 @@ __LIB__libodp_dpdk_la_SOURCES = \ ../linux-generic/odp_version.c \ ../linux-generic/odp_weak.c +if WITH_ML +__LIB__libodp_dpdk_la_SOURCES += \ + ../linux-generic/odp_ml.c +else +__LIB__libodp_dpdk_la_SOURCES += \ + ../linux-generic/odp_ml_null.c +endif + if ODP_ABI_COMPAT __LIB__libodp_dpdk_la_SOURCES += \ ../linux-generic/odp_atomic_api.c \ @@ -416,6 +428,7 @@ __LIB__libodp_dpdk_la_LIBADD += $(LIBCONFIG_LIBS) __LIB__libodp_dpdk_la_LIBADD += $(DPDK_LIBS_LIBODP) __LIB__libodp_dpdk_la_LIBADD += $(PTHREAD_LIBS) __LIB__libodp_dpdk_la_LIBADD += $(TIMER_LIBS) +__LIB__libodp_dpdk_la_LIBADD += $(ORT_LIBS) CHECK_GLOBALS_REGEX = " (odp_|_odp_|_deprecated_odp_|miniz_|mz_|tdefl_|tinfl_|mp_hdlr_init_odp_pool_ops)" diff --git a/platform/linux-dpdk/include/odp_config_internal.h b/platform/linux-dpdk/include/odp_config_internal.h index bc69610ca..c423ec14b 100644 --- a/platform/linux-dpdk/include/odp_config_internal.h +++ b/platform/linux-dpdk/include/odp_config_internal.h @@ -177,6 +177,15 @@ extern "C" { */ #define CONFIG_IPSEC_MAX_NUM_SA 4000 +/* Maximum number of ML models that can be created or loaded. */ +#define CONFIG_ML_MAX_MODELS 4 + +/* Maximum number of inputs for a ML model. */ +#define CONFIG_ML_MAX_INPUTS 4 + +/* Maximum number of outputs for a ML model. */ +#define CONFIG_ML_MAX_OUTPUTS 4 + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/libodp-dpdk.pc.in b/platform/linux-dpdk/libodp-dpdk.pc.in index c3ee4f7b9..8fcc4ac0a 100644 --- a/platform/linux-dpdk/libodp-dpdk.pc.in +++ b/platform/linux-dpdk/libodp-dpdk.pc.in @@ -8,5 +8,5 @@ Description: The ODP packet processing engine Version: @PKGCONFIG_VERSION@ Requires.private: libconfig Libs: -L${libdir} -l@ODP_LIB_NAME@ @DPDK_LIBS_NON_ABI_COMPAT@ @ATOMIC_LIBS_NON_ABI_COMPAT@ -Libs.private: @DPDK_LIBS_ABI_COMPAT@ @OPENSSL_STATIC_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ +Libs.private: @DPDK_LIBS_ABI_COMPAT@ @OPENSSL_STATIC_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ @ORT_LIBS@ Cflags: -I${includedir} @DPDK_CFLAGS@ diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index 0fcd4a5b3..5a2e7a8a6 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -11,6 +11,7 @@ m4_include([platform/linux-dpdk/m4/odp_openssl.m4]) m4_include([platform/linux-dpdk/m4/odp_pcapng.m4]) m4_include([platform/linux-dpdk/m4/odp_scheduler.m4]) m4_include([platform/linux-dpdk/m4/odp_wfe.m4]) +m4_include([platform/linux-dpdk/m4/odp_ml.m4]) ODP_EVENT_VALIDATION ODP_PTHREAD @@ -66,7 +67,7 @@ esac # Required for experimental rte_event_port_unlinks_in_progress() API DPDK_CFLAGS="${DPDK_CFLAGS} -DALLOW_EXPERIMENTAL_API" -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${ORT_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. @@ -77,6 +78,7 @@ AS_VAR_APPEND([PLAT_CFG_TEXT], [" pcap: ${have_pmd_pcap} pcapng: ${have_pcapng} wfe_locks: ${use_wfe_locks} + ml_support: ${ml_support} default_config_path: ${default_config_path}"]) ODP_CHECK_CFLAG([-Wno-error=cast-align]) diff --git a/platform/linux-dpdk/m4/odp_libconfig.m4 b/platform/linux-dpdk/m4/odp_libconfig.m4 index 7bcfb4505..2bf89ac2e 100644 --- a/platform/linux-dpdk/m4/odp_libconfig.m4 +++ b/platform/linux-dpdk/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [25]) +m4_define([_odp_config_version_minor], [26]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-dpdk/m4/odp_ml.m4 b/platform/linux-dpdk/m4/odp_ml.m4 new file mode 120000 index 000000000..6e76047e5 --- /dev/null +++ b/platform/linux-dpdk/m4/odp_ml.m4 @@ -0,0 +1 @@ +../../linux-generic/m4/odp_ml.m4 \ No newline at end of file diff --git a/platform/linux-dpdk/odp_event.c b/platform/linux-dpdk/odp_event.c index dca743029..0c2f3d22e 100644 --- a/platform/linux-dpdk/odp_event.c +++ b/platform/linux-dpdk/odp_event.c @@ -12,6 +12,7 @@ #include <odp/api/packet.h> #include <odp/api/timer.h> #include <odp/api/pool.h> +#include <odp/api/ml.h> #include <odp_buffer_internal.h> #include <odp_ipsec_internal.h> @@ -70,6 +71,9 @@ static inline void event_free(odp_event_t event, _odp_ev_id_t id) case ODP_EVENT_DMA_COMPL: odp_dma_compl_free(odp_dma_compl_from_event(event)); break; + case ODP_EVENT_ML_COMPL: + odp_ml_compl_free(odp_ml_compl_from_event(event)); + break; default: _ODP_ABORT("Invalid event type: %d\n", odp_event_type(event)); } @@ -118,6 +122,8 @@ int odp_event_is_valid(odp_event_t event) /* Fall through */ case ODP_EVENT_DMA_COMPL: /* Fall through */ + case ODP_EVENT_ML_COMPL: + /* Fall through */ case ODP_EVENT_PACKET_TX_COMPL: break; default: diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index 4d6f395c5..79c449f1d 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -59,6 +59,7 @@ enum init_stage { IPSEC_SAD_INIT, IPSEC_INIT, DMA_INIT, + ML_INIT, ALL_INIT /* All init stages completed */ }; @@ -103,6 +104,7 @@ static void disable_features(odp_global_data_ro_t *global_ro, global_ro->disable.traffic_mngr = init_param->not_used.feat.tm; global_ro->disable.compress = init_param->not_used.feat.compress; + global_ro->disable.ml = init_param->not_used.feat.ml; } static int read_pci_config(char **pci_cmd) @@ -331,6 +333,13 @@ static int term_global(enum init_stage stage) switch (stage) { case ALL_INIT: + case ML_INIT: + if (_odp_ml_term_global()) { + _ODP_ERR("ODP ML term failed.\n"); + rc = -1; + } + /* Fall through */ + case DMA_INIT: if (_odp_dma_term_global()) { _ODP_ERR("ODP DMA term failed.\n"); @@ -689,6 +698,12 @@ int odp_init_global(odp_instance_t *instance, } stage = DMA_INIT; + if (_odp_ml_init_global()) { + _ODP_ERR("ODP ML init failed.\n"); + goto init_failed; + } + stage = ML_INIT; + /* Dummy support for single instance */ *instance = (odp_instance_t)odp_global_ro.main_pid; diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 0d62b8f1d..dfd14a978 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -853,6 +853,8 @@ static const char *get_short_type_str(odp_pool_type_t type) return "V"; case ODP_POOL_DMA_COMPL: return "D"; + case ODP_POOL_ML_COMPL: + return "M"; default: return "-"; } @@ -947,6 +949,11 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info) info->dma_pool_param.uarea_size = pool->params.buf.uarea_size; info->dma_pool_param.cache_size = pool->params.buf.cache_size; + } else if (pool->type_2 == ODP_POOL_ML_COMPL) { + info->ml_pool_param.num = pool->params.buf.num; + info->ml_pool_param.uarea_size = pool->params.buf.uarea_size; + info->ml_pool_param.cache_size = pool->params.buf.cache_size; + } else { info->params = pool->params; } @@ -1092,6 +1099,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, case ODP_POOL_TIMEOUT: case ODP_POOL_VECTOR: case ODP_POOL_DMA_COMPL: + case ODP_POOL_ML_COMPL: memset(capa, 0, sizeof(odp_pool_ext_capability_t)); return 0; default: diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index 0ebb3e09f..1e67561bd 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -552,5 +552,8 @@ void odp_sys_config_print(void) _ODP_PRINT("CONFIG_POOLS: %i\n", CONFIG_POOLS); _ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); _ODP_PRINT("CONFIG_IPSEC_MAX_NUM_SA: %i\n", CONFIG_IPSEC_MAX_NUM_SA); + _ODP_PRINT("CONFIG_ML_MAX_MODELS: %i\n", CONFIG_ML_MAX_MODELS); + _ODP_PRINT("CONFIG_ML_MAX_INPUTS: %i\n", CONFIG_ML_MAX_INPUTS); + _ODP_PRINT("CONFIG_ML_MAX_OUTPUTS: %i\n", CONFIG_ML_MAX_OUTPUTS); _ODP_PRINT("\n"); } diff --git a/platform/linux-dpdk/test/crypto.conf b/platform/linux-dpdk/test/crypto.conf index 9b10b9a76..97fdea6f5 100644 --- a/platform/linux-dpdk/test/crypto.conf +++ b/platform/linux-dpdk/test/crypto.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.25" +config_file_version = "0.1.26" system: { # One crypto queue pair is required per thread for lockless operation diff --git a/platform/linux-dpdk/test/default-timer.conf b/platform/linux-dpdk/test/default-timer.conf index 495972c5e..3219854de 100644 --- a/platform/linux-dpdk/test/default-timer.conf +++ b/platform/linux-dpdk/test/default-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.25" +config_file_version = "0.1.26" timer: { # Use DPDK default timer API based implementation diff --git a/platform/linux-dpdk/test/process-mode.conf b/platform/linux-dpdk/test/process-mode.conf index a40fd7aa0..827eb6074 100644 --- a/platform/linux-dpdk/test/process-mode.conf +++ b/platform/linux-dpdk/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.25" +config_file_version = "0.1.26" dpdk: { process_mode_memory_mb = 1024 diff --git a/platform/linux-dpdk/test/sched-basic.conf b/platform/linux-dpdk/test/sched-basic.conf index a11d35706..2c11cb419 100644 --- a/platform/linux-dpdk/test/sched-basic.conf +++ b/platform/linux-dpdk/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.25" +config_file_version = "0.1.26" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { diff --git a/platform/linux-dpdk/test/stash-custom.conf b/platform/linux-dpdk/test/stash-custom.conf index bb9c37fda..62f314c4e 100644 --- a/platform/linux-dpdk/test/stash-custom.conf +++ b/platform/linux-dpdk/test/stash-custom.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.25" +config_file_version = "0.1.26" # Test overflow safe stash variant stash: { From f12bf4c10148583a1bcb6cedd4b97633116df26d Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Mon, 4 Mar 2024 09:09:43 +0200 Subject: [PATCH 133/147] Port 58b89a4f0 "linux-gen: example: ml: add ML examples based on onnx models" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- Makefile.am | 1 + platform/linux-dpdk/example/Makefile.am | 5 ++ platform/linux-dpdk/example/ml/.gitignore | 5 ++ platform/linux-dpdk/example/ml/Makefile.am | 54 +++++++++++++++++++ platform/linux-dpdk/example/ml/README.md | 1 + .../linux-dpdk/example/ml/example_digit.csv | 1 + platform/linux-dpdk/example/ml/mnist-12.onnx | 1 + .../linux-dpdk/example/ml/odp_ml_run_mnist.sh | 1 + .../example/ml/odp_ml_run_model_explorer.sh | 1 + .../example/ml/odp_ml_run_simple_linear.sh | 1 + .../linux-dpdk/example/ml/simple_linear.onnx | 1 + platform/linux-dpdk/m4/configure.m4 | 2 + 12 files changed, 74 insertions(+) create mode 100644 platform/linux-dpdk/example/Makefile.am create mode 100644 platform/linux-dpdk/example/ml/.gitignore create mode 100644 platform/linux-dpdk/example/ml/Makefile.am create mode 120000 platform/linux-dpdk/example/ml/README.md create mode 120000 platform/linux-dpdk/example/ml/example_digit.csv create mode 120000 platform/linux-dpdk/example/ml/mnist-12.onnx create mode 120000 platform/linux-dpdk/example/ml/odp_ml_run_mnist.sh create mode 120000 platform/linux-dpdk/example/ml/odp_ml_run_model_explorer.sh create mode 120000 platform/linux-dpdk/example/ml/odp_ml_run_simple_linear.sh create mode 120000 platform/linux-dpdk/example/ml/simple_linear.onnx diff --git a/Makefile.am b/Makefile.am index 1958c5a05..5fbe3681f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -12,6 +12,7 @@ if PLATFORM_IS_LINUX_DPDK PLATFORM_DIR = platform/linux-dpdk PLATFORM_DUMPCONF_DIR = platform/linux-dpdk/dumpconfig PLATFORM_TEST_DIR = platform/linux-dpdk/test +PLATFORM_EXAMPLE_DIR = platform/linux-dpdk/example endif SUBDIRS = \ diff --git a/platform/linux-dpdk/example/Makefile.am b/platform/linux-dpdk/example/Makefile.am new file mode 100644 index 000000000..84f337387 --- /dev/null +++ b/platform/linux-dpdk/example/Makefile.am @@ -0,0 +1,5 @@ +SUBDIRS = + +if WITH_ML +SUBDIRS += ml +endif diff --git a/platform/linux-dpdk/example/ml/.gitignore b/platform/linux-dpdk/example/ml/.gitignore new file mode 100644 index 000000000..d845f6bb5 --- /dev/null +++ b/platform/linux-dpdk/example/ml/.gitignore @@ -0,0 +1,5 @@ +model_explorer +simple_linear +mnist +*.log +*.trs diff --git a/platform/linux-dpdk/example/ml/Makefile.am b/platform/linux-dpdk/example/ml/Makefile.am new file mode 100644 index 000000000..7abbc3828 --- /dev/null +++ b/platform/linux-dpdk/example/ml/Makefile.am @@ -0,0 +1,54 @@ +include $(top_srcdir)/example/Makefile.inc + +LDADD += -lm + +bin_PROGRAMS = model_explorer simple_linear mnist + +simple_linear_SOURCES = \ + ../../../linux-generic/example/ml/simple_linear.c \ + ../../../linux-generic/example/ml/model_read.c \ + ../../../linux-generic/example/ml/model_read.h +model_explorer_SOURCES = \ + ../../../linux-generic/example/ml/model_explorer.c \ + ../../../linux-generic/example/ml/model_read.c \ + ../../../linux-generic/example/ml/model_read.h +mnist_SOURCES = \ + ../../../linux-generic/example/ml/mnist.c \ + ../../../linux-generic/example/ml/model_read.c \ + ../../../linux-generic/example/ml/model_read.h + +EXTRA_DIST = \ + odp_ml_run_mnist.sh \ + example_digit.csv \ + mnist-12.onnx \ + odp_ml_run_model_explorer.sh \ + odp_ml_run_simple_linear.sh \ + simple_linear.onnx + +if test_example +TESTS = \ + odp_ml_run_mnist.sh \ + odp_ml_run_model_explorer.sh \ + odp_ml_run_simple_linear.sh +endif + +# If building out-of-tree, make check will not copy the scripts and data to the +# $(builddir) assuming that all commands are run locally. However this prevents +# running tests on a remote target using LOG_COMPILER. +# So copy all script and data files explicitly here. +all-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + if [ -e $(srcdir)/$$f ]; then \ + mkdir -p $(builddir)/$$(dirname $$f); \ + cp -f $(srcdir)/$$f $(builddir)/$$f; \ + fi \ + done \ + fi + +clean-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + rm -f $(builddir)/$$f; \ + done \ + fi diff --git a/platform/linux-dpdk/example/ml/README.md b/platform/linux-dpdk/example/ml/README.md new file mode 120000 index 000000000..ddeec649f --- /dev/null +++ b/platform/linux-dpdk/example/ml/README.md @@ -0,0 +1 @@ +../../../linux-generic/example/ml/README.md \ No newline at end of file diff --git a/platform/linux-dpdk/example/ml/example_digit.csv b/platform/linux-dpdk/example/ml/example_digit.csv new file mode 120000 index 000000000..5e5514aaf --- /dev/null +++ b/platform/linux-dpdk/example/ml/example_digit.csv @@ -0,0 +1 @@ +../../../linux-generic/example/ml/example_digit.csv \ No newline at end of file diff --git a/platform/linux-dpdk/example/ml/mnist-12.onnx b/platform/linux-dpdk/example/ml/mnist-12.onnx new file mode 120000 index 000000000..94d4515b8 --- /dev/null +++ b/platform/linux-dpdk/example/ml/mnist-12.onnx @@ -0,0 +1 @@ +../../../linux-generic/example/ml/mnist-12.onnx \ No newline at end of file diff --git a/platform/linux-dpdk/example/ml/odp_ml_run_mnist.sh b/platform/linux-dpdk/example/ml/odp_ml_run_mnist.sh new file mode 120000 index 000000000..7d9c6f84c --- /dev/null +++ b/platform/linux-dpdk/example/ml/odp_ml_run_mnist.sh @@ -0,0 +1 @@ +../../../linux-generic/example/ml/odp_ml_run_mnist.sh \ No newline at end of file diff --git a/platform/linux-dpdk/example/ml/odp_ml_run_model_explorer.sh b/platform/linux-dpdk/example/ml/odp_ml_run_model_explorer.sh new file mode 120000 index 000000000..f28535b64 --- /dev/null +++ b/platform/linux-dpdk/example/ml/odp_ml_run_model_explorer.sh @@ -0,0 +1 @@ +../../../linux-generic/example/ml/odp_ml_run_model_explorer.sh \ No newline at end of file diff --git a/platform/linux-dpdk/example/ml/odp_ml_run_simple_linear.sh b/platform/linux-dpdk/example/ml/odp_ml_run_simple_linear.sh new file mode 120000 index 000000000..2691d9282 --- /dev/null +++ b/platform/linux-dpdk/example/ml/odp_ml_run_simple_linear.sh @@ -0,0 +1 @@ +../../../linux-generic/example/ml/odp_ml_run_simple_linear.sh \ No newline at end of file diff --git a/platform/linux-dpdk/example/ml/simple_linear.onnx b/platform/linux-dpdk/example/ml/simple_linear.onnx new file mode 120000 index 000000000..5893a9176 --- /dev/null +++ b/platform/linux-dpdk/example/ml/simple_linear.onnx @@ -0,0 +1 @@ +../../../linux-generic/example/ml/simple_linear.onnx \ No newline at end of file diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index 5a2e7a8a6..2aa5e7dd7 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -96,6 +96,8 @@ AM_CONDITIONAL([PLATFORM_IS_LINUX_DPDK], AC_CONFIG_FILES([platform/linux-dpdk/Makefile platform/linux-dpdk/libodp-dpdk.pc platform/linux-dpdk/dumpconfig/Makefile + platform/linux-dpdk/example/Makefile + platform/linux-dpdk/example/ml/Makefile platform/linux-dpdk/test/Makefile platform/linux-dpdk/test/example/Makefile platform/linux-dpdk/test/example/classifier/Makefile From 96d10c633a3f8e4149d0c9e947113ca97b5bd257 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Mon, 4 Mar 2024 10:15:10 +0200 Subject: [PATCH 134/147] Port da534723d "linux-gen: validation: ml: add tests based on onnx models for the new ML API" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/m4/configure.m4 | 1 + platform/linux-dpdk/test/Makefile.am | 6 ++++ .../test/validation/api/ml/.gitignore | 1 + .../test/validation/api/ml/Makefile.am | 29 +++++++++++++++++++ .../test/validation/api/ml/README.md | 1 + .../test/validation/api/ml/batch_add.onnx | 1 + .../test/validation/api/ml/batch_add_gen.py | 1 + .../test/validation/api/ml/gen_models.sh | 1 + .../test/validation/api/ml/requirements.txt | 1 + .../test/validation/api/ml/simple_linear.onnx | 1 + .../validation/api/ml/simple_linear_gen.py | 1 + 11 files changed, 44 insertions(+) create mode 100644 platform/linux-dpdk/test/validation/api/ml/.gitignore create mode 100644 platform/linux-dpdk/test/validation/api/ml/Makefile.am create mode 120000 platform/linux-dpdk/test/validation/api/ml/README.md create mode 120000 platform/linux-dpdk/test/validation/api/ml/batch_add.onnx create mode 120000 platform/linux-dpdk/test/validation/api/ml/batch_add_gen.py create mode 120000 platform/linux-dpdk/test/validation/api/ml/gen_models.sh create mode 120000 platform/linux-dpdk/test/validation/api/ml/requirements.txt create mode 120000 platform/linux-dpdk/test/validation/api/ml/simple_linear.onnx create mode 120000 platform/linux-dpdk/test/validation/api/ml/simple_linear_gen.py diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index 2aa5e7dd7..535cfc5fa 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -112,5 +112,6 @@ AC_CONFIG_FILES([platform/linux-dpdk/Makefile platform/linux-dpdk/test/example/switch/Makefile platform/linux-dpdk/test/performance/Makefile platform/linux-dpdk/test/performance/dmafwd/Makefile + platform/linux-dpdk/test/validation/api/ml/Makefile platform/linux-dpdk/test/validation/api/pktio/Makefile]) ]) diff --git a/platform/linux-dpdk/test/Makefile.am b/platform/linux-dpdk/test/Makefile.am index c210edbdc..2a33bfbcd 100644 --- a/platform/linux-dpdk/test/Makefile.am +++ b/platform/linux-dpdk/test/Makefile.am @@ -18,6 +18,12 @@ test_SCRIPTS = $(dist_check_SCRIPTS) SUBDIRS += validation/api/pktio \ example \ performance + +if WITH_ML +TESTS += validation/api/ml/ml_linux$(EXEEXT) +SUBDIRS += validation/api/ml +endif + else #performance tests refer to pktio_env if test_perf diff --git a/platform/linux-dpdk/test/validation/api/ml/.gitignore b/platform/linux-dpdk/test/validation/api/ml/.gitignore new file mode 100644 index 000000000..e31f902c4 --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/.gitignore @@ -0,0 +1 @@ +ml_linux diff --git a/platform/linux-dpdk/test/validation/api/ml/Makefile.am b/platform/linux-dpdk/test/validation/api/ml/Makefile.am new file mode 100644 index 000000000..40910d5c6 --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/Makefile.am @@ -0,0 +1,29 @@ +include ../Makefile.inc + +test_PROGRAMS = ml_linux +ml_linux_SOURCES = ../../../../../linux-generic/test/validation/api/ml/ml_linux.c + +EXTRA_DIST = \ + batch_add.onnx \ + simple_linear.onnx + +# If building out-of-tree, make check will not copy the scripts and data to the +# $(builddir) assuming that all commands are run locally. However this prevents +# running tests on a remote target using LOG_COMPILER. +# So copy all script and data files explicitly here. +all-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + if [ -e $(srcdir)/$$f ]; then \ + mkdir -p $(builddir)/$$(dirname $$f); \ + cp -f $(srcdir)/$$f $(builddir)/$$f; \ + fi \ + done \ + fi + +clean-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + rm -f $(builddir)/$$f; \ + done \ + fi diff --git a/platform/linux-dpdk/test/validation/api/ml/README.md b/platform/linux-dpdk/test/validation/api/ml/README.md new file mode 120000 index 000000000..d121a477d --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/README.md @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/README.md \ No newline at end of file diff --git a/platform/linux-dpdk/test/validation/api/ml/batch_add.onnx b/platform/linux-dpdk/test/validation/api/ml/batch_add.onnx new file mode 120000 index 000000000..b827c0e58 --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/batch_add.onnx @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/batch_add.onnx \ No newline at end of file diff --git a/platform/linux-dpdk/test/validation/api/ml/batch_add_gen.py b/platform/linux-dpdk/test/validation/api/ml/batch_add_gen.py new file mode 120000 index 000000000..695b6d399 --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/batch_add_gen.py @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/batch_add_gen.py \ No newline at end of file diff --git a/platform/linux-dpdk/test/validation/api/ml/gen_models.sh b/platform/linux-dpdk/test/validation/api/ml/gen_models.sh new file mode 120000 index 000000000..e9b25d58f --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/gen_models.sh @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/gen_models.sh \ No newline at end of file diff --git a/platform/linux-dpdk/test/validation/api/ml/requirements.txt b/platform/linux-dpdk/test/validation/api/ml/requirements.txt new file mode 120000 index 000000000..b94d5d389 --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/requirements.txt @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/requirements.txt \ No newline at end of file diff --git a/platform/linux-dpdk/test/validation/api/ml/simple_linear.onnx b/platform/linux-dpdk/test/validation/api/ml/simple_linear.onnx new file mode 120000 index 000000000..f471922d1 --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/simple_linear.onnx @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/simple_linear.onnx \ No newline at end of file diff --git a/platform/linux-dpdk/test/validation/api/ml/simple_linear_gen.py b/platform/linux-dpdk/test/validation/api/ml/simple_linear_gen.py new file mode 120000 index 000000000..53fb4f6ed --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/simple_linear_gen.py @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/simple_linear_gen.py \ No newline at end of file From 20bb2580698541af37cea8d23ae456ee49f95061 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Thu, 29 Feb 2024 16:49:27 +0200 Subject: [PATCH 135/147] Port 53d82cb76 "linux-gen: wait_until: add generic 32-bit wait until functions" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/Makefile.am | 16 ++++++++++------ .../linux-dpdk/arch/default/odp_wait_until.h | 1 + 2 files changed, 11 insertions(+), 6 deletions(-) create mode 120000 platform/linux-dpdk/arch/default/odp_wait_until.h diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 3973b3c3b..43513af99 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -321,8 +321,8 @@ noinst_HEADERS += arch/arm/odp_atomic.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h - + arch/default/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_AARCH64 __LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ @@ -349,7 +349,8 @@ noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/cpu_flags.h \ arch/aarch64/odp_cpu_idling.h \ arch/aarch64/odp_llsc.h \ - arch/aarch64/odp_random.h + arch/aarch64/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_DEFAULT __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ @@ -371,7 +372,8 @@ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h + arch/default/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_POWERPC __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ @@ -393,7 +395,8 @@ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h + arch/default/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_X86 __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ @@ -419,7 +422,8 @@ noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_random.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h + arch/default/odp_cpu_idling.h \ + arch/default/odp_wait_until.h endif __LIB__libodp_dpdk_la_LIBADD = $(ATOMIC_LIBS) diff --git a/platform/linux-dpdk/arch/default/odp_wait_until.h b/platform/linux-dpdk/arch/default/odp_wait_until.h new file mode 120000 index 000000000..d2e7b5316 --- /dev/null +++ b/platform/linux-dpdk/arch/default/odp_wait_until.h @@ -0,0 +1 @@ +../../../linux-generic/arch/default/odp_wait_until.h \ No newline at end of file From 237f41f1679cd7bc887fb4dc0376a6af563386f3 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Thu, 29 Feb 2024 16:53:46 +0200 Subject: [PATCH 136/147] Port 7f3328c03 "linux-gen: cpu: remove now unused generic idling header" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/Makefile.am | 4 ---- platform/linux-dpdk/arch/default/odp_cpu_idling.h | 1 - 2 files changed, 5 deletions(-) delete mode 120000 platform/linux-dpdk/arch/default/odp_cpu_idling.h diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 43513af99..59427dc2f 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -320,7 +320,6 @@ noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_llsc.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ arch/default/odp_random.h \ arch/default/odp_wait_until.h endif @@ -371,7 +370,6 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ arch/default/odp_random.h \ arch/default/odp_wait_until.h endif @@ -394,7 +392,6 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ arch/default/odp_random.h \ arch/default/odp_wait_until.h endif @@ -422,7 +419,6 @@ noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_random.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ arch/default/odp_wait_until.h endif diff --git a/platform/linux-dpdk/arch/default/odp_cpu_idling.h b/platform/linux-dpdk/arch/default/odp_cpu_idling.h deleted file mode 120000 index eb2d21c4b..000000000 --- a/platform/linux-dpdk/arch/default/odp_cpu_idling.h +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/default/odp_cpu_idling.h \ No newline at end of file From 0e8b04e1add19d541cea9a0f9f546b23e5cc74b8 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Thu, 29 Feb 2024 16:56:21 +0200 Subject: [PATCH 137/147] Port d4dd30fe4 "linux-gen: wait_until: add aarch64 specific wait until functions" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/Makefile.am | 4 +--- platform/linux-dpdk/arch/aarch64/odp_cpu_idling.h | 1 - platform/linux-dpdk/arch/aarch64/odp_llsc.h | 1 - platform/linux-dpdk/arch/aarch64/odp_wait_until.h | 1 + 4 files changed, 2 insertions(+), 5 deletions(-) delete mode 120000 platform/linux-dpdk/arch/aarch64/odp_cpu_idling.h delete mode 120000 platform/linux-dpdk/arch/aarch64/odp_llsc.h create mode 120000 platform/linux-dpdk/arch/aarch64/odp_wait_until.h diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 59427dc2f..13e0d35a6 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -346,10 +346,8 @@ endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ arch/aarch64/cpu_flags.h \ - arch/aarch64/odp_cpu_idling.h \ - arch/aarch64/odp_llsc.h \ arch/aarch64/odp_random.h \ - arch/default/odp_wait_until.h + arch/aarch64/odp_wait_until.h endif if ARCH_IS_DEFAULT __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ diff --git a/platform/linux-dpdk/arch/aarch64/odp_cpu_idling.h b/platform/linux-dpdk/arch/aarch64/odp_cpu_idling.h deleted file mode 120000 index c8230bb63..000000000 --- a/platform/linux-dpdk/arch/aarch64/odp_cpu_idling.h +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/aarch64/odp_cpu_idling.h \ No newline at end of file diff --git a/platform/linux-dpdk/arch/aarch64/odp_llsc.h b/platform/linux-dpdk/arch/aarch64/odp_llsc.h deleted file mode 120000 index eb8d1200b..000000000 --- a/platform/linux-dpdk/arch/aarch64/odp_llsc.h +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/aarch64/odp_llsc.h \ No newline at end of file diff --git a/platform/linux-dpdk/arch/aarch64/odp_wait_until.h b/platform/linux-dpdk/arch/aarch64/odp_wait_until.h new file mode 120000 index 000000000..f7d35f0ca --- /dev/null +++ b/platform/linux-dpdk/arch/aarch64/odp_wait_until.h @@ -0,0 +1 @@ +../../../linux-generic/arch/aarch64/odp_wait_until.h \ No newline at end of file From efba5fe433a69b203e6e3df56ab99d2a71001ba3 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Thu, 29 Feb 2024 16:57:52 +0200 Subject: [PATCH 138/147] Port f2c3a8785 "linux-gen: wait_until: clean 32-bit arm headers" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/Makefile.am | 2 -- platform/linux-dpdk/arch/arm/odp_cpu_idling.h | 1 - platform/linux-dpdk/arch/arm/odp_llsc.h | 1 - 3 files changed, 4 deletions(-) delete mode 120000 platform/linux-dpdk/arch/arm/odp_cpu_idling.h delete mode 120000 platform/linux-dpdk/arch/arm/odp_llsc.h diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 13e0d35a6..bbfb04672 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -316,8 +316,6 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/arm/odp_atomic.h \ arch/arm/odp_cpu.h \ - arch/arm/odp_cpu_idling.h \ - arch/arm/odp_llsc.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_random.h \ diff --git a/platform/linux-dpdk/arch/arm/odp_cpu_idling.h b/platform/linux-dpdk/arch/arm/odp_cpu_idling.h deleted file mode 120000 index 56afe9027..000000000 --- a/platform/linux-dpdk/arch/arm/odp_cpu_idling.h +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/arm/odp_cpu_idling.h \ No newline at end of file diff --git a/platform/linux-dpdk/arch/arm/odp_llsc.h b/platform/linux-dpdk/arch/arm/odp_llsc.h deleted file mode 120000 index b3f3f371f..000000000 --- a/platform/linux-dpdk/arch/arm/odp_llsc.h +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/arm/odp_llsc.h \ No newline at end of file From 63062b03ea391c4340ffaefc0a37658dbcb18b72 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Thu, 29 Feb 2024 16:59:27 +0200 Subject: [PATCH 139/147] Port 92282e857 "linux-gen: remove 32-bit arm specific atomics header" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/Makefile.am | 3 +-- platform/linux-dpdk/arch/arm/odp_atomic.h | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 120000 platform/linux-dpdk/arch/arm/odp_atomic.h diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index bbfb04672..0bfe35092 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -314,8 +314,7 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/wait_until_generic.h \ arch/default/odp/api/abi/wait_until.h endif -noinst_HEADERS += arch/arm/odp_atomic.h \ - arch/arm/odp_cpu.h \ +noinst_HEADERS += arch/arm/odp_cpu.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ arch/default/odp_random.h \ diff --git a/platform/linux-dpdk/arch/arm/odp_atomic.h b/platform/linux-dpdk/arch/arm/odp_atomic.h deleted file mode 120000 index 61a8c536b..000000000 --- a/platform/linux-dpdk/arch/arm/odp_atomic.h +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/arm/odp_atomic.h \ No newline at end of file From 39ca2e4e22699666502c744ee54ebb4391892457 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Fri, 1 Mar 2024 09:56:31 +0200 Subject: [PATCH 140/147] Port f74bd07ad "linux-gen: remove unnecessary pthread includes" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/odp_system_info.c | 1 - 1 file changed, 1 deletion(-) diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index 1e67561bd..7d5295e0b 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -27,7 +27,6 @@ #include <odp_packet_internal.h> #include <errno.h> -#include <pthread.h> #include <string.h> #include <stdio.h> #include <inttypes.h> From 38f1fda41c164e4f107f5b3f06258b2e679bbea9 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Fri, 1 Mar 2024 10:17:27 +0200 Subject: [PATCH 141/147] Port 90dd758b7 "linux-gen: timer: return time to the next timeout from timer_run()" Time to the next timeout is not available with DPDK timer implementation. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/include/odp_timer_internal.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/platform/linux-dpdk/include/odp_timer_internal.h b/platform/linux-dpdk/include/odp_timer_internal.h index 3cc8ca469..4d7efedd6 100644 --- a/platform/linux-dpdk/include/odp_timer_internal.h +++ b/platform/linux-dpdk/include/odp_timer_internal.h @@ -24,6 +24,8 @@ #include <rte_config.h> +#include <stdint.h> + /** * Internal Timeout header */ @@ -56,10 +58,13 @@ ODP_STATIC_ASSERT(sizeof(odp_timeout_hdr_t) <= 3 * RTE_CACHE_LINE_SIZE, void _odp_timer_run_inline(int dec); /* Static inline wrapper to minimize modification of schedulers. */ -static inline void timer_run(int dec) +static inline uint64_t timer_run(int dec) { if (odp_global_rw->inline_timers) _odp_timer_run_inline(dec); + + /* Time to the next timeout not available with DPDK timers */ + return UINT64_MAX; } #endif From 852c569e0c5c4217f31edac1119ce914dcdcfeda Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Fri, 1 Mar 2024 10:26:47 +0200 Subject: [PATCH 142/147] Port 22c7067de "linux-gen: timer: allow forcing a scan in _odp_timer_run_inline()" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/include/odp_timer_internal.h | 6 ++++++ platform/linux-dpdk/odp_timer.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/platform/linux-dpdk/include/odp_timer_internal.h b/platform/linux-dpdk/include/odp_timer_internal.h index 4d7efedd6..35a4911af 100644 --- a/platform/linux-dpdk/include/odp_timer_internal.h +++ b/platform/linux-dpdk/include/odp_timer_internal.h @@ -26,6 +26,12 @@ #include <stdint.h> +/* + * Use as the argument to timer_run() to force a scan and to ignore rate + * limit. + */ +#define TIMER_SCAN_FORCE INT32_MAX + /** * Internal Timeout header */ diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c index e01a541ad..f4c190aad 100644 --- a/platform/linux-dpdk/odp_timer.c +++ b/platform/linux-dpdk/odp_timer.c @@ -438,7 +438,7 @@ int _odp_timer_term_local(void) void _odp_timer_run_inline(int dec) { - int poll_interval = timer_global->poll_interval; + int poll_interval = (dec == TIMER_SCAN_FORCE) ? 0 : timer_global->poll_interval; odp_time_t now; int ret; From a14b6d76b84822997bb32ebdeb8c225fac66937a Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Fri, 1 Mar 2024 10:45:25 +0200 Subject: [PATCH 143/147] Port 218d405fe "build: re-enable parallel build" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/test/example/ipsec_api/Makefile.am | 2 -- platform/linux-dpdk/test/example/ipsec_crypto/Makefile.am | 2 -- 2 files changed, 4 deletions(-) diff --git a/platform/linux-dpdk/test/example/ipsec_api/Makefile.am b/platform/linux-dpdk/test/example/ipsec_api/Makefile.am index 101c97cdf..2535ad466 100644 --- a/platform/linux-dpdk/test/example/ipsec_api/Makefile.am +++ b/platform/linux-dpdk/test/example/ipsec_api/Makefile.am @@ -19,5 +19,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: diff --git a/platform/linux-dpdk/test/example/ipsec_crypto/Makefile.am b/platform/linux-dpdk/test/example/ipsec_crypto/Makefile.am index 101c97cdf..2535ad466 100644 --- a/platform/linux-dpdk/test/example/ipsec_crypto/Makefile.am +++ b/platform/linux-dpdk/test/example/ipsec_crypto/Makefile.am @@ -19,5 +19,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: From e499664e036651e82809c730230108db63593685 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Mon, 4 Mar 2024 15:24:31 +0200 Subject: [PATCH 144/147] Port 132cfb7f3 "linux-gen: sysinfo: do not save cpu frequency if it is not static" Port original commit from linux-generic. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/odp_system_info.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index 7d5295e0b..d9cddcb40 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -311,8 +311,9 @@ int _odp_system_info_init(void) num_cpus); /* Read and save all CPU frequencies for static mode */ - for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) - odp_global_ro.system_info.cpu_hz[i] = cpu_hz_current(i); + if (odp_global_ro.system_info.cpu_hz_static) + for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) + odp_global_ro.system_info.cpu_hz[i] = cpu_hz_current(i); /* By default, read max frequency from a cpufreq file */ for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) { From 615737d7b48bbb50fb3dae2f0b172bc00f6803fb Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Fri, 1 Mar 2024 10:57:03 +0200 Subject: [PATCH 145/147] linux-dpdk: crypto: avoid variable name shadowing Fixes -Wshadow compiler warnings. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/odp_crypto.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index aeb9c11e3..6170dd8df 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -707,12 +707,11 @@ static int cipher_gen_capability(const struct rte_crypto_param_range *key_size, for (uint32_t key_len = key_size_min; key_len <= key_size_max; key_len += key_inc) { - for (uint32_t iv_size = iv_size_min; - iv_size <= iv_size_max; iv_size += iv_inc) { + for (uint32_t iv_len = iv_size_min; iv_len <= iv_size_max; iv_len += iv_inc) { odp_crypto_cipher_capability_t capa; capa.key_len = key_len; - capa.iv_len = iv_size; + capa.iv_len = iv_len; capa.bit_mode = false; idx = cipher_capa_insert(src, &capa, idx, num_copy); @@ -918,14 +917,14 @@ static int auth_gen_capability(const struct rte_crypto_param_range *key_size, for (uint16_t key_len = key_size_min; key_len <= key_size_max; key_len += key_inc) { - for (uint16_t iv_size = iv_size_min; - iv_size <= iv_size_max; - iv_size += iv_inc) { + for (uint16_t iv_len = iv_size_min; + iv_len <= iv_size_max; + iv_len += iv_inc) { odp_crypto_auth_capability_t capa; capa.digest_len = digest_len; capa.key_len = key_len; - capa.iv_len = iv_size; + capa.iv_len = iv_len; capa.bit_mode = false; capa.aad_len.min = aad_size->min; capa.aad_len.max = aad_size->max; From bdfcc3da97e1973e46aec527ff46b99542790329 Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Fri, 1 Mar 2024 10:59:35 +0200 Subject: [PATCH 146/147] linux-dpdk: eventdev: avoid variable name shadowing Fixes -Wshadow compiler warning. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- platform/linux-dpdk/odp_schedule_eventdev.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform/linux-dpdk/odp_schedule_eventdev.c b/platform/linux-dpdk/odp_schedule_eventdev.c index d6f3ba2f7..4ef8a51b7 100644 --- a/platform/linux-dpdk/odp_schedule_eventdev.c +++ b/platform/linux-dpdk/odp_schedule_eventdev.c @@ -604,8 +604,8 @@ static inline uint16_t input_cached(odp_event_t out_ev[], unsigned int max_num, uint8_t first_queue = _odp_eventdev_local.cache.event[idx].queue_id; for (i = 0; i < max_num && _odp_eventdev_local.cache.count; i++) { - uint16_t idx = _odp_eventdev_local.cache.idx; - struct rte_event *event = &_odp_eventdev_local.cache.event[idx]; + uint16_t cache_idx = _odp_eventdev_local.cache.idx; + struct rte_event *event = &_odp_eventdev_local.cache.event[cache_idx]; if (odp_unlikely(event->queue_id != first_queue)) break; From 9ff786ed3d9d553f8e108eff4ee4ceec4adb585e Mon Sep 17 00:00:00 2001 From: Matias Elo <matias.elo@nokia.com> Date: Fri, 8 Mar 2024 15:06:23 +0200 Subject: [PATCH 147/147] linux-dpdk: list dpdk v23.11 as supported Add DPDK v23.11 to CI and list it as supported version in README. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Tuomas Taipale <tuomas.taipale@nokia.com> --- .github/workflows/ci-pipeline-arm64.yml | 11 +++++++++++ .github/workflows/ci-pipeline.yml | 9 +++++++++ platform/linux-dpdk/README | 3 ++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml index 780fc00cc..391ebfc3b 100644 --- a/.github/workflows/ci-pipeline-arm64.yml +++ b/.github/workflows/ci-pipeline-arm64.yml @@ -267,3 +267,14 @@ jobs: -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native-dpdk_21.11 /odp/scripts/ci/check.sh - if: ${{ failure() }} uses: ./.github/actions/run-failure-log + + Run_dpdk-23_11: + if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }} + runs-on: [self-hosted, ARM64] + steps: + - uses: OpenDataPlane/action-clean-up@main + - uses: actions/checkout@v4 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" + -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native-dpdk_23.11 /odp/scripts/ci/check.sh + - if: ${{ failure() }} + uses: ./.github/actions/run-failure-log diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 1cca49aaf..832d91ee1 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -356,6 +356,15 @@ jobs: - if: ${{ failure() }} uses: ./.github/actions/run-failure-log + Run_dpdk-23_11: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" + -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-dpdk_23.11 /odp/scripts/ci/check.sh + - if: ${{ failure() }} + uses: ./.github/actions/run-failure-log + Run_crypto: runs-on: ubuntu-20.04 strategy: diff --git a/platform/linux-dpdk/README b/platform/linux-dpdk/README index 421d3f958..c0298ab34 100644 --- a/platform/linux-dpdk/README +++ b/platform/linux-dpdk/README @@ -36,7 +36,8 @@ cmds below for Ubuntu, where it has been compiled and tested. On Ubuntu install pcap development library: sudo apt-get install libpcap-dev -Right now ODP-DPDK supports DPDK v21.11 and v22.11 (recommended version). +Right now ODP-DPDK supports DPDK v21.11, v22.11 (recommended version), and +v23.11. Compile DPDK ------------