From 7106316ca201b9d0084a6d083eb13fd8873bd7a9 Mon Sep 17 00:00:00 2001 From: dog Date: Thu, 19 Sep 2024 10:55:58 +0800 Subject: [PATCH 1/2] feat: 1. support prom scrape_protocols, 2. support prom enable_compression (#1719) --- core/prometheus/Constants.h | 15 ++ core/prometheus/schedulers/ScrapeConfig.cpp | 112 +++++++++- core/prometheus/schedulers/ScrapeConfig.h | 9 +- .../prometheus/schedulers/ScrapeScheduler.cpp | 4 +- .../prometheus/ScrapeConfigUnittest.cpp | 201 +++++++++++++++++- .../prometheus/ScrapeSchedulerUnittest.cpp | 2 +- 6 files changed, 327 insertions(+), 16 deletions(-) diff --git a/core/prometheus/Constants.h b/core/prometheus/Constants.h index fbd33597ae..6a60e3a161 100644 --- a/core/prometheus/Constants.h +++ b/core/prometheus/Constants.h @@ -50,6 +50,7 @@ const char* const SCHEME = "scheme"; const char* const METRICS_PATH = "metrics_path"; const char* const SCRAPE_INTERVAL = "scrape_interval"; const char* const SCRAPE_TIMEOUT = "scrape_timeout"; +const char* const SCRAPE_PROTOCOLS = "scrape_protocols"; const char* const HEADERS = "headers"; const char* const PARAMS = "params"; const char* const QUERY_STRING = "query_string"; @@ -71,6 +72,14 @@ const char* const PASSWORD = "password"; const char* const PASSWORD_FILE = "password_file"; const char* const BASIC_PREFIX = "Basic "; +// scrape protocols, from https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config +// text/plain, application/openmetrics-text will be used +// version of openmetrics is 1.0.0 or 0.0.1, from +// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#extensions-and-improvements +const char* const PrometheusProto = "PrometheusProto"; +const char* const PrometheusText0_0_4 = "PrometheusText0.0.4"; +const char* const OpenMetricsText0_0_1 = "OpenMetricsText0.0.1"; +const char* const OpenMetricsText1_0_0 = "OpenMetricsText1.0.0"; // metric labels const char* const JOB = "job"; @@ -94,4 +103,10 @@ const char* const UP = "up"; const char* const SCRAPE_TIMESTAMP_MILLISEC = "scrape_timestamp_millisec"; +// scrape config compression +const char* const ENABLE_COMPRESSION = "enable_compression"; +const char* const ACCEPT_ENCODING = "Accept-Encoding"; +const char* const GZIP = "gzip"; +const char* const IDENTITY = "identity"; + } // namespace logtail::prometheus diff --git a/core/prometheus/schedulers/ScrapeConfig.cpp b/core/prometheus/schedulers/ScrapeConfig.cpp index 177c9a7e27..ff6ee77b15 100644 --- a/core/prometheus/schedulers/ScrapeConfig.cpp +++ b/core/prometheus/schedulers/ScrapeConfig.cpp @@ -35,6 +35,7 @@ bool ScrapeConfig::Init(const Json::Value& scrapeConfig) { return false; } + if (scrapeConfig.isMember(prometheus::SCRAPE_INTERVAL) && scrapeConfig[prometheus::SCRAPE_INTERVAL].isString()) { string tmpScrapeIntervalString = scrapeConfig[prometheus::SCRAPE_INTERVAL].asString(); mScrapeIntervalSeconds = DurationToSecond(tmpScrapeIntervalString); @@ -43,6 +44,23 @@ bool ScrapeConfig::Init(const Json::Value& scrapeConfig) { string tmpScrapeTimeoutString = scrapeConfig[prometheus::SCRAPE_TIMEOUT].asString(); mScrapeTimeoutSeconds = DurationToSecond(tmpScrapeTimeoutString); } + if (scrapeConfig.isMember(prometheus::SCRAPE_PROTOCOLS) && scrapeConfig[prometheus::SCRAPE_PROTOCOLS].isArray()) { + if (!InitScrapeProtocols(scrapeConfig[prometheus::SCRAPE_PROTOCOLS])) { + LOG_ERROR(sLogger, ("scrape protocol config error", scrapeConfig[prometheus::SCRAPE_PROTOCOLS])); + return false; + } + } else { + Json::Value nullJson; + InitScrapeProtocols(nullJson); + } + + if (scrapeConfig.isMember(prometheus::ENABLE_COMPRESSION) + && scrapeConfig[prometheus::ENABLE_COMPRESSION].isBool()) { + InitEnableCompression(scrapeConfig[prometheus::ENABLE_COMPRESSION].asBool()); + } else { + InitEnableCompression(true); + } + if (scrapeConfig.isMember(prometheus::METRICS_PATH) && scrapeConfig[prometheus::METRICS_PATH].isString()) { mMetricsPath = scrapeConfig[prometheus::METRICS_PATH].asString(); } @@ -185,7 +203,7 @@ bool ScrapeConfig::InitBasicAuth(const Json::Value& basicAuth) { auto token = username + ":" + password; auto token64 = sdk::Base64Enconde(token); - mAuthHeaders[prometheus::A_UTHORIZATION] = prometheus::BASIC_PREFIX + token64; + mRequestHeaders[prometheus::A_UTHORIZATION] = prometheus::BASIC_PREFIX + token64; return true; } @@ -219,8 +237,98 @@ bool ScrapeConfig::InitAuthorization(const Json::Value& authorization) { return false; } - mAuthHeaders[prometheus::A_UTHORIZATION] = type + " " + credentials; + mRequestHeaders[prometheus::A_UTHORIZATION] = type + " " + credentials; + return true; +} + +bool ScrapeConfig::InitScrapeProtocols(const Json::Value& scrapeProtocols) { + static auto sScrapeProtocolsHeaders = std::map{ + {prometheus::PrometheusProto, + "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited"}, + {prometheus::PrometheusText0_0_4, "text/plain;version=0.0.4"}, + {prometheus::OpenMetricsText0_0_1, "application/openmetrics-text;version=0.0.1"}, + {prometheus::OpenMetricsText1_0_0, "application/openmetrics-text;version=1.0.0"}, + }; + static auto sDefaultScrapeProtocols = vector{ + prometheus::PrometheusText0_0_4, + prometheus::PrometheusProto, + prometheus::OpenMetricsText0_0_1, + prometheus::OpenMetricsText1_0_0, + }; + + auto join = [](const vector& strs, const string& sep) { + string result; + for (const auto& str : strs) { + if (!result.empty()) { + result += sep; + } + result += str; + } + return result; + }; + + auto getScrapeProtocols = [](const Json::Value& scrapeProtocols, vector& res) { + for (const auto& scrapeProtocol : scrapeProtocols) { + if (scrapeProtocol.isString()) { + res.push_back(scrapeProtocol.asString()); + } else { + LOG_ERROR(sLogger, ("scrape_protocols config error", "")); + return false; + } + } + return true; + }; + + auto validateScrapeProtocols = [](const vector& scrapeProtocols) { + set dups; + for (const auto& scrapeProtocol : scrapeProtocols) { + if (!sScrapeProtocolsHeaders.count(scrapeProtocol)) { + LOG_ERROR(sLogger, + ("unknown scrape protocol prometheusproto", scrapeProtocol)( + "supported", + "[OpenMetricsText0.0.1 OpenMetricsText1.0.0 PrometheusProto PrometheusText0.0.4]")); + return false; + } + if (dups.count(scrapeProtocol)) { + LOG_ERROR(sLogger, ("duplicated protocol in scrape_protocols", scrapeProtocol)); + return false; + } + dups.insert(scrapeProtocol); + } + return true; + }; + + vector tmpScrapeProtocols; + + if (!getScrapeProtocols(scrapeProtocols, tmpScrapeProtocols)) { + return false; + } + + // if scrape_protocols is empty, use default protocols + if (tmpScrapeProtocols.empty()) { + tmpScrapeProtocols = sDefaultScrapeProtocols; + } + if (!validateScrapeProtocols(tmpScrapeProtocols)) { + return false; + } + + auto weight = tmpScrapeProtocols.size() + 1; + for (auto& tmpScrapeProtocol : tmpScrapeProtocols) { + auto val = sScrapeProtocolsHeaders[tmpScrapeProtocol]; + val += ";q=0." + std::to_string(weight--); + tmpScrapeProtocol = val; + } + tmpScrapeProtocols.push_back("*/*;q=0." + ToString(weight)); + mRequestHeaders[prometheus::ACCEPT] = join(tmpScrapeProtocols, ","); return true; } +void ScrapeConfig::InitEnableCompression(bool enableCompression) { + if (enableCompression) { + mRequestHeaders[prometheus::ACCEPT_ENCODING] = prometheus::GZIP; + } else { + mRequestHeaders[prometheus::ACCEPT_ENCODING] = prometheus::IDENTITY; + } +} + } // namespace logtail \ No newline at end of file diff --git a/core/prometheus/schedulers/ScrapeConfig.h b/core/prometheus/schedulers/ScrapeConfig.h index 4f8e866cbf..eac6d9615a 100644 --- a/core/prometheus/schedulers/ScrapeConfig.h +++ b/core/prometheus/schedulers/ScrapeConfig.h @@ -12,6 +12,7 @@ namespace logtail { + class ScrapeConfig { public: std::string mJobName; @@ -20,7 +21,10 @@ class ScrapeConfig { std::string mMetricsPath; std::string mScheme; - std::map mAuthHeaders; + // auth header + // scrape_protocols Accept header: PrometheusProto, OpenMetricsText0.0.1, OpenMetricsText1.0.0, PrometheusText0.0.4 + // enable_compression Accept-Encoding header: gzip, identity + std::map mRequestHeaders; int64_t mMaxScrapeSizeBytes; int64_t mSampleLimit; @@ -37,10 +41,13 @@ class ScrapeConfig { private: bool InitBasicAuth(const Json::Value& basicAuth); bool InitAuthorization(const Json::Value& authorization); + bool InitScrapeProtocols(const Json::Value& scrapeProtocols); + void InitEnableCompression(bool enableCompression); #ifdef APSARA_UNIT_TEST_MAIN friend class ScrapeConfigUnittest; #endif }; + } // namespace logtail \ No newline at end of file diff --git a/core/prometheus/schedulers/ScrapeScheduler.cpp b/core/prometheus/schedulers/ScrapeScheduler.cpp index f0494c60ae..f5013eebea 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.cpp +++ b/core/prometheus/schedulers/ScrapeScheduler.cpp @@ -67,7 +67,7 @@ void ScrapeScheduler::OnMetricResult(const HttpResponse& response, uint64_t time if (response.mStatusCode != 200) { mScrapeResponseSizeBytes = 0; string headerStr; - for (const auto& [k, v] : mScrapeConfigPtr->mAuthHeaders) { + for (const auto& [k, v] : mScrapeConfigPtr->mRequestHeaders) { headerStr.append(k).append(":").append(v).append(";"); } LOG_WARNING(sLogger, @@ -144,7 +144,7 @@ std::unique_ptr ScrapeScheduler::BuildScrapeTimerEvent(std::chrono:: mPort, mScrapeConfigPtr->mMetricsPath, mScrapeConfigPtr->mQueryString, - mScrapeConfigPtr->mAuthHeaders, + mScrapeConfigPtr->mRequestHeaders, "", mScrapeConfigPtr->mScrapeTimeoutSeconds, mScrapeConfigPtr->mScrapeIntervalSeconds diff --git a/core/unittest/prometheus/ScrapeConfigUnittest.cpp b/core/unittest/prometheus/ScrapeConfigUnittest.cpp index 8560892071..65e7aeb8da 100644 --- a/core/unittest/prometheus/ScrapeConfigUnittest.cpp +++ b/core/unittest/prometheus/ScrapeConfigUnittest.cpp @@ -17,6 +17,8 @@ class ScrapeConfigUnittest : public testing::Test { void TestAuth(); void TestBasicAuth(); void TestAuthorization(); + void TestScrapeProtocols(); + void TestEnableCompression(); private: void SetUp() override; @@ -55,6 +57,12 @@ void ScrapeConfigUnittest::TestInit() { "scrape_interval": "30s", "scrape_timeout": "30s", "metrics_path": "/metrics", + "scrape_protocols": [ + "PrometheusText0.0.4", + "PrometheusProto", + "OpenMetricsText0.0.1" + ], + "enable_compression": false, "scheme": "http", "basic_auth": { "username": "test_user", @@ -92,8 +100,17 @@ void ScrapeConfigUnittest::TestInit() { APSARA_TEST_EQUAL(scrapeConfig.mMetricsPath, "/metrics"); APSARA_TEST_EQUAL(scrapeConfig.mScheme, "http"); + // scrape protocols + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Accept"], + "text/plain;version=0.0.4;q=0.4,application/" + "vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.3," + "application/openmetrics-text;version=0.0.1;q=0.2,*/*;q=0.1"); + + // disable compression + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Accept-Encoding"], "identity"); + // basic auth - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQ="); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQ="); APSARA_TEST_EQUAL(scrapeConfig.mMaxScrapeSizeBytes, 1024 * 1024 * 1024); APSARA_TEST_EQUAL(scrapeConfig.mSampleLimit, 10000); @@ -150,9 +167,9 @@ void ScrapeConfigUnittest::TestBasicAuth() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); APSARA_TEST_TRUE(scrapeConfig.Init(config)); - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQ="); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQ="); - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); configStr = R"JSON({ "job_name": "test_job", "scrape_interval": "30s", @@ -167,10 +184,10 @@ void ScrapeConfigUnittest::TestBasicAuth() { APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); APSARA_TEST_TRUE(scrapeConfig.Init(config)); - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQuZmlsZQ=="); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Basic dGVzdF91c2VyOnRlc3RfcGFzc3dvcmQuZmlsZQ=="); // error - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); configStr = R"JSON({ "job_name": "test_job", "scrape_interval": "30s", @@ -206,12 +223,12 @@ void ScrapeConfigUnittest::TestAuthorization() { })JSON"; APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); APSARA_TEST_TRUE(scrapeConfig.Init(config)); // bearer auth - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Bearer test_token"); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Bearer test_token"); - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); // default Bearer auth configStr = R"JSON({ @@ -226,15 +243,179 @@ void ScrapeConfigUnittest::TestAuthorization() { })JSON"; APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); - scrapeConfig.mAuthHeaders.clear(); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL(scrapeConfig.mRequestHeaders["Authorization"], "Bearer " + mKey); +} + +void ScrapeConfigUnittest::TestScrapeProtocols() { + Json::Value config; + ScrapeConfig scrapeConfig; + string errorMsg; + string configStr; + + // default + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http" + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL( + "text/plain;version=0.0.4;q=0.5,application/" + "vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.4,application/" + "openmetrics-text;version=0.0.1;q=0.3,application/openmetrics-text;version=1.0.0;q=0.2,*/*;q=0.1", + scrapeConfig.mRequestHeaders["Accept"]); + + // custom quality protocols + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["PrometheusProto", "OpenMetricsText1.0.0", "PrometheusText0.0.4", "OpenMetricsText0.0.1"] + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL( + "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.5," + "application/openmetrics-text;version=1.0.0;q=0.4," + "text/plain;version=0.0.4;q=0.3,application/openmetrics-text;version=0.0.1;q=0.2,*/*;q=0.1", + scrapeConfig.mRequestHeaders["Accept"]); + + // only prometheus0.0.4 protocols + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["PrometheusText0.0.4"] + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL( + "text/plain;version=0.0.4;q=0.2,*/*;q=0.1", + scrapeConfig.mRequestHeaders["Accept"]); + + // Capital error + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["prometheusproto"] + })JSON"; + + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_FALSE(scrapeConfig.Init(config)); + + // OpenMetricsText1.0.0 duplication error + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["OpenMetricsText1.0.0", "PrometheusProto", "OpenMetricsText1.0.0"] + })JSON"; + + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_FALSE(scrapeConfig.Init(config)); + + // protocols invalid + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["OpenMetricsText1.0.0", "PrometheusProto", 999] + })JSON"; + + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_FALSE(scrapeConfig.Init(config)); + + // unknown protocol + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "scrape_protocols": ["OpenMetricsText"] + })JSON"; + + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_FALSE(scrapeConfig.Init(config)); +} + +void ScrapeConfigUnittest::TestEnableCompression() { + Json::Value config; + ScrapeConfig scrapeConfig; + string errorMsg; + string configStr; + + // default + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http" + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL("gzip", scrapeConfig.mRequestHeaders["Accept-Encoding"]); + + // disable + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "enable_compression": false + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); + APSARA_TEST_TRUE(scrapeConfig.Init(config)); + APSARA_TEST_EQUAL("identity", scrapeConfig.mRequestHeaders["Accept-Encoding"]); + + // enable + configStr = R"JSON({ + "job_name": "test_job", + "scrape_interval": "30s", + "scrape_timeout": "30s", + "metrics_path": "/metrics", + "scheme": "http", + "enable_compression": true + })JSON"; + APSARA_TEST_TRUE(ParseJsonTable(configStr, config, errorMsg)); + scrapeConfig.mRequestHeaders.clear(); APSARA_TEST_TRUE(scrapeConfig.Init(config)); - APSARA_TEST_EQUAL(scrapeConfig.mAuthHeaders["Authorization"], "Bearer " + mKey); + APSARA_TEST_EQUAL("gzip", scrapeConfig.mRequestHeaders["Accept-Encoding"]); } UNIT_TEST_CASE(ScrapeConfigUnittest, TestInit); UNIT_TEST_CASE(ScrapeConfigUnittest, TestAuth); UNIT_TEST_CASE(ScrapeConfigUnittest, TestBasicAuth); UNIT_TEST_CASE(ScrapeConfigUnittest, TestAuthorization); +UNIT_TEST_CASE(ScrapeConfigUnittest, TestScrapeProtocols); +UNIT_TEST_CASE(ScrapeConfigUnittest, TestEnableCompression); } // namespace logtail diff --git a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp index 951507b97b..fd3d540a92 100644 --- a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp +++ b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp @@ -56,7 +56,7 @@ class ScrapeSchedulerUnittest : public testing::Test { mScrapeConfig->mScrapeIntervalSeconds = 10; mScrapeConfig->mScrapeTimeoutSeconds = 10; mScrapeConfig->mMetricsPath = "/metrics"; - mScrapeConfig->mAuthHeaders = {{"Authorization", "Bearer xxxxx"}}; + mScrapeConfig->mRequestHeaders = {{"Authorization", "Bearer xxxxx"}}; mHttpResponse.mBody = "# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.\n" From 96afc79e01d1fb3b821f21f4e2fa9265f106b8a1 Mon Sep 17 00:00:00 2001 From: Zhu Shunjia Date: Thu, 19 Sep 2024 11:13:57 +0800 Subject: [PATCH 2/2] feat: provider APIs return multi remote config providers (#1708) * provider APIs return multi remote config providers * restore virtual to VIRTUAL * polish cmake files * add unittest path * update build.md * change vector to map and store the path info Change-Id: I0d049a051c971469e749e6c6478d1e7b43fc5e75 * refactor RemoteConfigProvider APIs Change-Id: I52a04db594048da93014d133632ffb578597aa0d * fix typo Change-Id: Iec15ca29fcbe49eb07e1cd7aa8cf632ad0f815bb * simpify GetRemoteConfigProviders API Change-Id: I18096a8bbc2f62776401bc56ad8893946a9b1b18 --- core/CMakeLists.txt | 20 +++++---- core/application/Application.cpp | 12 +++--- .../CommonConfigProvider.cpp | 2 +- .../CommonConfigProvider.h | 2 +- .../LegacyCommonConfigProvider.cpp | 2 +- .../LegacyCommonConfigProvider.h | 0 core/go_pipeline/LogtailPlugin.cpp | 14 +++---- core/monitor/LogFileProfiler.cpp | 4 +- core/monitor/LogtailAlarm.cpp | 8 ++-- core/monitor/MetricExportor.cpp | 4 +- core/monitor/MetricExportor.h | 2 +- core/monitor/Monitor.cpp | 8 ++-- core/monitor/Monitor.h | 1 + core/plugin/flusher/sls/DiskBufferWriter.cpp | 4 +- core/plugin/flusher/sls/FlusherSLS.cpp | 4 +- core/provider/CMakeLists.txt | 39 +++++++++++++++++ core/provider/Provider.cpp | 41 ++++++++++++++++++ core/provider/Provider.h | 37 ++++++++++++++++ core/unittest/CMakeLists.txt | 2 + .../config/CommonConfigProviderUnittest.cpp | 2 +- core/unittest/provider/CMakeLists.txt | 22 ++++++++++ core/unittest/provider/ProviderUnittest.cpp | 42 +++++++++++++++++++ docs/cn/installation/sources/build.md | 10 +++++ 23 files changed, 240 insertions(+), 42 deletions(-) rename core/config/{provider => common_provider}/CommonConfigProvider.cpp (99%) rename core/config/{provider => common_provider}/CommonConfigProvider.h (98%) rename core/config/{provider => common_provider}/LegacyCommonConfigProvider.cpp (99%) rename core/config/{provider => common_provider}/LegacyCommonConfigProvider.h (100%) create mode 100644 core/provider/CMakeLists.txt create mode 100644 core/provider/Provider.cpp create mode 100644 core/provider/Provider.h create mode 100644 core/unittest/provider/CMakeLists.txt create mode 100644 core/unittest/provider/ProviderUnittest.cpp diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 6c36c26eeb..39073d2271 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -33,6 +33,8 @@ cmake_dependent_option(ENABLE_STATIC_LINK_CRT "Build Logtail by linking CRT stat option(WITHOUTGDB "Build Logtail without gdb") option(WITHSPL "Build Logtail and UT with SPL" ON) option(BUILD_LOGTAIL_UT "Build unit test for Logtail") +set(PROVIDER_PATH "provider" CACHE PATH "Path to the provider module") # external provider path can be set with -DPROVIDER_PATH +set(UNITTEST_PATH "unittest" CACHE PATH "Path to the unittest module") # external unittest path can be set with -DUNITTEST_PATH if (BUILD_LOGTAIL_SHARED_LIBRARY AND WITHSPL) message(FATEL_ERROR, "Generating logtail shared library is not supported to be linked with SPL. WITHSPL should be set OFF.") @@ -109,13 +111,13 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/processor/links.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/flusher.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/links.cmake) -# Subdirectories (modules). except for common, input, processor, flusher, observer, helper and spl. +# Subdirectories (modules). except for common, input, processor, flusher, observer, helper, spl, and provider. set(SUB_DIRECTORIES_LIST application app_config checkpoint container_manager logger go_pipeline monitor profile_sender models - config config/feedbacker config/provider config/watcher + config config/watcher pipeline pipeline/batch pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer runner runner/sink/http - protobuf/config_server/v1 protobuf/config_server/v2 protobuf/sls + protobuf/sls file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling prometheus prometheus/labels prometheus/schedulers prometheus/async ebpf ebpf/observer ebpf/security ebpf/handler @@ -161,6 +163,10 @@ endif() # remove several files in go_pipeline list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.h) + +# add provider +add_subdirectory("${PROVIDER_PATH}" "${CMAKE_BINARY_DIR}/provider") + if(MSVC) # remove linux event listener file(GLOB REMOVE_EVENT_LISTENER_SOURCES file_server/event_listener/*_Linux.cpp file_server/event_listener/*_Linux.h) @@ -231,12 +237,12 @@ if(BUILD_LOGTAIL OR BUILD_LOGTAIL_SHARED_LIBRARY) flusher_link(${LOGTAIL_TARGET}) all_link(${LOGTAIL_TARGET}) common_link(${LOGTAIL_TARGET}) + target_link_libraries(${LOGTAIL_TARGET} provider) endif() # Logtail UT. if (BUILD_LOGTAIL_UT) - message(STATUS "Build unittest.") - add_definitions(-DVIRTUAL=virtual) + message(STATUS "Build unittest.") function(delete_gcda_files target_directory) if(EXISTS "${target_directory}") message(STATUS "Deleting .gcda files in ${target_directory}") @@ -249,7 +255,5 @@ if (BUILD_LOGTAIL_UT) delete_gcda_files(".") include(CTest) enable_testing() - add_subdirectory(unittest) -else() - add_definitions(-DVIRTUAL= ) + add_subdirectory("${UNITTEST_PATH}" "${CMAKE_BINARY_DIR}/unittest") endif () \ No newline at end of file diff --git a/core/application/Application.cpp b/core/application/Application.cpp index d86fc6a92d..3ed86d1e65 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -60,8 +60,7 @@ #include "streamlog/StreamLogManager.h" #endif #else -#include "config/provider/CommonConfigProvider.h" -#include "config/provider/LegacyCommonConfigProvider.h" +#include "provider/Provider.h" #endif DEFINE_FLAG_BOOL(ilogtail_disable_core, "disable core in worker process", true); @@ -242,8 +241,7 @@ void Application::Start() { // GCOVR_EXCL_START EnterpriseConfigProvider::GetInstance()->Init("enterprise"); LegacyConfigProvider::GetInstance()->Init("legacy"); #else - CommonConfigProvider::GetInstance()->Init("common_v2"); - LegacyCommonConfigProvider::GetInstance()->Init("common"); + InitRemoteConfigProviders(); #endif LogtailAlarm::GetInstance()->Init(); @@ -370,8 +368,10 @@ void Application::Exit() { EnterpriseConfigProvider::GetInstance()->Stop(); LegacyConfigProvider::GetInstance()->Stop(); #else - CommonConfigProvider::GetInstance()->Stop(); - LegacyCommonConfigProvider::GetInstance()->Stop(); + auto remoteConfigProviders = GetRemoteConfigProviders(); + for (auto& provider : remoteConfigProviders) { + provider->Stop(); + } #endif LogtailMonitor::GetInstance()->Stop(); diff --git a/core/config/provider/CommonConfigProvider.cpp b/core/config/common_provider/CommonConfigProvider.cpp similarity index 99% rename from core/config/provider/CommonConfigProvider.cpp rename to core/config/common_provider/CommonConfigProvider.cpp index 77e67b2c5f..c96d5e8399 100644 --- a/core/config/provider/CommonConfigProvider.cpp +++ b/core/config/common_provider/CommonConfigProvider.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "config/provider/CommonConfigProvider.h" +#include "CommonConfigProvider.h" #include diff --git a/core/config/provider/CommonConfigProvider.h b/core/config/common_provider/CommonConfigProvider.h similarity index 98% rename from core/config/provider/CommonConfigProvider.h rename to core/config/common_provider/CommonConfigProvider.h index 75f20521b7..804e8cb50c 100644 --- a/core/config/provider/CommonConfigProvider.h +++ b/core/config/common_provider/CommonConfigProvider.h @@ -125,7 +125,7 @@ class CommonConfigProvider : public ConfigProvider, ConfigFeedbackable { ConfigServerAddress GetOneConfigServerAddress(bool changeConfigServer); - VIRTUAL bool SendHttpRequest(const std::string& operation, + virtual bool SendHttpRequest(const std::string& operation, const std::string& reqBody, const std::string& configType, const std::string& requestId, diff --git a/core/config/provider/LegacyCommonConfigProvider.cpp b/core/config/common_provider/LegacyCommonConfigProvider.cpp similarity index 99% rename from core/config/provider/LegacyCommonConfigProvider.cpp rename to core/config/common_provider/LegacyCommonConfigProvider.cpp index 1a017758b6..f5d000711f 100644 --- a/core/config/provider/LegacyCommonConfigProvider.cpp +++ b/core/config/common_provider/LegacyCommonConfigProvider.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "config/provider/LegacyCommonConfigProvider.h" +#include "LegacyCommonConfigProvider.h" #include diff --git a/core/config/provider/LegacyCommonConfigProvider.h b/core/config/common_provider/LegacyCommonConfigProvider.h similarity index 100% rename from core/config/provider/LegacyCommonConfigProvider.h rename to core/config/common_provider/LegacyCommonConfigProvider.h diff --git a/core/go_pipeline/LogtailPlugin.cpp b/core/go_pipeline/LogtailPlugin.cpp index bb90dcdecc..ccff0763fc 100644 --- a/core/go_pipeline/LogtailPlugin.cpp +++ b/core/go_pipeline/LogtailPlugin.cpp @@ -30,7 +30,7 @@ #include "monitor/LogtailAlarm.h" #include "pipeline/PipelineManager.h" #include "pipeline/queue/SenderQueueManager.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" DEFINE_FLAG_BOOL(enable_sls_metrics_format, "if enable format metrics in SLS metricstore log pattern", false); DEFINE_FLAG_BOOL(enable_containerd_upper_dir_detect, @@ -171,22 +171,22 @@ int LogtailPlugin::SendPbV2(const char* configName, FlusherSLS* pConfig = NULL; if (configNameStr == alarmConfig->mLogstore) { pConfig = alarmConfig; - pConfig->mProject = ProfileSender::GetInstance()->GetDefaultProfileProjectName(); - pConfig->mRegion = ProfileSender::GetInstance()->GetDefaultProfileRegion(); + pConfig->mProject = GetProfileSender()->GetDefaultProfileProjectName(); + pConfig->mRegion = GetProfileSender()->GetDefaultProfileRegion(); if (pConfig->mProject.empty()) { return 0; } } else if (configNameStr == profileConfig->mLogstore) { pConfig = profileConfig; - pConfig->mProject = ProfileSender::GetInstance()->GetDefaultProfileProjectName(); - pConfig->mRegion = ProfileSender::GetInstance()->GetDefaultProfileRegion(); + pConfig->mProject = GetProfileSender()->GetDefaultProfileProjectName(); + pConfig->mRegion = GetProfileSender()->GetDefaultProfileRegion(); if (pConfig->mProject.empty()) { return 0; } } else if (configNameStr == containerConfig->mLogstore) { pConfig = containerConfig; - pConfig->mProject = ProfileSender::GetInstance()->GetDefaultProfileProjectName(); - pConfig->mRegion = ProfileSender::GetInstance()->GetDefaultProfileRegion(); + pConfig->mProject = GetProfileSender()->GetDefaultProfileProjectName(); + pConfig->mRegion = GetProfileSender()->GetDefaultProfileRegion(); if (pConfig->mProject.empty()) { return 0; } diff --git a/core/monitor/LogFileProfiler.cpp b/core/monitor/LogFileProfiler.cpp index 701cd03a33..b9dbaf2794 100644 --- a/core/monitor/LogFileProfiler.cpp +++ b/core/monitor/LogFileProfiler.cpp @@ -27,7 +27,7 @@ #include "common/version.h" #include "file_server/ConfigManager.h" #include "logger/Logger.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" #include "pipeline/queue/QueueKeyManager.h" DEFINE_FLAG_INT32(profile_data_send_interval, "interval of send LogFile/DomainSocket profile data, seconds", 600); @@ -261,7 +261,7 @@ void LogFileProfiler::SendProfileData(bool forceSend) { } } UpdateDumpData(logGroup, detail, logstore); - ProfileSender::GetInstance()->SendToProfileProject(region, logGroup); + GetProfileSender()->SendToProfileProject(region, logGroup); } while (true); DumpToLocal(curTime, forceSend, detail, logstore); mLastSendTime = curTime; diff --git a/core/monitor/LogtailAlarm.cpp b/core/monitor/LogtailAlarm.cpp index 65cf05314d..cde2320161 100644 --- a/core/monitor/LogtailAlarm.cpp +++ b/core/monitor/LogtailAlarm.cpp @@ -23,7 +23,7 @@ #include "common/TimeUtil.h" #include "common/version.h" #include "protobuf/sls/sls_logs.pb.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SenderQueueManager.h" @@ -195,7 +195,7 @@ void LogtailAlarm::SendAllRegionAlarm() { // check sender queue status, if invalid jump this region QueueKey alarmPrjLogstoreKey = QueueKeyManager::GetInstance()->GetKey( - "-flusher_sls-" + ProfileSender::GetInstance()->GetProfileProjectName(region) + "#logtail_alarm"); + "-flusher_sls-" + GetProfileSender()->GetProfileProjectName(region) + "#logtail_alarm"); if (!SenderQueueManager::GetInstance()->IsValidToPush(alarmPrjLogstoreKey)) { // jump this region ++sendRegionIndex; @@ -264,7 +264,7 @@ void LogtailAlarm::SendAllRegionAlarm() { continue; } // this is an anonymous send and non lock send - ProfileSender::GetInstance()->SendToProfileProject(region, logGroup); + GetProfileSender()->SendToProfileProject(region, logGroup); } while (true); } @@ -298,7 +298,7 @@ void LogtailAlarm::SendAlarm(const LogtailAlarmType alarmType, } // ignore alarm for profile data - if (ProfileSender::GetInstance()->IsProfileData(region, projectName, category)) { + if (GetProfileSender()->IsProfileData(region, projectName, category)) { return; } // LOG_DEBUG(sLogger, ("Add Alarm", region)("projectName", projectName)("alarm index", diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp index f8a1b10298..e8198d8150 100644 --- a/core/monitor/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -90,10 +90,10 @@ void MetricExportor::SendToSLS(std::map& logGr logGroup->set_source(LogFileProfiler::mIpAddr); logGroup->set_topic(METRIC_TOPIC_TYPE); if (METRIC_REGION_DEFAULT == iter->first) { - ProfileSender::GetInstance()->SendToProfileProject(ProfileSender::GetInstance()->GetDefaultProfileRegion(), + GetProfileSender()->SendToProfileProject(GetProfileSender()->GetDefaultProfileRegion(), *logGroup); } else { - ProfileSender::GetInstance()->SendToProfileProject(iter->first, *logGroup); + GetProfileSender()->SendToProfileProject(iter->first, *logGroup); } delete logGroup; } diff --git a/core/monitor/MetricExportor.h b/core/monitor/MetricExportor.h index dfb524814a..09c3908a3f 100644 --- a/core/monitor/MetricExportor.h +++ b/core/monitor/MetricExportor.h @@ -17,7 +17,7 @@ #pragma once #include "Monitor.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" namespace logtail { diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index 54cf59b0fb..f3bfe2b391 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -50,7 +50,7 @@ #include "config/provider/EnterpriseConfigProvider.h" #endif #include "pipeline/PipelineManager.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" using namespace std; using namespace sls_logs; @@ -299,7 +299,7 @@ bool LogtailMonitor::SendStatusProfile(bool suicide) { AddLogContent(logPtr, "plugin_stats", PipelineManager::GetInstance()->GetPluginStatistics()); // Metrics. vector allProfileRegion; - ProfileSender::GetInstance()->GetAllProfileRegion(allProfileRegion); + GetProfileSender()->GetAllProfileRegion(allProfileRegion); UpdateMetric("region", allProfileRegion); #ifdef __ENTERPRISE__ UpdateMetric("config_update_count", EnterpriseConfigProvider::GetInstance()->GetConfigUpdateTotalCount()); @@ -347,10 +347,10 @@ bool LogtailMonitor::SendStatusProfile(bool suicide) { } if (i == allProfileRegion.size() - 1) { - ProfileSender::GetInstance()->SendToProfileProject(allProfileRegion[i], logGroup); + GetProfileSender()->SendToProfileProject(allProfileRegion[i], logGroup); } else { LogGroup copyLogGroup = logGroup; - ProfileSender::GetInstance()->SendToProfileProject(allProfileRegion[i], copyLogGroup); + GetProfileSender()->SendToProfileProject(allProfileRegion[i], copyLogGroup); } } return true; diff --git a/core/monitor/Monitor.h b/core/monitor/Monitor.h index d0cac3314e..5fcc551dd5 100644 --- a/core/monitor/Monitor.h +++ b/core/monitor/Monitor.h @@ -24,6 +24,7 @@ #include "LogtailMetric.h" #include "MetricConstants.h" #include "MetricStore.h" + #if defined(_MSC_VER) #include #endif diff --git a/core/plugin/flusher/sls/DiskBufferWriter.cpp b/core/plugin/flusher/sls/DiskBufferWriter.cpp index 49bb2789d0..94471e1e54 100644 --- a/core/plugin/flusher/sls/DiskBufferWriter.cpp +++ b/core/plugin/flusher/sls/DiskBufferWriter.cpp @@ -27,7 +27,7 @@ #include "protobuf/sls/sls_logs.pb.h" #include "logger/Logger.h" #include "monitor/LogtailAlarm.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" #include "pipeline/queue/QueueKeyManager.h" #include "pipeline/queue/SLSSenderQueueItem.h" #include "sdk/Exception.h" @@ -765,7 +765,7 @@ SendResult DiskBufferWriter::SendToNetSync(sdk::Client* sendClient, bufferMeta.logstore(), ""); // no region - if (!ProfileSender::GetInstance()->IsProfileData("", bufferMeta.project(), bufferMeta.logstore())) + if (!GetProfileSender()->IsProfileData("", bufferMeta.project(), bufferMeta.logstore())) LOG_ERROR(sLogger, ("send data to SLS fail, error_code", errorCode)("error_message", ex.GetMessage())( "endpoint", sendClient->GetRawSlsHost())("projectName", bufferMeta.project())( diff --git a/core/plugin/flusher/sls/FlusherSLS.cpp b/core/plugin/flusher/sls/FlusherSLS.cpp index 5f965a4535..1650d9fb79 100644 --- a/core/plugin/flusher/sls/FlusherSLS.cpp +++ b/core/plugin/flusher/sls/FlusherSLS.cpp @@ -33,7 +33,7 @@ #include "plugin/flusher/sls/SLSClientManager.h" #include "plugin/flusher/sls/SLSResponse.h" #include "plugin/flusher/sls/SendResult.h" -#include "profile_sender/ProfileSender.h" +#include "provider/Provider.h" #include "runner/FlusherRunner.h" #include "sdk/Common.h" #include "sls_control/SLSControl.h" @@ -610,7 +610,7 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item) auto data = static_cast(item); string configName = HasContext() ? GetContext().GetConfigName() : ""; - bool isProfileData = ProfileSender::GetInstance()->IsProfileData(mRegion, mProject, data->mLogstore); + bool isProfileData = GetProfileSender()->IsProfileData(mRegion, mProject, data->mLogstore); int32_t curTime = time(NULL); auto curSystemTime = chrono::system_clock::now(); if (slsResponse.mStatusCode == 200) { diff --git a/core/provider/CMakeLists.txt b/core/provider/CMakeLists.txt new file mode 100644 index 0000000000..b0e0feb6c3 --- /dev/null +++ b/core/provider/CMakeLists.txt @@ -0,0 +1,39 @@ +# Copyright 2024 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.22) +project(provider) + +file(GLOB LIB_SOURCE_FILES *.cpp *.h) + +set(PROVIDER_SUB_DIRECTORIES_LIST + profile_sender config/feedbacker config/provider config/common_provider protobuf/config_server/v1 protobuf/config_server/v2 +) + +foreach(DIR_NAME IN LISTS PROVIDER_SUB_DIRECTORIES_LIST) + file(GLOB TEMP_SOURCE_FILES + "${CMAKE_CURRENT_SOURCE_DIR}/../${DIR_NAME}/*.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/../${DIR_NAME}/*.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/../${DIR_NAME}/*.c" + ) + list(APPEND LIB_SOURCE_FILES ${TEMP_SOURCE_FILES}) +endforeach() + +add_library(${PROJECT_NAME} STATIC ${LIB_SOURCE_FILES}) + +# message(STATUS "PROVIDER SOURCE FILES: ${LIB_SOURCE_FILES}") + +foreach(DIR_NAME IN LISTS PROVIDER_SUB_DIRECTORIES_LIST) + target_include_directories(provider PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../${DIR_NAME}) +endforeach() \ No newline at end of file diff --git a/core/provider/Provider.cpp b/core/provider/Provider.cpp new file mode 100644 index 0000000000..52523b256c --- /dev/null +++ b/core/provider/Provider.cpp @@ -0,0 +1,41 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Provider.h" + +#include "config/common_provider/CommonConfigProvider.h" +#include "config/common_provider/LegacyCommonConfigProvider.h" + +namespace logtail { + + +std::vector GetRemoteConfigProviders() { + std::vector providers; + providers.push_back(LegacyCommonConfigProvider::GetInstance()); + providers.push_back(CommonConfigProvider::GetInstance()); + return providers; +} + +void InitRemoteConfigProviders() { + CommonConfigProvider::GetInstance()->Init("common_v2"); + LegacyCommonConfigProvider::GetInstance()->Init("common"); +} + +ProfileSender* GetProfileSender() { + return ProfileSender::GetInstance(); +} + +} // namespace logtail \ No newline at end of file diff --git a/core/provider/Provider.h b/core/provider/Provider.h new file mode 100644 index 0000000000..999aaa8a62 --- /dev/null +++ b/core/provider/Provider.h @@ -0,0 +1,37 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "config/provider/ConfigProvider.h" +#include "profile_sender/ProfileSender.h" + +namespace logtail { +/* + providers modules are used to replace the default implementation of ilogtail. +*/ + +// GetRemoteConfigProviders returns a vector of pairs for remote config providers. +// It currently returns two providers: LegacyCommonConfigProvider and CommonConfigProvider. +std::vector GetRemoteConfigProviders(); + +// InitRemoteConfigProviders initializes the remote config providers. +// It currently initializes the LegacyCommonConfigProvider and CommonConfigProvider. +void InitRemoteConfigProviders(); + +// GetProfileSender returns the ProfileSender instance. +ProfileSender* GetProfileSender(); +} // namespace logtail \ No newline at end of file diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 111bff640b..ba42e5bf98 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -44,6 +44,7 @@ macro(add_core_subdir) add_subdirectory(plugin) add_subdirectory(polling) add_subdirectory(processor) + add_subdirectory(provider) add_subdirectory(queue) add_subdirectory(reader) add_subdirectory(sdk) @@ -68,6 +69,7 @@ macro(ut_link ut_link_withspl) flusher_link(${UT_BASE_TARGET}) all_link(${UT_BASE_TARGET}) common_link(${UT_BASE_TARGET}) + target_link_libraries(${UT_BASE_TARGET} provider) endmacro() set(SOURCE_FILES_CORE ${FRAMEWORK_SOURCE_FILES} ${PLUGIN_SOURCE_FILES_CORE}) diff --git a/core/unittest/config/CommonConfigProviderUnittest.cpp b/core/unittest/config/CommonConfigProviderUnittest.cpp index d68f049327..e434210b7c 100644 --- a/core/unittest/config/CommonConfigProviderUnittest.cpp +++ b/core/unittest/config/CommonConfigProviderUnittest.cpp @@ -19,7 +19,7 @@ #include "common/FileSystemUtil.h" #include "common/version.h" #include "config/ConfigDiff.h" -#include "config/provider/CommonConfigProvider.h" +#include "config/common_provider/CommonConfigProvider.h" #include "config/watcher/ConfigWatcher.h" #include "gmock/gmock.h" #include "pipeline/PipelineManager.h" diff --git a/core/unittest/provider/CMakeLists.txt b/core/unittest/provider/CMakeLists.txt new file mode 100644 index 0000000000..e3f7f7abaf --- /dev/null +++ b/core/unittest/provider/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2024 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.22) +project(provider_unittest) + +add_executable(provider_unittest ProviderUnittest.cpp) +target_link_libraries(provider_unittest ${UT_BASE_TARGET}) + +include(GoogleTest) +gtest_discover_tests(provider_unittest) diff --git a/core/unittest/provider/ProviderUnittest.cpp b/core/unittest/provider/ProviderUnittest.cpp new file mode 100644 index 0000000000..7fb455699b --- /dev/null +++ b/core/unittest/provider/ProviderUnittest.cpp @@ -0,0 +1,42 @@ +// Copyright 2023 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "unittest/Unittest.h" +#include "provider/Provider.h" + + +namespace logtail { + +class ProviderUnittest : public testing::Test { +public: + void TestGetRemoteConfigProvider(); + void TestGetProfileSender(); + +}; +void ProviderUnittest::TestGetRemoteConfigProvider() { + auto remoteConfigProviders = GetRemoteConfigProviders(); + APSARA_TEST_GT(remoteConfigProviders.size(), 0); +} + +void ProviderUnittest::TestGetProfileSender() { + auto profileSender = GetProfileSender(); + APSARA_TEST_NOT_EQUAL(nullptr, profileSender); +} + + +UNIT_TEST_CASE(ProviderUnittest, TestGetRemoteConfigProvider) +UNIT_TEST_CASE(ProviderUnittest, TestGetProfileSender) +} // namespace logtail + +UNIT_TEST_MAIN \ No newline at end of file diff --git a/docs/cn/installation/sources/build.md b/docs/cn/installation/sources/build.md index 8eca9dded5..60703e3c0a 100644 --- a/docs/cn/installation/sources/build.md +++ b/docs/cn/installation/sources/build.md @@ -119,3 +119,13 @@ cd 到ilogtail/scripts目录下，执行windows64_build.bat(windows32_build.bat) - PluginAdapter.dll (插件接口) - PluginBase.dll (插件lib) - PluginBase.h + + +## 编译时替换外部模块 +iLogtail通过 Provider 模块暴露出一些拓展点，这些拓展点可以由用户自行实现，并通过编译时CMAKE DPROVIDER_PATH选项替换掉默认的实现。 + +示例： + +```bash +cmake -DPROVIDER_PATH=../../../core_extensions/provider .. +``` \ No newline at end of file