diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp index 7ef47dd1bf..212ca06afc 100644 --- a/core/app_config/AppConfig.cpp +++ b/core/app_config/AppConfig.cpp @@ -54,14 +54,7 @@ void AppConfig::LoadAddrConfig(const Json::Value& confJson) { string host = configServerAddress[0]; int32_t port = atoi(configServerAddress[1].c_str()); - std::string exception; - // regular expressions to verify ip - boost::regex reg_ip - = boost::regex("(?:(?:1[0-9][0-9]\.)|(?:2[0-4][0-9]\.)|(?:25[0-5]\.)|(?:[1-9][0-9]\.)|(?:[0-9]\.)){3}(?" - ":(?:1[0-9][0-9])|(?:2[0-4][0-9])|(?:25[0-5])|(?:[1-9][0-9])|(?:[0-9]))"); - if (!BoostRegexMatch(host.c_str(), reg_ip, exception)) - LOG_WARNING(sLogger, ("ilogtail_configserver_address", "parse fail")("exception", exception)); - else if (port < 1 || port > 65535) + if (port < 1 || port > 65535) LOG_WARNING(sLogger, ("ilogtail_configserver_address", "illegal port")("port", port)); else mConfigServerAddresses.push_back(ConfigServerAddress(host, port)); diff --git a/core/app_config/AppConfigBase.cpp b/core/app_config/AppConfigBase.cpp index 26c581da69..999009d883 100644 --- a/core/app_config/AppConfigBase.cpp +++ b/core/app_config/AppConfigBase.cpp @@ -149,6 +149,7 @@ DECLARE_FLAG_INT32(polling_dir_first_watch_timeout); DECLARE_FLAG_INT32(polling_file_first_watch_timeout); DECLARE_FLAG_INT32(modify_check_interval); DECLARE_FLAG_INT32(ignore_file_modify_timeout); +DEFINE_FLAG_STRING(host_path_blacklist, "host path matches substring in blacklist will be ignored", ""); namespace logtail { @@ -720,6 +721,21 @@ void AppConfigBase::LoadResourceConf(const Json::Value& confJson) { } } + if (!STRING_FLAG(host_path_blacklist).empty()) { +#ifdef _MSC_VER + static const std::string delim = ";"; +#else + static const std::string delim = ":"; +#endif + auto blacklist = SplitString(TrimString(STRING_FLAG(host_path_blacklist)), delim); + for (const auto& s : blacklist) { + auto s1 = TrimString(s); + if (!s1.empty()) { + mHostPathBlacklist.emplace_back(std::move(s1)); + } + } + } + if (!LoadInt32Parameter(mSendDataPort, confJson, "data_server_port", "ALIYUN_LOGTAIL_DATA_SERVER_PORT")) { mSendDataPort = INT32_FLAG(data_server_port); } @@ -1224,4 +1240,12 @@ void AppConfigBase::SetLogtailSysConfDir(const std::string& dirPath) { mUserRemoteYamlConfigDirPath)); } +bool AppConfigBase::IsHostPathMatchBlacklist(const string& dirPath) const { + for (auto& dp : mHostPathBlacklist) { + if (dirPath.find(dp) != std::string::npos) { + return true; + } + } + return false; +} } // namespace logtail diff --git a/core/app_config/AppConfigBase.h b/core/app_config/AppConfigBase.h index b82b1e9e05..48d94241d1 100644 --- a/core/app_config/AppConfigBase.h +++ b/core/app_config/AppConfigBase.h @@ -137,6 +137,8 @@ class AppConfigBase { // local time to adjust logs' time automatically. bool mEnableLogTimeAutoAdjust = false; + std::vector mHostPathBlacklist; + /** * @brief Load ConfigServer, DataServer and network interface * @@ -370,6 +372,8 @@ class AppConfigBase { inline bool EnableLogTimeAutoAdjust() const { return mEnableLogTimeAutoAdjust; } + bool IsHostPathMatchBlacklist(const std::string& dirPath) const; + #ifdef APSARA_UNIT_TEST_MAIN friend class SenderUnittest; friend class ConfigUpdatorUnittest; diff --git a/core/config_manager/ConfigManager.cpp b/core/config_manager/ConfigManager.cpp index a5cccb220e..8339337f53 100644 --- a/core/config_manager/ConfigManager.cpp +++ b/core/config_manager/ConfigManager.cpp @@ -305,7 +305,7 @@ ConfigManager::SendHeartbeat(const AppConfig::ConfigServerAddress& configServerA } catch (const sdk::LOGException& e) { LOG_WARNING( sLogger, - ("SendHeartBeat", "fail")("reqBody", reqBody)("errCode", e.GetErrorCode())("errMsg", e.GetMessage())); + ("SendHeartBeat", "fail")("reqBody", reqBody)("errCode", e.GetErrorCode())("errMsg", e.GetMessage())("host", configServerAddress.host)("port", configServerAddress.port)); return emptyResult; } } diff --git a/core/config_manager/ConfigManagerBase.cpp b/core/config_manager/ConfigManagerBase.cpp index c67388605b..144f1b3099 100644 --- a/core/config_manager/ConfigManagerBase.cpp +++ b/core/config_manager/ConfigManagerBase.cpp @@ -1149,6 +1149,10 @@ bool ConfigManagerBase::LoadJsonConfig(const Json::Value& jsonRoot, bool localFl // if checkTimeout, will not register the dir which is timeout // if not checkTimeout, will register the dir which is timeout and add it to the timeout list bool ConfigManagerBase::RegisterHandlersRecursively(const std::string& path, Config* config, bool checkTimeout) { + if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(path)) { + LOG_INFO(sLogger, ("ignore path matching host path blacklist", path)); + return false; + } bool result = false; if (checkTimeout && config->IsTimeout(path)) return result; @@ -1285,6 +1289,10 @@ bool ConfigManagerBase::RegisterHandlers() { } void ConfigManagerBase::RegisterWildcardPath(Config* config, const string& path, int32_t depth) { + if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(path)) { + LOG_INFO(sLogger, ("ignore path matching host path blacklist", path)); + return; + } bool finish; if ((depth + 1) == ((int)config->mWildcardPaths.size() - 1)) finish = true; @@ -1455,6 +1463,10 @@ bool ConfigManagerBase::RegisterDirectory(const std::string& source, const std:: } bool ConfigManagerBase::RegisterHandlersWithinDepth(const std::string& path, Config* config, int depth) { + if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(path)) { + LOG_INFO(sLogger, ("ignore path matching host path blacklist", path)); + return false; + } if (depth <= 0) { DirCheckPointPtr dirCheckPoint; if (CheckPointManager::Instance()->GetDirCheckPoint(path, dirCheckPoint) == false) @@ -1498,6 +1510,10 @@ bool ConfigManagerBase::RegisterHandlersWithinDepth(const std::string& path, Con // path not terminated by '/', path already registered bool ConfigManagerBase::RegisterDescendants(const string& path, Config* config, int withinDepth) { + if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(path)) { + LOG_INFO(sLogger, ("ignore path matching host path blacklist", path)); + return false; + } if (withinDepth <= 0) { return true; } diff --git a/core/controller/EventDispatcherBase.cpp b/core/controller/EventDispatcherBase.cpp index 883563df38..b609a5f8d6 100644 --- a/core/controller/EventDispatcherBase.cpp +++ b/core/controller/EventDispatcherBase.cpp @@ -161,6 +161,10 @@ EventDispatcherBase::~EventDispatcherBase() { } bool EventDispatcherBase::RegisterEventHandler(const char* path, Config* config, EventHandler*& handler) { + if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(path)) { + LOG_INFO(sLogger, ("ignore path matching host path blacklist", path)); + return false; + } // @todo // if this path belong to many config, if register one config with max_depth 0, then it will register fail if (!config->WithinMaxDepth(path)) { diff --git a/core/polling/PollingDirFile.cpp b/core/polling/PollingDirFile.cpp index 37edf266fd..03a8a48de3 100644 --- a/core/polling/PollingDirFile.cpp +++ b/core/polling/PollingDirFile.cpp @@ -20,6 +20,7 @@ #include #endif #include +#include "app_config/AppConfig.h" #include "common/Flags.h" #include "common/StringTools.h" #include "common/ErrorUtil.h" @@ -342,6 +343,10 @@ bool PollingDirFile::PollingNormalConfigPath( return false; string dirPath = obj.empty() ? srcPath : PathJoin(srcPath, obj); + if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(dirPath)) { + LOG_INFO(sLogger, ("ignore path matching host path blacklist", dirPath)); + return false; + } bool isNewDirectory = false; if (!CheckAndUpdateDirMatchCache(dirPath, statBuf, isNewDirectory)) return true; @@ -464,6 +469,10 @@ bool PollingDirFile::PollingNormalConfigPath( // corresponding value in mConstWildcardPaths, call PollingNormalConfigPath or call // PollingWildcardConfigPath recursively. bool PollingDirFile::PollingWildcardConfigPath(const Config* pConfig, const string& dirPath, int depth) { + if (AppConfig::GetInstance()->IsHostPathMatchBlacklist(dirPath)) { + LOG_INFO(sLogger, ("ignore path matching host path blacklist", dirPath)); + return false; + } auto const wildcardPathSize = static_cast(pConfig->mWildcardPaths.size()); if (depth - wildcardPathSize > pConfig->mMaxDepth) return false; diff --git a/docs/cn/configuration/system-config.md b/docs/cn/configuration/system-config.md index 076475e55c..053377a4e2 100644 --- a/docs/cn/configuration/system-config.md +++ b/docs/cn/configuration/system-config.md @@ -13,6 +13,7 @@ | `config_update_interval` | Int | 本地配置热加载的更新间隔,单位为秒。
**注意:此参数仅对社区版有效。** | | `data_server_port` | Int |

用于控制 `flusher_sls` 往 `SLS` 发送的协议类型。

取值范围:443(默认),表示使用 `HTTPS` 协议发送;80表示使用 `HTTP` 协议发送。

如果使用`SLS`内网域名写入,建议使用`HTTP`协议发送,提高传输性能。

| | `send_running_status` | Bool | 为了更好的了解 `iLogtail` 的使用情况,以便做出更有针对性的发展规划,`iLogtail` 会上报一些脱敏后的运行统计信息。您也可以手动关闭此开关。 | +| `host_path_blacklist` | String | 全局主机路径黑名单,黑名单为子串匹配,Linux下多个子串以:分隔,Windows下以;分隔。比如禁止采集NAS挂载,可以配置为`/volumes/kubernetes.io~csi/nas-`。 ### 典型配置 diff --git a/scripts/gen_build_scripts.sh b/scripts/gen_build_scripts.sh index a901475dc2..5c8446ca27 100755 --- a/scripts/gen_build_scripts.sh +++ b/scripts/gen_build_scripts.sh @@ -60,7 +60,7 @@ ram_limit_nproc=\$((ram_size / 1024 / 768)) EOF if [ $EXPORT_GO_ENVS ]; then - envs=($(go env | grep -E 'GOPRIVATE=".+"|GOPROXY=".+"')) + envs=($(go env | grep -E 'GOPRIVATE=(".+"|'\''.+'\'')|GOPROXY=(".+"|'\''.+'\'')')) for v in ${envs[@]}; do echo "go env -w $v" >> $BUILD_SCRIPT_FILE done