add alarm

alibaba · Dec 11, 2023 · e5c8af9 · e5c8af9
1 parent 33fe1d0
commit e5c8af9
Show file tree

Hide file tree

Showing 43 changed files with 1,639 additions and 444 deletions.
diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp
@@ -450,7 +450,7 @@ void AppConfig::LoadResourceConf(const Json::Value& confJson) {
         mSendRequestConcurrency = confJson["send_request_concurrency"].asInt();
     else
         mSendRequestConcurrency = INT32_FLAG(send_request_concurrency);
-    LogtailMonitor::Instance()->UpdateConstMetric("send_request_concurrency", mSendRequestConcurrency);
+    LogtailMonitor::GetInstance()->UpdateConstMetric("send_request_concurrency", mSendRequestConcurrency);
 
     if (confJson.isMember("process_thread_count") && confJson["process_thread_count"].isInt())
         mProcessThreadCount = confJson["process_thread_count"].asInt();

diff --git a/core/application/Application.cpp b/core/application/Application.cpp
@@ -50,6 +50,7 @@ DEFINE_FLAG_INT32(file_tags_update_interval, "second", 1);
 DEFINE_FLAG_INT32(config_scan_interval, "seconds", 10);
 DEFINE_FLAG_INT32(profiling_check_interval, "seconds", 60);
 DEFINE_FLAG_INT32(tcmalloc_release_memory_interval, "force release memory held by tcmalloc, seconds", 300);
+DEFINE_FLAG_INT32(exit_flushout_duration, "exit process flushout duration", 20 * 1000);
 
 DECLARE_FLAG_BOOL(send_prefer_real_ip);
 DECLARE_FLAG_BOOL(global_network_success);
@@ -112,7 +113,7 @@ void Application::Init() {
     const string& configIP = AppConfig::GetInstance()->GetConfigIP();
     if (!configIP.empty()) {
         LogFileProfiler::mIpAddr = configIP;
-        LogtailMonitor::Instance()->UpdateConstMetric("logtail_ip", GetHostIp());
+        LogtailMonitor::GetInstance()->UpdateConstMetric("logtail_ip", GetHostIp());
     } else if (!interface.empty()) {
         LogFileProfiler::mIpAddr = GetHostIp(interface);
         if (LogFileProfiler::mIpAddr.empty()) {
@@ -130,7 +131,7 @@ void Application::Init() {
     const string& configHostName = AppConfig::GetInstance()->GetConfigHostName();
     if (!configHostName.empty()) {
         LogFileProfiler::mHostname = configHostName;
-        LogtailMonitor::Instance()->UpdateConstMetric("logtail_hostname", GetHostName());
+        LogtailMonitor::GetInstance()->UpdateConstMetric("logtail_hostname", GetHostName());
     }
 
     int32_t systemBootTime = AppConfig::GetInstance()->GetSystemBootTime();
@@ -166,15 +167,16 @@ void Application::Init() {
 }
 
 void Application::Start() {
-    LogtailMonitor::Instance()->UpdateConstMetric("start_time", GetTimeStamp(time(NULL), "%Y-%m-%d %H:%M:%S"));
+    LogtailMonitor::GetInstance()->UpdateConstMetric("start_time", GetTimeStamp(time(NULL), "%Y-%m-%d %H:%M:%S"));
 
 #if defined(__ENTERPRISE__) && defined(_MSC_VER)
     InitWindowsSignalObject();
 #endif
     // flusher_sls should always be loaded, since profiling will rely on this.
-    Sender::Instance()->InitSender();
+    Sender::Instance()->Init();
 
-    LogtailMonitor::Instance()->InitMonitor();
+    LogtailAlarm::GetInstance()->Init();
+    LogtailMonitor::GetInstance()->Init();
 
     // add local config dir
     filesystem::path localConfigPath
@@ -280,9 +282,27 @@ bool Application::TryGetUUID() {
 }
 
 void Application::Exit() {
+#if defined(__ENTERPRISE__) && defined(__linux__)
+    if (AppConfig::GetInstance()->ShennongSocketEnabled()) {
+        ShennongManager::GetInstance()->Stop();
+    }
+#endif
+
     PipelineManager::GetInstance()->StopAllPipelines();
+
     PluginRegistry::GetInstance()->UnloadPlugins();
+
+    LogtailMonitor::GetInstance()->Stop();
     LogtailAlarm::GetInstance()->Stop();
+    // from now on, alarm should not be used.
+
+    if (!(Sender::Instance()->FlushOut(INT32_FLAG(exit_flushout_duration)))) {
+        LOG_WARNING(sLogger, ("flush SLS sender data", "failed"));
+    } else {
+        LOG_INFO(sLogger, ("flush SLS sender data", "succeeded"));
+    }
+
+
 #if defined(_MSC_VER)
     ReleaseWindowsSignalObject();
 #endif
@@ -335,7 +355,7 @@ void Application::CheckCriticalCondition(int32_t curTime) {
         _exit(1);
     }
 
-    LogtailMonitor::Instance()->UpdateMetric("last_send_time", GetTimeStamp(lastSendTime, "%Y-%m-%d %H:%M:%S"));
+    LogtailMonitor::GetInstance()->UpdateMetric("last_send_time", GetTimeStamp(lastSendTime, "%Y-%m-%d %H:%M:%S"));
 }
 
 bool Application::GetUUIDThread() {

diff --git a/core/common/ParamExtractor.h b/core/common/ParamExtractor.h
@@ -26,32 +26,59 @@
 
 #include "common/StringTools.h"
 #include "logger/Logger.h"
+#include "monitor/LogtailAlarm.h"
 
-#define PARAM_ERROR_RETURN(logger, msg, module, config) \
+#define PARAM_ERROR_RETURN(logger, alarm, msg, module, config, project, logstore, region) \
     if (module.empty()) { \
         LOG_ERROR(logger, ("failed to parse config", msg)("config", config)); \
+        alarm.SendAlarm( \
+            CATEGORY_CONFIG_ALARM, std::string(msg) + ": abort, config: " + config, project, logstore, region); \
     } else { \
         LOG_ERROR(logger, ("failed to parse config", msg)("module", module)("config", config)); \
+        alarm.SendAlarm(CATEGORY_CONFIG_ALARM, \
+                        std::string(msg) + ": abort, module: " + module + ", config: " + config, \
+                        project, \
+                        logstore, \
+                        region); \
     } \
     return false
-#define PARAM_WARNING_IGNORE(logger, msg, module, config) \
+#define PARAM_WARNING_IGNORE(logger, alarm, msg, module, config, project, logstore, region) \
     if (module.empty()) { \
         LOG_WARNING(logger, \
                     ("problem encountered in config parsing", msg)("action", "ignore param")("config", config)); \
+        alarm.SendAlarm( \
+            CATEGORY_CONFIG_ALARM, std::string(msg) + ": ignore param, config: " + config, project, logstore, region); \
     } else { \
         LOG_WARNING(logger, \
                     ("problem encountered in config parsing", \
                      msg)("action", "ignore param")("module", module)("config", config)); \
+        alarm.SendAlarm(CATEGORY_CONFIG_ALARM, \
+                        std::string(msg) + ": ignore param, module: " + module + ", config: " + config, \
+                        project, \
+                        logstore, \
+                        region); \
     }
-#define PARAM_WARNING_DEFAULT(logger, msg, val, module, config) \
+#define PARAM_WARNING_DEFAULT(logger, alarm, msg, val, module, config, project, logstore, region) \
     if (module.empty()) { \
         LOG_WARNING(logger, \
                     ("problem encountered in config parsing", \
                      msg)("action", "use default value instead")("default value", ToString(val))("config", config)); \
+        alarm.SendAlarm(CATEGORY_CONFIG_ALARM, \
+                        std::string(msg) + ": use default value instead, default value: " + ToString(val) \
+                            + ", config: " + config, \
+                        project, \
+                        logstore, \
+                        region); \
     } else { \
         LOG_WARNING(logger, \
                     ("problem encountered in config parsing", msg)("action", "use default value instead")( \
                         "default value", ToString(val))("module", module)("config", config)); \
+        alarm.SendAlarm(CATEGORY_CONFIG_ALARM, \
+                        std::string(msg) + ": use default value instead, default value: " + ToString(val) \
+                            + ", module: " + module + ", config: " + config, \
+                        project, \
+                        logstore, \
+                        region); \
     }
 
 namespace logtail {