Merge branch 'main' into go-config

.github/workflows/build-core-ut.yaml

@@ -82,7 +82,7 @@ jobs:
         run: make unittest_core
 
       - name: Unit Test Coverage
-        run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --root . --lcov coverage.lcov --txt coverage.txt -e \".*sdk.*\" -e \".*observer.*\" -e \".*log_pb.*\" -e \".*unittest.*\" -e \".*config_server.*\" -e \".*fuse.*\" -e \".*go_pipeline.*\""
+        run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --root . --lcov coverage.lcov --txt coverage.txt -e \".*sdk.*\" -e \".*observer.*\" -e \".*protobuf.*\" -e \".*unittest.*\" -e \".*config_server.*\" -e \".*fuse.*\" -e \".*go_pipeline.*\""
 
       - name: Setup Python3.10
         uses: actions/setup-python@v5

.gitignore

@@ -55,7 +55,7 @@ _deps
 # Custom
 /build/
 core/build/
-core/log_pb/*.pb.*
+core/protobuf/sls/*.pb.*
 core/common/Version.cpp
 !/Makefile
 # Enterprise

config_server/protocol/v2/README.md

@@ -22,7 +22,7 @@
         string running_status = 8;                  // Human readable running status
         int64 startup_time = 9;                     // Required, Agent's startup time
         repeated ConfigInfo pipeline_configs = 10;  // Information about the current PIPELINE_CONFIG held by the Agent
-        repeated ConfigInfo process_configs = 11;   // Information about the current AGENT_CONFIG held by the Agent
+        repeated ConfigInfo instance_configs = 11;   // Information about the current AGENT_CONFIG held by the Agent
         repeated CommandInfo custom_commands = 12;  // Information about command history
         uint64 flags = 13;                          // Predefined command flag
         bytes opaque = 14;                          // Opaque data for extension
@@ -74,8 +74,8 @@
         UnspecifiedAgentCapability = 0;
         // The Agent can accept pipeline configuration from the Server.
         AcceptsPipelineConfig          = 0x00000001;
-        // The Agent can accept process configuration from the Server.
-        AcceptsProcessConfig           = 0x00000002;
+        // The Agent can accept instance configuration from the Server.
+        AcceptsInstanceConfig           = 0x00000002;
         // The Agent can accept custom command from the Server.
         AcceptsCustomCommand           = 0x00000004;
 
@@ -100,7 +100,7 @@
         uint64 capabilities = 3;                            // Bitmask of flags defined by ServerCapabilities enum
 
         repeated ConfigDetail pipeline_config_updates = 4;  // Agent's pipeline config update status
-        repeated ConfigDetail process_config_updates = 5;   // Agent's process config update status
+        repeated ConfigDetail instance_config_updates = 5;   // Agent's instance config update status
         repeated CommandDetail custom_command_updates = 6;  // Agent's commands updates
         uint64 flags = 7;                                   // Predefined command flag
         bytes opaque = 8;                                   // Opaque data for extension
@@ -126,8 +126,8 @@
         RembersAttribute                   = 0x00000001;
         // The Server can remember pipeline config status.
         RembersPipelineConfigStatus        = 0x00000002;
-        // The Server can remember process config status.
-        RembersProcessConfigStatus         = 0x00000004;
+        // The Server can remember instance config status.
+        RembersInstanceConfigStatus         = 0x00000004;
         // The Server can remember custom command status.
         RembersCustomCommandStatus         = 0x00000008;
 
@@ -150,7 +150,7 @@
         // restarted and lost state).
         ReportFullState           = 0x00000001;
         FetchPipelineConfigDetail = 0x00000002;
-        FetchProcessConfigDetail  = 0x00000004;
+        FetchInstanceConfigDetail  = 0x00000004;
         // bits before 2^16 (inclusive) are reserved for future official fields
     }
 
@@ -168,7 +168,7 @@ Server：应当通过capbilitiies上报Server自身的能力，这样如果新
 
 Client：Agent启动后第一次向Server汇报全量信息，request字段应填尽填。request\_id、sequence\_num、capabilities、instance\_id、agent\_type、startup\_time为必填字段。
 
-Server：Server根据上报的信息返回响应。pipeline\_config\_updates、process\_config\_updates中包含agent需要同步的配置，updates中必然包含name和version，是否包含详情context和detail取决于server端实现。custom\_command_updates包含要求agent执行的命令command中必然包含type、name和expire\_time。
+Server：Server根据上报的信息返回响应。pipeline\_config\_updates、instance\_config\_updates中包含agent需要同步的配置，updates中必然包含name和version，是否包含详情context和detail取决于server端实现。custom\_command_updates包含要求agent执行的命令command中必然包含type、name和expire\_time。
 
 Server是否保存Client信息取决于Server实现，如果服务端找不到或保存的sequence\_num + 1 ≠ 心跳的sequence\_num，那么就立刻返回并且flags中必须设置ReportFullStatus标识位。
 
@@ -192,15 +192,15 @@ Server：同注册
 
 ### 进程配置
 
-若Server的注册/心跳响应中有process\_config\_updates.detail
+若Server的注册/心跳响应中有instance\_config\_updates.detail
 
 Client：直接从response中获得detail，应用成功后下次心跳需要上报完整状态。
 
 若Server的响应不包含detail
 
-Client：根据process\_config\_updates的信息构造FetchProcessConfigRequest
+Client：根据instance\_config\_updates的信息构造FetchInstanceConfigRequest
 
-Server：返回FetchProcessConfigResponse
+Server：返回FetchInstanceConfigResponse
 
 Client获取到多个进程配置时，自动合并，若产生冲突默认行为是未定义。
 

config_server/protocol/v2/agent.proto

@@ -49,8 +49,8 @@ enum AgentCapabilities {
     UnspecifiedAgentCapability = 0;
     // The Agent can accept pipeline configuration from the Server.
     AcceptsPipelineConfig          = 0x00000001;
-    // The Agent can accept process configuration from the Server.
-    AcceptsProcessConfig           = 0x00000002;
+    // The Agent can accept instance configuration from the Server.
+    AcceptsInstanceConfig           = 0x00000002;
     // The Agent can accept custom command from the Server.
     AcceptsCustomCommand           = 0x00000004;
 
@@ -81,7 +81,7 @@ message HeartbeatRequest {
     string running_status = 8;                  // Human readable running status
     int64 startup_time = 9;                     // Required, Agent's startup time
     repeated ConfigInfo pipeline_configs = 10;  // Information about the current PIPELINE_CONFIG held by the Agent
-    repeated ConfigInfo process_configs = 11;   // Information about the current AGENT_CONFIG held by the Agent
+    repeated ConfigInfo instance_configs = 11;  // Information about the current AGENT_CONFIG held by the Agent
     repeated CommandInfo custom_commands = 12;  // Information about command history
     uint64 flags = 13;                          // Predefined command flag
     bytes opaque = 14;                          // Opaque data for extension
@@ -102,20 +102,15 @@ message CommandDetail {
     int64 expire_time = 4;          // After which the command can be safely removed from history
 }
 
-message ServerErrorResponse {
-    int32 error_code = 1;           // None-zero value indicates error
-    string error_message = 2;       // Error message
-}
-
 enum ServerCapabilities {
     // The capabilities field is unspecified.
     UnspecifiedServerCapability = 0;
     // The Server can remember agent attributes.
     RembersAttribute                   = 0x00000001;
     // The Server can remember pipeline config status.
     RembersPipelineConfigStatus        = 0x00000002;
-    // The Server can remember process config status.
-    RembersProcessConfigStatus         = 0x00000004;
+    // The Server can remember instance config status.
+    RembersInstanceConfigStatus         = 0x00000004;
     // The Server can remember custom command status.
     RembersCustomCommandStatus         = 0x00000008;
 
@@ -133,25 +128,25 @@ enum ResponseFlags {
     // restarted and lost state).
     ReportFullState           = 0x00000001;
     FetchPipelineConfigDetail = 0x00000002;
-    FetchProcessConfigDetail  = 0x00000004;
+    FetchInstanceConfigDetail = 0x00000004;
     // bits before 2^16 (inclusive) are reserved for future official fields
 }
 
 // ConfigServer's response to Agent's request
 message HeartbeatResponse {
     bytes request_id = 1;  
-    ServerErrorResponse error_response = 2;             // Set value indicates error
+    CommonResponse commonResponse = 2;                  // Set common response
     uint64 capabilities = 3;                            // Bitmask of flags defined by ServerCapabilities enum
 
     repeated ConfigDetail pipeline_config_updates = 4;  // Agent's pipeline config update status
-    repeated ConfigDetail process_config_updates = 5;   // Agent's process config update status
+    repeated ConfigDetail instance_config_updates = 5;  // Agent's instance config update status
     repeated CommandDetail custom_command_updates = 6;  // Agent's commands updates
     uint64 flags = 7;                                   // Predefined command flag
     bytes opaque = 8;                                   // Opaque data for extension
 }
 
 // API: /Agent/FetchPipelineConfig/
-// API: /Agent/FetchProcessConfig/
+// API: /Agent/FetchInstanceConfig/
 // Agent request to ConfigServer, pulling details of the config
 message FetchConfigRequest {
     bytes request_id = 1; 
@@ -162,6 +157,12 @@ message FetchConfigRequest {
 // ConfigServer response to Agent's request
 message FetchConfigResponse {
     bytes request_id = 1;                     
-    ServerErrorResponse error_response = 2;
+    CommonResponse commonResponse = 2;
     repeated ConfigDetail config_details = 3;   // config detail
+}
+
+message CommonResponse
+{
+    int32 status = 1;
+    bytes errorMessage = 2;
 }

core/CMakeLists.txt

@@ -102,19 +102,24 @@ set(PLUGIN_SOURCE_FILES_CORE "")
 set(PLUGIN_SOURCE_FILES_SPL "")
 include(${CMAKE_CURRENT_SOURCE_DIR}/common/common.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/common/links.cmake)
-include(${CMAKE_CURRENT_SOURCE_DIR}/input/input.cmake)
-include(${CMAKE_CURRENT_SOURCE_DIR}/input/links.cmake)
-include(${CMAKE_CURRENT_SOURCE_DIR}/processor/processor.cmake)
-include(${CMAKE_CURRENT_SOURCE_DIR}/processor/links.cmake)
-include(${CMAKE_CURRENT_SOURCE_DIR}/flusher/flusher.cmake)
-include(${CMAKE_CURRENT_SOURCE_DIR}/flusher/links.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/input/input.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/input/links.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/processor/processor.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/processor/links.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/flusher.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/links.cmake)
 
 # Subdirectories (modules). except for common, input, processor, flusher, observer, helper and spl.
 set(SUB_DIRECTORIES_LIST
-        batch application app_config checkpoint compression config config/feedbacker config/provider config/watcher config_manager config_server_pb/v1 config_server_pb/v2 
-        container_manager controller event event_handler event_listener file_server go_pipeline log_pb logger 
-        models monitor parser pipeline plugin plugin/creator plugin/instance plugin/interface polling
-        profile_sender queue reader sdk sender serializer sls_control fuse prometheus prometheus/labels prometheus/schedulers prometheus/async sink/http route ebpf/observer ebpf/security ebpf/handler ebpf runner
+        application app_config checkpoint container_manager logger go_pipeline monitor profile_sender models
+        config config/feedbacker config/provider config/watcher
+        pipeline pipeline/batch pipeline/compression pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer
+        runner runner/sink/http
+        protobuf/config_server/v1 protobuf/config_server/v2 protobuf/sls
+        file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling
+        prometheus prometheus/labels prometheus/schedulers prometheus/async
+        ebpf ebpf/observer ebpf/security ebpf/handler
+        parser sls_control sdk fuse
         )
 if (LINUX)
     if (ENABLE_ENTERPRISE)
@@ -158,11 +163,11 @@ list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/
 
 if(MSVC)
     # remove linux event listener
-    file(GLOB REMOVE_EVENT_LISTENER_SOURCES event_listener/*_Linux.cpp event_listener/*_Linux.h)
+    file(GLOB REMOVE_EVENT_LISTENER_SOURCES file_server/event_listener/*_Linux.cpp file_server/event_listener/*_Linux.h)
     list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${REMOVE_EVENT_LISTENER_SOURCES})
 elseif(UNIX)
     # remove windows event listener
-    file(GLOB REMOVE_EVENT_LISTENER_SOURCES event_listener/*_Windows.cpp event_listener/*_Windows.h)
+    file(GLOB REMOVE_EVENT_LISTENER_SOURCES file_server/event_listener/*_Windows.cpp file_server/event_listener/*_Windows.h)
     list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${REMOVE_EVENT_LISTENER_SOURCES})
     if (LINUX)
         # observer

core/app_config/AppConfig.cpp

@@ -21,12 +21,12 @@
 #include "common/JsonUtil.h"
 #include "common/LogtailCommonFlags.h"
 #include "common/RuntimeUtil.h"
-#include "config_manager/ConfigManager.h"
+#include "file_server/ConfigManager.h"
 #include "logger/Logger.h"
 #include "monitor/LogFileProfiler.h"
 #include "monitor/LogtailAlarm.h"
 #include "monitor/Monitor.h"
-#include "reader/LogFileReader.h"
+#include "file_server/reader/LogFileReader.h"
 #ifdef __ENTERPRISE__
 #include "config/provider/EnterpriseConfigProvider.h"
 #endif
@@ -149,7 +149,9 @@ DEFINE_FLAG_INT32(max_holded_data_size,
 DEFINE_FLAG_INT32(pub_max_holded_data_size,
                   "for every id and metric name, the max data size can be holded in memory (default 512KB)",
                   512 * 1024);
-DEFINE_FLAG_STRING(metrics_report_method, "method to report metrics (default none, means logtail will not report metrics)", "sls");
+DEFINE_FLAG_STRING(metrics_report_method,
+                   "method to report metrics (default none, means logtail will not report metrics)",
+                   "sls");
 
 DEFINE_FLAG_STRING(loong_collector_operator_service, "loong collector operator service", "");
 DEFINE_FLAG_INT32(loong_collector_operator_service_port, "loong collector operator service port", 8888);
@@ -1055,8 +1057,8 @@ void AppConfig::RecurseParseJsonToFlags(const Json::Value& confJson, std::string
                 SetConfigFlag(fullName, jsonvalue.toStyledString());
             } else {
                 APSARA_LOG_INFO(sLogger,
-                            ("Set config flag failed", "can not convert json value to flag")("flag name", fullName)(
-                                "jsonvalue", jsonvalue.toStyledString()));
+                                ("Set config flag failed", "can not convert json value to flag")("flag name", fullName)(
+                                    "jsonvalue", jsonvalue.toStyledString()));
             }
         }
     }

core/app_config/AppConfig.h

@@ -24,7 +24,7 @@
 #include <vector>
 
 #include "common/Lock.h"
-#include "log_pb/sls_logs.pb.h"
+#include "protobuf/sls/sls_logs.pb.h"
 
 namespace logtail {