Merge branch 'main' into fix/skywalking-input/miss-metric

.devcontainer/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail-build-linux:gcc_9.3.1-1
+FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail-build-linux:gcc_9.3.1-2
 
 ARG USERNAME=admin
 USER root
@@ -34,4 +34,4 @@ RUN source /tmp/.env && rm /tmp/.env; \
 
 USER $USERNAME
 
-RUN go env -w GO111MODULE=on && go env -w GOPROXY=https://goproxy.cn,direct
+RUN go env -w GO111MODULE=on && go env -w GOPROXY=https://goproxy.cn,direct

.github/workflows/build.yaml

@@ -80,6 +80,7 @@ jobs:
           BUILD_LOGTAIL_UT: OFF
           ENABLE_COMPATIBLE_MODE: ON
           ENABLE_STATIC_LINK_CRT: ON
+          WITHOUTGDB: ON
         run: make dist && scripts/check_glibc.sh
 
       - name: Build Docker

.github/workflows/e2e-core.yaml

@@ -67,6 +67,9 @@ jobs:
           sudo chmod +x /usr/local/bin/docker-compose
 
       - name: E2E Core Structure Test
+        env:
+          BUILD_LOGTAIL_UT: OFF
+          WITHOUTGDB: ON
         run: make e2e-core
 
   result:

.github/workflows/e2e.yaml

@@ -72,6 +72,9 @@ jobs:
         run: docker --version
 
       - name: E2E Test
+        env:
+          BUILD_LOGTAIL_UT: OFF
+          WITHOUTGDB: ON
         run: make e2e
 
       - name: UnitTest E2e Engine

CHANGELOG.md

@@ -36,30 +36,13 @@ your changes, such as:
 - [public] [both] [updated] add a new feature
 
 ## [Unreleased]
-
-- [public] [both] [added] refactoried C++ process pipeline
-- [public] [both] [added] support use accelerate processors with go processors
-- [public] [both] [added] add new logtail metric module
-- [public] [both] [added] ddd JSON flatten protocol, data can be flattened and then brushed into storage such as Kafka and ES.
-- [public] [both] [added] use env `LOGTAIL_LOG_LEVEL` to control ilogtail log level
-- [public] [both] [updated] support continue/end regex patterns to split multiline log
-- [public] [both] [updated] support reader flush timeout
-- [public] [both] [updated] Flusher Kafka V2: support send the message with headers to kafka
-- [public] [both] [updated] update gcc version to 9.3.1
-- [public] [both] [updated] add make flag WITHOUTGDB
-- [public] [both] [updated] cache incomplete line in memory to avoid repeated read system call
-- [public] [both] [fixed] Add APSARA\_LOG\_TRACE to solve the problem of not being able to find LOG\_TRACE.
-- [public] [both] [fixed] fix multiline is splitted if not flushed to disk together
-- [public] [both] [fixed] fix line is truncated if \0 is in the middle of line
-- [public] [both] [fixed] container cannot exit for file reopened by checkpoint
-- [public] [both] [fixed] fix filename being mismatched to the deleted file if the deleted file size is 0 and their inode is same
-- [public] [both] [fixed] fix config server panic caused by concurrent read and write shared object
-- [public] [both] [fixed] timezone adjust not working with apsara\_log
-- [public] [both] [added] support plugin ProcessorParseTimestampNative
-- [public] [both] [added] support plugin ProcessorOtelMetric
-- [public] [both] [updated] skywalking plugin support to capture `db.connection_string` tag 
-- [public] [both] [added] support plugin ProcessorParseApsaraNative
-- [public] [both] [added] support plugin ProcessorParseDelimiterNative
-- [public] [both] [added] support plugin ProcessorFilterNative
-- [public] [both] [added] support plugin ProcessorDesensitizeNative
 - [public] [both] [fixed] fix dropping jvm metrics when collecting multiple skywalking instances's data
+- [public] [both] [fixed] fix elasticsearch flusher authentication tls config and http config
+- [public] [both] [fixed] fix profiling wrong type when the different profiling type having same stack. 
+- [public] [both] [added] add UsingOldContentTag. When UsingOldContentTag is set to false, the Tag is now placed in the Meta instead of Logs during cgo.
+- [public] [both] [fixed] fix send local buffer failed when upgrade iLogtail from version earlier than 1.3.
+- [public] [both] [updated] Updated strptime_ns to parse %c format from "%x %X" to "%a %b %d %H:%M:%S %Y" for consistent behavior with striptime.
+- [public] [both] [fixed] fix topic key does not support underscore.
+- [public] [both] [fixed] fix jmxfetch status error when exist multi jmxfetch config in the same machine.
+- [public] [both] [fixed] fix increasing WSS memory issue in collected containers.
+- [public] [both] [fixed] fix cannot log blacklist config error

Makefile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 .DEFAULT_GOAL := all
-VERSION ?= 1.7.1
+VERSION ?= 1.8.0
 DOCKER_PUSH ?= false
 DOCKER_REPOSITORY ?= aliyun/ilogtail
 BUILD_REPOSITORY ?= aliyun/ilogtail_build

changes/v1.8.0.md

@@ -0,0 +1,64 @@
+# 1.8.0
+
+## Changes
+
+All issues and pull requests are [here](https://github.com/alibaba/ilogtail/milestone/21).
+
+### Features
+
+- [public] [both] [added] refactoried C++ process pipeline
+- [public] [both] [added] support use accelerate processors with go processors
+- [public] [both] [added] add new logtail metric module
+- [public] [both] [added] ddd JSON flatten protocol, data can be flattened and then brushed into storage such as Kafka and ES.
+- [public] [both] [added] use env `LOGTAIL_LOG_LEVEL` to control ilogtail log level
+- [public] [both] [added] add global host path blacklist
+- [public] [both] [updated] support continue/end regex patterns to split multiline log
+- [public] [both] [updated] support reader flush timeout
+- [public] [both] [updated] flusher Kafka V2: support send the message with headers to kafka
+- [public] [both] [updated] sls flusher supports nanosecond timestamp
+- [public] [both] [updated] update gcc version to 9.3.1
+- [public] [both] [updated] add make flag WITHOUTGDB
+- [public] [both] [updated] cache incomplete line in memory to avoid repeated read system call
+- [public] [both] [updated] unify metrics format
+- [public] [both] [updated] verify host ip against NIC to get more accurate host ip
+- [public] [both] [added] support plugin ProcessorParseTimestampNative
+- [public] [both] [added] support plugin processor_otel_metric
+- [public] [both] [updated] skywalking plugin support to capture `db.connection_string` tag
+- [public] [both] [added] support plugin ProcessorParseApsaraNative
+- [public] [both] [added] support plugin ProcessorParseDelimiterNative
+- [public] [both] [added] support plugin ProcessorFilterNative
+- [public] [both] [added] support plugin ProcessorDesensitizeNative
+- [public] [both] [added] add resource tag for env created logtail config
+
+### Fixed
+
+- [public] [both] [fixed] remove the ip restrict of config server address, allow domain and hostname
+- [public] [both] [fixed] add APSARA\_LOG\_TRACE to solve the problem of not being able to find LOG\_TRACE.
+- [public] [both] [fixed] fix collection duplication for statefulset sidecar that may mount same volume on different nodes
+- [public] [both] [fixed] fix multiline is splitted if not flushed to disk together
+- [public] [both] [fixed] fix line is truncated if \0 is in the middle of line
+- [public] [both] [fixed] container cannot exit for file reopened by checkpoint
+- [public] [both] [fixed] fix filename being mismatched to the deleted file if the deleted file size is 0 and their inode is same
+- [public] [both] [fixed] fix config server panic caused by concurrent read and write shared object
+- [public] [both] [fixed] add expand array switch in json processor for backward compatibility
+- [public] [both] [fixed] timezone adjust not working with apsara\_log
+- [public] [both] [fixed] fix parse json without newline may result in partial parsing
+- [public] [both] [fixed] fix json does not split correctly if beginning buffer contains invalid json
+
+### Doc
+
+
+## Download
+
+| **Filename** | **OS** | **Arch** | **SHA256 Checksum** |
+|  ----  | ----  | ----  | ----  |
+|[ilogtail-1.8.0.linux-amd64.tar.gz](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.linux-amd64.tar.gz)|Linux|x86-64|[ilogtail-1.8.0.linux-amd64.tar.gz.sha256](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.linux-amd64.tar.gz.sha256)|
+|[ilogtail-1.8.0.linux-arm64.tar.gz](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.linux-arm64.tar.gz)|Linux|arm64|[ilogtail-1.8.0.linux-arm64.tar.gz.sha256](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.linux-arm64.tar.gz.sha256)|
+|[ilogtail-1.8.0.windows-amd64.zip](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.windows-amd64.zip)|Windows|x86-64|[ilogtail-1.8.0.windows-amd64.zip.sha256](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.windows-amd64.zip.sha256)|
+
+## Docker Image
+
+**Docker Pull Command**
+``` bash
+docker pull sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail:1.8.0
+```

core/CMakeLists.txt

@@ -23,7 +23,7 @@ option(WITHOUTGDB "Build Logtail without gdb" OFF)
 
 # Name/Version information.
 if (NOT DEFINED LOGTAIL_VERSION)
-    set(LOGTAIL_VERSION "1.7.1")
+    set(LOGTAIL_VERSION "1.8.0")
 endif ()
 message(STATUS "Version: ${LOGTAIL_VERSION}")
 
@@ -87,9 +87,9 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/dependencies.cmake)
 # Subdirectories (modules).
 set(SUB_DIRECTORIES_LIST
         aggregator app_config checkpoint common config config_manager config_server_pb
-        controller event event_handler event_listener file_server helper input log_pb logger models monitor
-        parser pipeline plugin polling processor reader profile_sender sender shennong sdk
-        fuse sls_control
+        controller event event_handler event_listener file_server go_pipeline helper input log_pb logger models monitor
+        parser pipeline plugin plugin/creator plugin/instance plugin/interface polling processor processor/daemon profile_sender reader sdk sender shennong sls_control
+        fuse
         )
 if (UNIX)
     set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} streamlog)
@@ -121,7 +121,7 @@ target_link_libraries(${LOGTAIL_TARGET}
         logger
         config_manager
         checkpoint
-        processor
+        processor_daemon
         controller
         pipeline
         plugin
@@ -131,6 +131,7 @@ target_link_libraries(${LOGTAIL_TARGET}
         profile_sender
         app_config
         sls_control
+        go_pipeline
         )
 if (UNIX)
     target_link_libraries(${LOGTAIL_TARGET} streamlog)

core/aggregator/Aggregator.cpp

@@ -128,13 +128,17 @@ bool Aggregator::Add(const std::string& projectName,
                      sls_logs::LogGroup& logGroup,
                      const Config* config,
                      DATA_MERGE_TYPE mergeType,
-                     const uint32_t logGroupSize,
+                     uint32_t logGroupSize,
                      const std::string& defaultRegion,
                      const std::string& filename,
                      const LogGroupContext& context) {
-    if ((logGroupSize == 0 && logGroup.ByteSize() > INT32_FLAG(max_send_log_group_size))
-        || (int32_t)logGroupSize > INT32_FLAG(max_send_log_group_size)) {
-        LOG_ERROR(sLogger, ("invalid log group size", logGroupSize)("real size", logGroup.ByteSize()));
+    if (logGroupSize == 0) {
+        logGroupSize = logGroup.ByteSize();
+    }
+    if ((int32_t)logGroupSize > INT32_FLAG(max_send_log_group_size)) {
+        LOG_ERROR(sLogger,
+                  ("log group size exceed limit. actual size", logGroupSize)("size limit",
+                                                                             INT32_FLAG(max_send_log_group_size)));
         return false;
     }
 
@@ -213,7 +217,7 @@ bool Aggregator::Add(const std::string& projectName,
 
     LogGroup discardLogGroup;
     vector<MergeItem*> sendDataVec;
-    int32_t logByteSize = (logGroupSize == 0 ? logGroup.ByteSize() : logGroupSize) / logSize;
+    int32_t logByteSize = logGroupSize / logSize;
     decltype(logGroup.mutable_logs()->mutable_data()) mutableLogPtr = logGroup.mutable_logs()->mutable_data();
     int32_t neededIdx = 0;
     int32_t discardLogSize = logSize - neededLogSize;

core/aggregator/Aggregator.h

@@ -130,7 +130,7 @@ class Aggregator {
              sls_logs::LogGroup& logGroup,
              const Config* config,
              DATA_MERGE_TYPE mergeType,
-             const uint32_t logGroupSize,
+             uint32_t logGroupSize,
              const std::string& defaultRegion = "",
              const std::string& filename = "",
              const LogGroupContext& context = LogGroupContext());

core/common/MachineInfoUtil.cpp

@@ -23,6 +23,9 @@
 #include <sys/utsname.h>
 #include <pwd.h>
 #include <netdb.h>
+#include <map>
+#include <list>
+#include <algorithm>
 #elif defined(_MSC_VER)
 #include <WinSock2.h>
 #include <Windows.h>
@@ -161,6 +164,32 @@ std::string GetHostName() {
     return std::string(hostname);
 }
 
+std::list<std::string> GetNicIpv4IPList() {
+    struct ifaddrs* ifAddrStruct = NULL;
+    void* tmpAddrPtr = NULL;
+    std::list<std::string> ipList;
+    getifaddrs(&ifAddrStruct);
+    for (struct ifaddrs* ifa = ifAddrStruct; ifa != NULL; ifa = ifa->ifa_next) {
+        if (ifa->ifa_addr == NULL) {
+            continue;
+        }
+        if (ifa->ifa_addr->sa_family == AF_INET) {
+            tmpAddrPtr = &((struct sockaddr_in*)ifa->ifa_addr)->sin_addr;
+            char addressBuffer[INET_ADDRSTRLEN] = "";
+            inet_ntop(AF_INET, tmpAddrPtr, addressBuffer, INET_ADDRSTRLEN);
+            std::string ip(addressBuffer);
+            // The loopback on most Linux distributions is lo, however it is not portable. For example loopback in OSX
+            // is lo0.
+            if (0 == strcmp("lo", ifa->ifa_name) || ip.empty() || StartWith(ip, "127.")) {
+                continue;
+            }
+            ipList.emplace_back(std::move(ip));
+        }
+    }
+    freeifaddrs(ifAddrStruct);
+    return ipList;
+}
+
 std::string GetHostIpByHostName() {
     std::string hostname = GetHostName();
 
@@ -175,7 +204,26 @@ std::string GetHostIpByHostName() {
         return "";
     }
 #if defined(__linux__)
-    struct in_addr* addr = (struct in_addr*)entry->h_addr_list[0];
+    int i = 0;
+    std::list<std::string> ipList;
+    ipList = GetNicIpv4IPList();
+    if (!ipList.empty()) {
+        int isExistValidIP = 0;
+        while (entry->h_addr_list[i] != NULL) {
+            struct in_addr addr;
+            memcpy(&addr, entry->h_addr_list[i], sizeof(struct in_addr));
+            auto item = std::find(ipList.begin(), ipList.end(), inet_ntoa(addr));
+            if (item != ipList.end()) {
+                isExistValidIP = 1;
+                break;
+            }
+            i++;
+        }
+        if (0 == isExistValidIP) {
+            i = 0;
+        }
+    }
+    struct in_addr* addr = (struct in_addr*)entry->h_addr_list[i];
     if (addr == NULL) {
         return "";
     }
@@ -264,45 +312,28 @@ std::string GetHostIp(const std::string& intf) {
 
 std::string GetAnyAvailableIP() {
 #if defined(__linux__)
-    struct ifaddrs* ifaddr = NULL;
-    if (getifaddrs(&ifaddr) == -1) {
-        APSARA_LOG_ERROR(sLogger, ("get any available IP error", errno));
-        return "";
-    }
-
     std::string retIP;
     char host[NI_MAXHOST];
-    for (struct ifaddrs* ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
-        if (ifa->ifa_addr == NULL) {
-            continue;
-        }
-
-        const int family = ifa->ifa_addr->sa_family;
-        const char* familyName = (family == AF_PACKET) ? "AF_PACKET"
-            : (family == AF_INET)                      ? "AF_INET"
-            : (family == AF_INET6)                     ? "AF_INET6"
-                                                       : "???";
-        APSARA_LOG_DEBUG(sLogger, ("interface name", ifa->ifa_name)("family", family)("family name", familyName));
-        if (family == AF_INET) {
-            int s = getnameinfo(ifa->ifa_addr, sizeof(struct sockaddr_in), host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
-            if (s != 0) {
-                APSARA_LOG_WARNING(sLogger, ("getnameinfo error", gai_strerror(s))("interface name", ifa->ifa_name));
+    std::list<std::string> ipList;
+    ipList = GetNicIpv4IPList();
+    if (!ipList.empty()) {
+        for (std::string ip : ipList) {
+            struct sockaddr_in sa;
+            sa.sin_family = AF_INET;
+            sa.sin_port = 0;
+            int result = inet_pton(AF_INET, ip.c_str(), &sa.sin_addr);
+            if (result != 1) {
                 continue;
             }
-
-            std::string ip(host);
-            APSARA_LOG_DEBUG(sLogger, ("interface name", ifa->ifa_name)("IP", ip));
-            if (!ip.empty() && !StartWith(ip, "127.")) {
-                retIP = ip;
-                break;
-            }
-            if (retIP.empty()) {
-                retIP = ip;
+            int s = getnameinfo(
+                (struct sockaddr*)&sa, sizeof(struct sockaddr_in), host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
+            if (s != 0) {
+                continue;
             }
+            retIP = ip;
+            break;
         }
     }
-
-    freeifaddrs(ifaddr);
     return retIP;
 
 #elif defined(_MSC_VER)
@@ -447,12 +478,12 @@ size_t FetchECSMetaCallback(char* buffer, size_t size, size_t nmemb, std::string
     if (NULL == buffer) {
         return 0;
     }
- 
+
     size_t sizes = size * nmemb;
     res->append(buffer, sizes);
     return sizes;
 }
- 
+
 ECSMeta FetchECSMeta() {
     CURL* curl;
     for (size_t retryTimes = 1; retryTimes <= 5; retryTimes++) {
@@ -486,7 +517,7 @@ ECSMeta FetchECSMeta() {
                 if (instanceItr != doc.MemberEnd() && (instanceItr->value.IsString())) {
                     metaObj.instanceID = instanceItr->value.GetString();
                 }
-        
+
                 rapidjson::Value::ConstMemberIterator userItr = doc.FindMember("owner-account-id");
                 if (userItr != doc.MemberEnd() && userItr->value.IsString()) {
                     metaObj.userID = userItr->value.GetString();