Merge pull request #1176 from alibaba/1.8

merge 1.8
alibaba · Oct 11, 2023 · a2f1078 · a2f1078
2 parents a3623ec + 529a5c0
commit a2f1078
Show file tree

Hide file tree

Showing 19 changed files with 143 additions and 52 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -36,29 +36,3 @@ your changes, such as:
 - [public] [both] [updated] add a new feature
 
 ## [Unreleased]
-
-- [public] [both] [added] refactoried C++ process pipeline
-- [public] [both] [added] support use accelerate processors with go processors
-- [public] [both] [added] add new logtail metric module
-- [public] [both] [added] ddd JSON flatten protocol, data can be flattened and then brushed into storage such as Kafka and ES.
-- [public] [both] [added] use env `LOGTAIL_LOG_LEVEL` to control ilogtail log level
-- [public] [both] [updated] support continue/end regex patterns to split multiline log
-- [public] [both] [updated] support reader flush timeout
-- [public] [both] [updated] Flusher Kafka V2: support send the message with headers to kafka
-- [public] [both] [updated] update gcc version to 9.3.1
-- [public] [both] [updated] add make flag WITHOUTGDB
-- [public] [both] [updated] cache incomplete line in memory to avoid repeated read system call
-- [public] [both] [fixed] Add APSARA\_LOG\_TRACE to solve the problem of not being able to find LOG\_TRACE.
-- [public] [both] [fixed] fix multiline is splitted if not flushed to disk together
-- [public] [both] [fixed] fix line is truncated if \0 is in the middle of line
-- [public] [both] [fixed] container cannot exit for file reopened by checkpoint
-- [public] [both] [fixed] fix filename being mismatched to the deleted file if the deleted file size is 0 and their inode is same
-- [public] [both] [fixed] fix config server panic caused by concurrent read and write shared object
-- [public] [both] [fixed] timezone adjust not working with apsara\_log
-- [public] [both] [added] support plugin ProcessorParseTimestampNative
-- [public] [both] [added] support plugin ProcessorOtelMetric
-- [public] [both] [updated] skywalking plugin support to capture `db.connection_string` tag 
-- [public] [both] [added] support plugin ProcessorParseApsaraNative
-- [public] [both] [added] support plugin ProcessorParseDelimiterNative
-- [public] [both] [added] support plugin ProcessorFilterNative
-- [public] [both] [added] support plugin ProcessorDesensitizeNative
diff --git a/Makefile b/Makefile
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 .DEFAULT_GOAL := all
-VERSION ?= 1.7.1
+VERSION ?= 1.8.0
 DOCKER_PUSH ?= false
 DOCKER_REPOSITORY ?= aliyun/ilogtail
 BUILD_REPOSITORY ?= aliyun/ilogtail_build

diff --git a/changes/v1.8.0.md b/changes/v1.8.0.md
@@ -0,0 +1,64 @@
+# 1.8.0
+
+## Changes
+
+All issues and pull requests are [here](https://github.com/alibaba/ilogtail/milestone/21).
+
+### Features
+
+- [public] [both] [added] refactoried C++ process pipeline
+- [public] [both] [added] support use accelerate processors with go processors
+- [public] [both] [added] add new logtail metric module
+- [public] [both] [added] ddd JSON flatten protocol, data can be flattened and then brushed into storage such as Kafka and ES.
+- [public] [both] [added] use env `LOGTAIL_LOG_LEVEL` to control ilogtail log level
+- [public] [both] [added] add global host path blacklist
+- [public] [both] [updated] support continue/end regex patterns to split multiline log
+- [public] [both] [updated] support reader flush timeout
+- [public] [both] [updated] flusher Kafka V2: support send the message with headers to kafka
+- [public] [both] [updated] sls flusher supports nanosecond timestamp
+- [public] [both] [updated] update gcc version to 9.3.1
+- [public] [both] [updated] add make flag WITHOUTGDB
+- [public] [both] [updated] cache incomplete line in memory to avoid repeated read system call
+- [public] [both] [updated] unify metrics format
+- [public] [both] [updated] verify host ip against NIC to get more accurate host ip
+- [public] [both] [added] support plugin ProcessorParseTimestampNative
+- [public] [both] [added] support plugin processor_otel_metric
+- [public] [both] [updated] skywalking plugin support to capture `db.connection_string` tag
+- [public] [both] [added] support plugin ProcessorParseApsaraNative
+- [public] [both] [added] support plugin ProcessorParseDelimiterNative
+- [public] [both] [added] support plugin ProcessorFilterNative
+- [public] [both] [added] support plugin ProcessorDesensitizeNative
+- [public] [both] [added] add resource tag for env created logtail config
+
+### Fixed
+
+- [public] [both] [fixed] remove the ip restrict of config server address, allow domain and hostname
+- [public] [both] [fixed] add APSARA\_LOG\_TRACE to solve the problem of not being able to find LOG\_TRACE.
+- [public] [both] [fixed] fix collection duplication for statefulset sidecar that may mount same volume on different nodes
+- [public] [both] [fixed] fix multiline is splitted if not flushed to disk together
+- [public] [both] [fixed] fix line is truncated if \0 is in the middle of line
+- [public] [both] [fixed] container cannot exit for file reopened by checkpoint
+- [public] [both] [fixed] fix filename being mismatched to the deleted file if the deleted file size is 0 and their inode is same
+- [public] [both] [fixed] fix config server panic caused by concurrent read and write shared object
+- [public] [both] [fixed] add expand array switch in json processor for backward compatibility
+- [public] [both] [fixed] timezone adjust not working with apsara\_log
+- [public] [both] [fixed] fix parse json without newline may result in partial parsing
+- [public] [both] [fixed] fix json does not split correctly if beginning buffer contains invalid json
+
+### Doc
+
+
+## Download
+
+| **Filename** | **OS** | **Arch** | **SHA256 Checksum** |
+|  ----  | ----  | ----  | ----  |
+|[ilogtail-1.8.0.linux-amd64.tar.gz](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.linux-amd64.tar.gz)|Linux|x86-64|[ilogtail-1.8.0.linux-amd64.tar.gz.sha256](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.linux-amd64.tar.gz.sha256)|
+|[ilogtail-1.8.0.linux-arm64.tar.gz](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.linux-arm64.tar.gz)|Linux|arm64|[ilogtail-1.8.0.linux-arm64.tar.gz.sha256](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.linux-arm64.tar.gz.sha256)|
+|[ilogtail-1.8.0.windows-amd64.zip](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.windows-amd64.zip)|Windows|x86-64|[ilogtail-1.8.0.windows-amd64.zip.sha256](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.windows-amd64.zip.sha256)|
+
+## Docker Image
+
+**Docker Pull Command**
+``` bash
+docker pull sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail:1.8.0
+```
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
@@ -23,7 +23,7 @@ option(WITHOUTGDB "Build Logtail without gdb" OFF)
 
 # Name/Version information.
 if (NOT DEFINED LOGTAIL_VERSION)
-    set(LOGTAIL_VERSION "1.7.1")
+    set(LOGTAIL_VERSION "1.8.0")
 endif ()
 message(STATUS "Version: ${LOGTAIL_VERSION}")
 
@@ -88,8 +88,8 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/dependencies.cmake)
 set(SUB_DIRECTORIES_LIST
         aggregator app_config checkpoint common config config_manager config_server_pb
         controller event event_handler event_listener file_server helper input log_pb logger models monitor
-        parser pipeline plugin polling processor reader profile_sender sender shennong sdk
-        fuse sls_control
+        parser pipeline plugin polling processor profile_sender reader sdk sender shennong sls_control
+        fuse
         )
 if (UNIX)
     set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} streamlog)

diff --git a/core/logtail_windows.cpp b/core/logtail_windows.cpp
@@ -148,7 +148,6 @@ void do_worker_process() {
     ConfigManager::GetInstance()->LoadConfig(AppConfig::GetInstance()->GetUserConfigPath());
     PipelineManager::GetInstance()->LoadAllPipelines();
     ConfigManager::GetInstance()->LoadDockerConfig();
-
     // mNameConfigMap is empty, configExistFlag is false
     bool configExistFlag = !ConfigManager::GetInstance()->GetAllConfig().empty();
 

diff --git a/docker/Dockerfile_build b/docker/Dockerfile_build
@@ -19,7 +19,7 @@ WORKDIR /src
 COPY . .
 
 ARG HOST_OS=Linux
-ARG VERSION=1.7.1
+ARG VERSION=1.8.0
 
 USER root
 

diff --git a/docker/Dockerfile_development_part b/docker/Dockerfile_development_part
@@ -15,7 +15,7 @@
 FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail-build-linux:gcc_9.3.1-1
 
 ARG HOST_OS=Linux
-ARG VERSION=1.7.1
+ARG VERSION=1.8.0
 
 USER root
 WORKDIR /ilogtail

diff --git a/docker/Dockerfile_production b/docker/Dockerfile_production
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 FROM --platform=$TARGETPLATFORM centos:centos7.9.2009 as build
-ARG VERSION=1.7.1
+ARG VERSION=1.8.0
 ARG TARGETPLATFORM
 WORKDIR /usr/local
 COPY dist/ilogtail-${VERSION}.linux-*.tar.gz .
@@ -28,7 +28,7 @@ ENV container docker
 RUN yum update -y && yum upgrade -y && yum -y clean all && rm -fr /var/cache && rm -rf /core.*
 
 ARG HOST_OS=Linux
-ARG VERSION=1.7.1
+ARG VERSION=1.8.0
 ARG TARGETPLATFORM
 
 COPY --from=build /usr/local/ilogtail-${VERSION} /usr/local/ilogtail

diff --git a/docs/cn/data-pipeline/processor/processor-otel-metric.md b/docs/cn/data-pipeline/processor/processor-otel-metric.md
@@ -1,4 +1,4 @@
-# processor_otel_trace插件
+# processor_otel_metric插件
 
 ## 版本
 
@@ -26,7 +26,7 @@ inputs:
     LogPath: .
     FilePattern: simple.log
 processors:
-  - Type: processor_otel_trace
+  - Type: processor_otel_metric
     SourceKey: "content"
     Format: protojson
 flushers:

diff --git a/docs/cn/installation/release-notes.md b/docs/cn/installation/release-notes.md
@@ -1,5 +1,60 @@
 # 发布历史
 
+## 1.8.0
+
+### 发布记录
+
+发版日期：2023 年 10 月 10 日
+
+新功能
+
+* C++流水线重构，处理流程插件化 [#995](https://github.com/alibaba/ilogtail/pull/995)
+* 多行切分新增continue/end模式，支持多个模式组合 [#978](https://github.com/alibaba/ilogtail/pull/978)
+* 支持超时切分行 [#1067](https://github.com/alibaba/ilogtail/pull/1067)
+* SLS Flusher 支持纳秒级日志时间[#1097](https://github.com/alibaba/ilogtail/issues/1097)
+* 新增全局主机路径黑名单 [#1073](https://github.com/alibaba/ilogtail/pull/1073)
+* 新增trace解析插件`processor_otel_metric` [#1130](https://github.com/alibaba/ilogtail/pull/1130)
+* 环境变量方式创建采集配置时添加资源标签 [#1147](https://github.com/alibaba/ilogtail/pull/1147)
+
+优化
+
+* 编译镜像GCC升级到9.3.1 [#1108](https://github.com/alibaba/ilogtail/pull/1108)
+* Kafka V2插件支持写数据时增加header [#1065](https://github.com/alibaba/ilogtail/issues/1065)
+* 缓存未构成完整行的日志，减少读文件系统调用 [#1142](https://github.com/alibaba/ilogtail/pull/1142)
+* 支持使用环境变量控制日志打印级别 [#959](https://github.com/alibaba/ilogtail/issues/959)
+* 支持flusher输出时使用扁平格式 [#1112](https://github.com/alibaba/ilogtail/pull/1112)
+* skywalking插件支持捕获`db.connection_string`标签 [#1131](https://github.com/alibaba/ilogtail/pull/1131)
+* 统一对metrics V1模型的格式化处理 [#1060](https://github.com/alibaba/ilogtail/pull/1060)
+* 校验网卡IP以获取更精确地主机IP [#1019](https://github.com/alibaba/ilogtail/pull/1019)
+
+问题修复
+
+* 解决采集有挂载卷的statefulset漂移到不同节点时数据重复采集的问题 [#1081](https://github.com/alibaba/ilogtail/issues/1081)
+* 修复json processor支持array解析后的兼容性问题 [#1088](https://github.com/alibaba/ilogtail/issues/1088)
+* 修复空文件inode复用后导致采集到的日志文件名错误的问题 [#1102](https://github.com/alibaba/ilogtail/issues/1102)
+* 修复因checkpoint重新打开文件导致容器无法退出的问题 [#1109](https://github.com/alibaba/ilogtail/pull/1109)
+* 修复stdout文件路径为软链时无法采集容器stdout的问题 [#1037](https://github.com/alibaba/ilogtail/issues/1037)
+* 修复飞天日志时间无法调整时区的问题 [#1123](https://github.com/alibaba/ilogtail/pull/1123)
+* 修复json模式日志最后没有回车可能解析不正确的问题 [#1126](https://github.com/alibaba/ilogtail/issues/1126)
+* 修复读取到的数据开头含有非法jon时json解析异常的问题 [#1161](https://github.com/alibaba/ilogtail/issues/1161)
+
+[详情和源代码](https://github.com/alibaba/ilogtail/blob/main/changes/v1.7.1.md)
+
+### 下载
+
+| 文件名                                                                                                                                          | 系统    | 架构     | SHA256 校验码                                                       |
+| -------------------------------------------------------------------------------------------------------------------------------------------- | ----- | ------ | ---------------------------------------------------------------- |
+| [ilogtail-1.8.0.linux-amd64.tar.gz](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.linux-amd64.tar.gz) | Linux | x86-64 | 90f60c372379311880d78a30609e76924f07d1fc8fdbcb944a78c01cc94de891 |
+| [ilogtail-1.8.0.linux-arm64.tar.gz](https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/1.8.0/ilogtail-1.8.0.linux-arm64.tar.gz) | Linux | arm64  | 6737c2f59f4a462d98e46fe5dca8d471a6634531e2cd8a2feb4a6beaaff22d20 |
+
+### Docker 镜像
+
+**Docker Pull 命令**&#x20;
+
+``` bash
+docker pull sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail:1.8.0
+```
+
 ## 1.7.1
 
 ### 发布记录
@@ -26,7 +81,6 @@
 问题修复
 
 * 修复 service_go_profile可能panic的问题 [#1036](https://github.com/alibaba/ilogtail/pull/1036)
-* 修复stdout文件路径为软链时无法采集容器stdout的问题 [#1037](https://github.com/alibaba/ilogtail/issues/1037)
 * 修复zstd批量发送问题 [#1006](https://github.com/alibaba/ilogtail/pull/1006)
 * 修复service_otlp插件无法退出的问题 [#1040](https://github.com/alibaba/ilogtail/pull/1040)
 * 修复env创建配置时ttl值非法与空行为不一致的问题 [#1045](https://github.com/alibaba/ilogtail/issues/1045)

diff --git a/docs/en/guides/How-to-build-with-docker.md b/docs/en/guides/How-to-build-with-docker.md
@@ -13,24 +13,24 @@ make solib
 
 ### Build image. 
     - The default {DOCKER_REPOSITORY} is `aliyun/ilogtail`.
-    - The default {VERSION} is `1.7.1`.
+    - The default {VERSION} is `1.8.0`.
     - The default {DOCKER_PUSH} is `false`. When the option is configured as true, the built images would also be pushed to the {DOCKER_REPOSITORY} with {VERSION} tag.
 
     ```shell
         DOCKER_PUSH={DOCKER_PUSH} DOCKER_REPOSITORY={DOCKER_REPOSITORY} VERSION={VERSION} make wholedocker
     ```
-   So when you exec `make wholedocker` command, the built image named as `aliyun/ilogtail:1.7.1` would be stored in local repository.
+   So when you exec `make wholedocker` command, the built image named as `aliyun/ilogtail:1.8.0` would be stored in local repository.
 
 
 ## Build Pure Go image.
 If the features that you want to use only in Go part, such as collecting stdout logs or metrics inputs, you cloud build a pure Go image to use.
 
 ### Build image.
     - The default {DOCKER_REPOSITORY} is `aliyun/ilogtail`.
-    - The default {VERSION} is `1.7.1`.
+    - The default {VERSION} is `1.8.0`.
     - The default {DOCKER_PUSH} is `false`. When the option is configured as true, the built images would also be pushed to the {DOCKER_REPOSITORY} with {VERSION} tag.
 
     ```shell
         DOCKER_PUSH={DOCKER_PUSH} DOCKER_REPOSITORY={DOCKER_REPOSITORY} VERSION={VERSION} make docker
     ```
-So when you exec `make docker` command, the built image named as `aliyun/ilogtail:1.7.1` would be stored in local repository.
+So when you exec `make docker` command, the built image named as `aliyun/ilogtail:1.8.0` would be stored in local repository.
diff --git a/docs/en/guides/How-to-do-manual-test.md b/docs/en/guides/How-to-do-manual-test.md
@@ -114,8 +114,8 @@ program written by yourself.
 ### Run [Logtail AlibabaCloud](https://help.aliyun.com/document_detail/28979.html) on container
 
 1. Run `make docker` to compile [Logtail AlibabaCloud](https://help.aliyun.com/document_detail/28979.html) docker
-   images named `aliyun/ilogtail:1.7.1`.
-2. Rename `aliyun/ilogtail:1.7.1` to a custom name and push to the remotes, such
-   as `registry.cn-beijing.aliyuncs.com/aliyun/ilogtail:1.7.1`.
+   images named `aliyun/ilogtail:1.8.0`.
+2. Rename `aliyun/ilogtail:1.8.0` to a custom name and push to the remotes, such
+   as `registry.cn-beijing.aliyuncs.com/aliyun/ilogtail:1.8.0`.
 3. Replace the mirror of `logtail-ds` of [ACK](https://www.aliyun.com/product/list/alibabacloudnative) or your self
    platform and restart.
diff --git a/scripts/docker_build.sh b/scripts/docker_build.sh
@@ -56,7 +56,7 @@ function check_docker_buildkit_support {
 ARCH=$(arch)
 CATEGORY=$1
 GENERATED_HOME=$2
-VERSION=${3:-1.7.1}
+VERSION=${3:-1.8.0}
 REPOSITORY=${4:-aliyun/ilogtail}
 PUSH=${5:-false}
 USE_DOCKER_BUILDKIT=${6:-${DOCKER_BUILD_USE_BUILDKIT:-$(check_docker_buildkit_support)}}

diff --git a/scripts/gen_build_scripts.sh b/scripts/gen_build_scripts.sh
@@ -24,7 +24,7 @@ set -o pipefail
 # e2e: Build plugin dynamic lib with GOC and build the CPP part.
 CATEGORY=$1
 GENERATED_HOME=$2
-VERSION=${3:-1.7.1}
+VERSION=${3:-1.8.0}
 REPOSITORY=${4:-aliyun/ilogtail}
 OUT_DIR=${5:-output}
 EXPORT_GO_ENVS=${6:-${DOCKER_BUILD_EXPORT_GO_ENVS:-true}}

diff --git a/scripts/plugin_build.sh b/scripts/plugin_build.sh
@@ -27,7 +27,7 @@ function os() {
 MOD=${1:-mod}
 BUILDMODE=${2:-default}
 OUT_DIR=${3:-output}
-VERSION=${4:-1.7.1}
+VERSION=${4:-1.8.0}
 PLUGINS_CONFIG_FILE=${5:-${PLUGINS_CONFIG_FILE:-plugins.yml,external_plugins.yml}}
 GO_MOD_FILE=${6:-${GO_MOD_FILE:-go.mod}}
 NAME=ilogtail

diff --git a/scripts/windows32_build.bat b/scripts/windows32_build.bat
@@ -6,7 +6,7 @@ REM 2. Build iLogtail.
 REM 3. Build iLogtail plugin.
 REM 4. Make package.
 
-set ILOGTAIL_VERSION=1.7.1
+set ILOGTAIL_VERSION=1.8.0
 if not "%1" == "" set ILOGTAIL_VERSION=%1
 set CurrentPath=%~dp0
 set P1Path=

diff --git a/scripts/windows64_build.bat b/scripts/windows64_build.bat
@@ -6,7 +6,7 @@ REM 2. Build iLogtail.
 REM 3. Build iLogtail plugin.
 REM 4. Make package.
 
-set ILOGTAIL_VERSION=1.7.1
+set ILOGTAIL_VERSION=1.8.0
 if not "%1" == "" set ILOGTAIL_VERSION=%1
 set CurrentPath=%~dp0
 set P1Path=

diff --git a/scripts/windows64_dist.bat b/scripts/windows64_dist.bat
@@ -5,7 +5,7 @@ REM 1. Set environments.
 REM 2. Copy output to dist package dir.
 REM 3. Pack dir to zip archive.
 
-set ILOGTAIL_VERSION=1.7.1
+set ILOGTAIL_VERSION=1.8.0
 if not "%1" == "" set ILOGTAIL_VERSION=%1
 set CurrentPath=%~dp0
 set P1Path=

diff --git a/test/engine/boot/compose.go b/test/engine/boot/compose.go
@@ -55,7 +55,7 @@ services:
       interval: 1s
       retries: 10
   ilogtailC:
-    image: aliyun/ilogtail:1.7.1
+    image: aliyun/ilogtail:1.8.0
     hostname: ilogtail
     privileged: true
     pid: host