From bb3d0ed37d347f26f5751d281d59ff0b64b5c422 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=83=A8=E9=99=8C?= Date: Fri, 8 Jul 2022 22:27:05 +0800 Subject: [PATCH] add data_pipelines templates (#191) --- core/config_manager/ConfigYamlToJson.cpp | 2 +- core/profile_sender/ProfileSender.cpp | 4 +-- .../data_pipelines/file_delimiter.yaml | 34 +++++++++++++++++++ example_config/data_pipelines/file_json.yaml | 32 +++++++++++++++++ .../data_pipelines/file_multiline.yaml | 31 +++++++++++++++++ example_config/data_pipelines/file_reg.yaml | 33 ++++++++++++++++++ .../user_yaml_config.d/file_simple.yaml | 14 -------- .../user_yaml_config.d/stdout_simple.yaml | 14 -------- 8 files changed, 133 insertions(+), 31 deletions(-) create mode 100644 example_config/data_pipelines/file_delimiter.yaml create mode 100644 example_config/data_pipelines/file_json.yaml create mode 100644 example_config/data_pipelines/file_multiline.yaml create mode 100644 example_config/data_pipelines/file_reg.yaml diff --git a/core/config_manager/ConfigYamlToJson.cpp b/core/config_manager/ConfigYamlToJson.cpp index 9bce019942..8398ac36f5 100644 --- a/core/config_manager/ConfigYamlToJson.cpp +++ b/core/config_manager/ConfigYamlToJson.cpp @@ -445,7 +445,7 @@ bool ConfigYamlToJson::GenerateLocalJsonConfigForSLSFulsher(const YAML::Node& ya bool ConfigYamlToJson::FillupDefalutUserJsonConfig(const WorkMode& workMode, Json::Value& userJsonConfig) { if (workMode.mIsFileMode) { if (!userJsonConfig.isMember("max_depth")) { - userJsonConfig["max_depth"] = 10; + userJsonConfig["max_depth"] = 0; } // To compatible with the old configuration, introduce placeholder for file mode. diff --git a/core/profile_sender/ProfileSender.cpp b/core/profile_sender/ProfileSender.cpp index bb31f91c41..167db7a3da 100644 --- a/core/profile_sender/ProfileSender.cpp +++ b/core/profile_sender/ProfileSender.cpp @@ -48,8 +48,8 @@ void ProfileSender::SendRunningStatus(sls_logs::LogGroup& logGroup) { } std::string region = "cn-shanghai"; - std::string project = "logtail-opensource"; - std::string logstore = "logtail-online"; + std::string project = "ilogtail-community-edition"; + std::string logstore = "ilogtail-online"; std::string endpoint = region + ".log.aliyuncs.com"; Json::Value logtailStatus; diff --git a/example_config/data_pipelines/file_delimiter.yaml b/example_config/data_pipelines/file_delimiter.yaml new file mode 100644 index 0000000000..27d72a8239 --- /dev/null +++ b/example_config/data_pipelines/file_delimiter.yaml @@ -0,0 +1,34 @@ +# Copyright 2022 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable: true +inputs: + - Type: file_log + LogPath: /home/test-dir/test_log + FilePattern: delimiter.log +processors: + - Type: processor_split_char + SourceKey: content + SplitSep: "|" + SplitKeys: + - time + - k1 + - k2 +flushers: + - Type: flusher_sls + Endpoint: cn-xxx.log.aliyuncs.com + ProjectName: test_project + LogstoreName: test_logstore + - Type: flusher_stdout + OnlyStdout: true \ No newline at end of file diff --git a/example_config/data_pipelines/file_json.yaml b/example_config/data_pipelines/file_json.yaml new file mode 100644 index 0000000000..50070d7e54 --- /dev/null +++ b/example_config/data_pipelines/file_json.yaml @@ -0,0 +1,32 @@ +# Copyright 2022 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable: true +inputs: + - Type: file_log + LogPath: /home/test-dir/test_log + FilePattern: json.log +processors: + - Type: processor_json + SourceKey: content + KeepSource: false + ExpandDepth: 1 + ExpandConnector: "" +flushers: + - Type: flusher_sls + Endpoint: cn-xxx.log.aliyuncs.com + ProjectName: test_project + LogstoreName: test_logstore + - Type: flusher_stdout + OnlyStdout: true \ No newline at end of file diff --git a/example_config/data_pipelines/file_multiline.yaml b/example_config/data_pipelines/file_multiline.yaml new file mode 100644 index 0000000000..b66d1e3f24 --- /dev/null +++ b/example_config/data_pipelines/file_multiline.yaml @@ -0,0 +1,31 @@ +# Copyright 2022 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable: true +inputs: + - Type: file_log + LogPath: /home/test-dir/test_log + FilePattern: multiline.log +processors: + - Type: processor_split_log_regex + SplitRegex: \[\d+-\d+-\d+\s\d+:\d+:\d+]\s.* + SplitKey: content + PreserveOthers: true +flushers: + - Type: flusher_sls + Endpoint: cn-xxx.log.aliyuncs.com + ProjectName: test_project + LogstoreName: test_logstore + - Type: flusher_stdout + OnlyStdout: true \ No newline at end of file diff --git a/example_config/data_pipelines/file_reg.yaml b/example_config/data_pipelines/file_reg.yaml new file mode 100644 index 0000000000..1eb4fee384 --- /dev/null +++ b/example_config/data_pipelines/file_reg.yaml @@ -0,0 +1,33 @@ +# Copyright 2022 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable: true +inputs: + - Type: file_log + LogPath: /home/test-dir/test_log + FilePattern: reg.log +processors: + - Type: processor_regex + SourceKey: content + Regex: (\d*-\d*-\d*)\s(.*) + Keys: + - time + - msg +flushers: + - Type: flusher_sls + Endpoint: cn-xxx.log.aliyuncs.com + ProjectName: test_project + LogstoreName: test_logstore + - Type: flusher_stdout + OnlyStdout: true \ No newline at end of file diff --git a/example_config/start_with_container/user_yaml_config.d/file_simple.yaml b/example_config/start_with_container/user_yaml_config.d/file_simple.yaml index ddb0876a82..27a5278ecb 100644 --- a/example_config/start_with_container/user_yaml_config.d/file_simple.yaml +++ b/example_config/start_with_container/user_yaml_config.d/file_simple.yaml @@ -12,20 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Copyright 2022 iLogtail Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - enable: true inputs: - Type: file_log diff --git a/example_config/start_with_container/user_yaml_config.d/stdout_simple.yaml b/example_config/start_with_container/user_yaml_config.d/stdout_simple.yaml index 853f74f0d9..fbcf52b8d8 100644 --- a/example_config/start_with_container/user_yaml_config.d/stdout_simple.yaml +++ b/example_config/start_with_container/user_yaml_config.d/stdout_simple.yaml @@ -12,20 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Copyright 2022 iLogtail Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - enable: true inputs: - Type: service_docker_stdout