diff --git a/core/processor/ProcessorParseDelimiterNative.cpp b/core/processor/ProcessorParseDelimiterNative.cpp index 9f273c82ed..5893ea6184 100644 --- a/core/processor/ProcessorParseDelimiterNative.cpp +++ b/core/processor/ProcessorParseDelimiterNative.cpp @@ -15,11 +15,12 @@ */ #include "processor/ProcessorParseDelimiterNative.h" + #include "common/Constants.h" #include "models/LogEvent.h" +#include "monitor/MetricConstants.h" #include "parser/LogParser.h" #include "plugin/instance/ProcessorInstance.h" -#include "monitor/MetricConstants.h" namespace logtail { @@ -247,8 +248,14 @@ bool ProcessorParseDelimiterNative::SplitString( size_t pos = begIdx; size_t top = endIdx - d_size; while (pos <= top) { - const char* pch = strstr(buffer + pos, mSeparator.c_str()); - size_t pos2 = pch == NULL ? endIdx : (pch - buffer); + const char* pch = std::search(buffer + pos, buffer + endIdx, mSeparator.begin(), mSeparator.end()); + size_t pos2; + // if not found, pos2 = endIdx + if (pch == buffer + endIdx) { + pos2 = endIdx; + } else { + pos2 = pch - buffer; + } if (pos2 != pos) { colBegIdxs.push_back(pos); colLens.push_back(pos2 - pos); diff --git a/core/processor/ProcessorSplitLogStringNative.h b/core/processor/ProcessorSplitLogStringNative.h index e5e9d88e57..f6ec13d922 100644 --- a/core/processor/ProcessorSplitLogStringNative.h +++ b/core/processor/ProcessorSplitLogStringNative.h @@ -14,9 +14,10 @@ * limitations under the License. */ -#include "plugin/interface/Processor.h" -#include #include +#include + +#include "plugin/interface/Processor.h" namespace logtail { @@ -41,6 +42,7 @@ class ProcessorSplitLogStringNative : public Processor { bool mEnableLogPositionMeta = false; #ifdef APSARA_UNIT_TEST_MAIN friend class ProcessorRegexStringNativeUnittest; + friend class ProcessorParseDelimiterNativeUnittest; #endif }; diff --git a/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp b/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp index b4d8819170..e1e36fe01a 100644 --- a/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseDelimiterNativeUnittest.cpp @@ -13,13 +13,14 @@ // limitations under the License. #include -#include "unittest/Unittest.h" #include "common/JsonUtil.h" #include "config/Config.h" -#include "processor/ProcessorParseDelimiterNative.h" #include "models/LogEvent.h" #include "plugin/instance/ProcessorInstance.h" +#include "processor/ProcessorParseDelimiterNative.h" +#include "processor/ProcessorSplitLogStringNative.h" +#include "unittest/Unittest.h" namespace logtail { @@ -34,6 +35,7 @@ class ProcessorParseDelimiterNativeUnittest : public ::testing::Test { } void TestInit(); + void TestMultipleLines(); void TestProcessWholeLine(); void TestProcessQuote(); void TestProcessKeyOverwritten(); @@ -46,6 +48,7 @@ class ProcessorParseDelimiterNativeUnittest : public ::testing::Test { }; UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestInit); +UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestMultipleLines); UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestProcessWholeLine); UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestProcessQuote); UNIT_TEST_CASE(ProcessorParseDelimiterNativeUnittest, TestProcessKeyOverwritten); @@ -70,6 +73,84 @@ void ProcessorParseDelimiterNativeUnittest::TestInit() { APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); } +void ProcessorParseDelimiterNativeUnittest::TestMultipleLines() { + // make config + Config config; + config.mLogType = DELIMITER_LOG; + config.mLogBeginReg = ""; + config.mAdvancedConfig.mEnableLogPositionMeta = false; + config.mSeparator = "@@"; + config.mQuote = '\000'; + config.mColumnKeys = {"a", "b", "c"}; + config.mDiscardUnmatch = true; + config.mUploadRawLog = false; + config.mAdvancedConfig.mRawLogTag = "__raw__"; + // make events + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::string inJson = R"({ + "events" : + [ + { + "contents" : + { + "content" : "123@@456@@789 +012@@345@@678 +", + "log.file.offset": "0" + }, + "timestamp" : 12345678901, + "type" : 1 + } + ] + })"; + eventGroup.FromJsonString(inJson); + // run function ProcessorSplitLogStringNative + ProcessorSplitLogStringNative processorSplitLogStringNative; + processorSplitLogStringNative.SetContext(mContext); + std::string pluginId = "testID"; + ComponentConfig componentConfig(pluginId, config); + APSARA_TEST_TRUE_FATAL(processorSplitLogStringNative.Init(componentConfig)); + processorSplitLogStringNative.Process(eventGroup); + // run function ProcessorParseDelimiterNative + ProcessorParseDelimiterNative& processorDelimiterNative = *(new ProcessorParseDelimiterNative); + ProcessorInstance processorInstance(&processorDelimiterNative, pluginId); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + processorDelimiterNative.Process(eventGroup); + std::string expectJson = R"({ + "events" : + [ + { + "contents" : + { + "a": "123", + "b": "456", + "c": "789", + "log.file.offset": "0" + }, + "timestamp" : 12345678901, + "timestampNanosecond": 0, + "type" : 1 + }, + { + "contents" : + { + "a": "012", + "b": "345", + "c": "678", + "log.file.offset": "0" + }, + "timestamp" : 12345678901, + "timestampNanosecond": 0, + "type" : 1 + } + ] + })"; + // judge result + std::string outJson = eventGroup.ToJsonString(); + APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); +} + void ProcessorParseDelimiterNativeUnittest::TestProcessWholeLine() { // make config Config config; @@ -558,10 +639,10 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventKeepUnmatch() { APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); APSARA_TEST_EQUAL_FATAL(count, processorInstance.mProcInRecordsTotal->GetValue()); std::string expectValue = "value1"; - APSARA_TEST_EQUAL_FATAL((expectValue.length())*count, processor.mProcParseInSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL((expectValue.length()) * count, processor.mProcParseInSizeBytes->GetValue()); APSARA_TEST_EQUAL_FATAL(count, processorInstance.mProcOutRecordsTotal->GetValue()); expectValue = "__raw_log__value1"; - APSARA_TEST_EQUAL_FATAL((expectValue.length())*count, processor.mProcParseOutSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL((expectValue.length()) * count, processor.mProcParseOutSizeBytes->GetValue()); APSARA_TEST_EQUAL_FATAL(0, processor.mProcDiscardRecordsTotal->GetValue()); @@ -646,7 +727,7 @@ void ProcessorParseDelimiterNativeUnittest::TestProcessEventDiscardUnmatch() { APSARA_TEST_EQUAL_FATAL(count, processor.GetContext().GetProcessProfile().parseFailures); APSARA_TEST_EQUAL_FATAL(count, processorInstance.mProcInRecordsTotal->GetValue()); std::string expectValue = "value1"; - APSARA_TEST_EQUAL_FATAL((expectValue.length())*count, processor.mProcParseInSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL((expectValue.length()) * count, processor.mProcParseInSizeBytes->GetValue()); // discard unmatch, so output is 0 APSARA_TEST_EQUAL_FATAL(0, processorInstance.mProcOutRecordsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(0, processor.mProcParseOutSizeBytes->GetValue());