Support sync curl function in common http module. (#1758)

core/common/common.cmake

@@ -30,6 +30,7 @@ list(APPEND THIS_SOURCE_FILES_LIST ${XX_HASH_SOURCE_FILES})
 list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/memory/SourceBuffer.h)
 list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/http/AsynCurlRunner.cpp ${CMAKE_SOURCE_DIR}/common/http/Curl.cpp ${CMAKE_SOURCE_DIR}/common/http/HttpResponse.cpp)
 list(APPEND THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/timer/Timer.cpp ${CMAKE_SOURCE_DIR}/common/timer/HttpRequestTimerEvent.cpp)
+
 # remove several files in common
 list(REMOVE_ITEM THIS_SOURCE_FILES_LIST ${CMAKE_SOURCE_DIR}/common/BoostRegexValidator.cpp ${CMAKE_SOURCE_DIR}/common/GetUUID.cpp)
 

core/common/http/AsynCurlRunner.cpp

@@ -194,9 +194,9 @@ void AsynCurlRunner::HandleCompletedRequests() {
                 }
                 default:
                     // considered as network error
-                    if (++request->mTryCnt <= request->mMaxTryCnt) {
+                    if (request->mTryCnt <= request->mMaxTryCnt) {
                         LOG_WARNING(sLogger,
-                                    ("failed to send request", "retry immediately")("retryCnt", request->mTryCnt)(
+                                    ("failed to send request", "retry immediately")("retryCnt", request->mTryCnt++)(
                                         "errMsg", curl_easy_strerror(msg->data.result)));
                         // free first，becase mPrivateData will be reset in AddRequestToClient
                         if (request->mPrivateData) {

core/common/http/Curl.cpp

@@ -14,7 +14,14 @@
 
 #include "common/http/Curl.h"
 
+#include <cstdint>
+#include <map>
+#include <string>
+
 #include "common/DNSCache.h"
+#include "app_config/AppConfig.h"
+#include "logger/Logger.h"
+#include "common/http/HttpResponse.h"
 
 using namespace std;
 
@@ -128,4 +135,44 @@ CURL* CreateCurlHandler(const std::string& method,
 
     return curl;
 }
+
+bool SendHttpRequest(std::unique_ptr<HttpRequest>&& request, HttpResponse& response) {
+    curl_slist* headers = NULL;
+    CURL* curl = CreateCurlHandler(request->mMethod,
+                                request->mHTTPSFlag,
+                                request->mHost,
+                                request->mPort,
+                                request->mUrl,
+                                request->mQueryString,
+                                request->mHeader,
+                                request->mBody,
+                                response,
+                                headers,
+                                request->mTimeout,
+                                AppConfig::GetInstance()->IsHostIPReplacePolicyEnabled(),
+                                AppConfig::GetInstance()->GetBindInterface());
+    if (curl == NULL) {
+        LOG_ERROR(sLogger, ("failed to init curl handler", "failed to init curl client")("request address", request.get()));
+        return false;
+    }
+    bool success = false;
+    while (request->mTryCnt <= request->mMaxTryCnt) {
+        CURLcode res = curl_easy_perform(curl);
+        if (res == CURLE_OK) {
+            long http_code = 0;
+            curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
+            response.mStatusCode = (int32_t)http_code;
+            success = true;
+            break;
+        } else {
+            LOG_WARNING(sLogger,("failed to send request", "retry immediately")("retryCnt", request->mTryCnt++)("errMsg", curl_easy_strerror(res))("request address", request.get()));
+        }
+    } 
+    if (headers != NULL) {
+        curl_slist_free_all(headers);
+    }
+    curl_easy_cleanup(curl);
+    return success;
+}
+
 } // namespace logtail

core/common/http/Curl.h

@@ -21,7 +21,9 @@
 #include <cstdint>
 #include <map>
 #include <string>
+#include <memory>
 
+#include "common/http/HttpRequest.h"
 #include "common/http/HttpResponse.h"
 
 namespace logtail {
@@ -40,4 +42,6 @@ CURL* CreateCurlHandler(const std::string& method,
                         bool replaceHostWithIp = true,
                         const std::string& intf = "");
 
+bool SendHttpRequest(std::unique_ptr<HttpRequest>&& request, HttpResponse& response);
+
 } // namespace logtail

core/common/http/HttpRequest.h

@@ -89,4 +89,5 @@ struct AsynHttpRequest : public HttpRequest {
     virtual void OnSendDone(const HttpResponse& response) = 0;
 };
 
+
 } // namespace logtail

core/plugin/flusher/sls/FlusherSLS.cpp

@@ -616,14 +616,13 @@ void FlusherSLS::OnSendDone(const HttpResponse& response, SenderQueueItem* item)
             cpt->Commit();
             cpt->IncreaseSequenceID();
         }
-
-        GetRegionConcurrencyLimiter(mRegion)->OnSuccess();
-        DealSenderQueueItemAfterSend(item, false);
         LOG_DEBUG(sLogger,
                   ("send data to sls succeeded, item address", item)("request id", slsResponse.mRequestId)(
                       "config", configName)("region", mRegion)("project", mProject)("logstore", data->mLogstore)(
                       "response time", curTime - data->mLastSendTime)("total send time", curTime - data->mEnqueTime)(
                       "try cnt", data->mTryCnt)("endpoint", data->mCurrentEndpoint)("is profile data", isProfileData));
+        GetRegionConcurrencyLimiter(mRegion)->OnSuccess();
+        DealSenderQueueItemAfterSend(item, false);
     } else {
         OperationOnFail operation;
         SendResult sendResult = ConvertErrorCode(slsResponse.mErrorCode);

core/unittest/common/CMakeLists.txt

@@ -51,6 +51,9 @@ target_link_libraries(http_request_timer_event_unittest ${UT_BASE_TARGET})
 add_executable(timer_unittest timer/TimerUnittest.cpp)
 target_link_libraries(timer_unittest ${UT_BASE_TARGET})
 
+add_executable(curl_unittest http/CurlUnittest.cpp)
+target_link_libraries(curl_unittest ${UT_BASE_TARGET})
+
 include(GoogleTest)
 gtest_discover_tests(common_simple_utils_unittest)
 gtest_discover_tests(common_logfileoperator_unittest)
@@ -62,3 +65,5 @@ gtest_discover_tests(yaml_util_unittest)
 gtest_discover_tests(safe_queue_unittest)
 gtest_discover_tests(http_request_timer_event_unittest)
 gtest_discover_tests(timer_unittest)
+gtest_discover_tests(curl_unittest)
+

core/unittest/common/http/CurlUnittest.cpp

@@ -0,0 +1,44 @@
+// Copyright 2024 iLogtail Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/http/HttpRequest.h"
+#include "common/http/HttpResponse.h"
+#include "common/http/Curl.h"
+#include "unittest/Unittest.h"
+
+
+using namespace std;
+
+namespace logtail {
+
+class CurlUnittest : public ::testing::Test {
+public:
+    void TestSendHttpRequest();
+};
+
+
+void CurlUnittest::TestSendHttpRequest() {
+    std::unique_ptr<HttpRequest> request;
+    HttpResponse res;
+    request = std::make_unique<HttpRequest>("GET", false, "example.com", 80, "/path", "", map<string, string>(), "", 10, 3);
+    bool success = SendHttpRequest(std::move(request), res);
+    APSARA_TEST_TRUE(success);
+    APSARA_TEST_EQUAL(404, res.mStatusCode);
+}
+
+UNIT_TEST_CASE(CurlUnittest, TestSendHttpRequest)
+
+} // namespace logtail
+
+UNIT_TEST_MAIN