Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

speedup CI UT job #1606

Merged
merged 8 commits into from
Jul 18, 2024
Merged

speedup CI UT job #1606

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
1662648
Split the original UT CI job into two separate jobs: one with SPL and…
alph00 Jul 11, 2024
5b11b0a
fix
alph00 Jul 16, 2024
7f453c4
fix
alph00 Jul 16, 2024
20ae021
fix
alph00 Jul 17, 2024
e9e7621
fix
alph00 Jul 17, 2024
1ccc757
fix
alph00 Jul 17, 2024
e2c3f13
fix
alph00 Jul 17, 2024
8fcc0aa
fix
alph00 Jul 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/workflows/build-core-ut.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,14 +74,15 @@ jobs:
ENABLE_COMPATIBLE_MODE: OFF
ENABLE_STATIC_LINK_CRT: ON
WITHOUTGDB: ON
MAKE_JOBS: 8
MAKE_JOBS: 16
# BUILD_TYPE: Debug # TODO: Uncomment when memory management is refined
run: CURRENT_DIR=$(pwd) && sed -i "s|/src|$CURRENT_DIR|g" docker/Dockerfile_build && make core PATH_IN_DOCKER=$(pwd)

- name: Unit Test
run: make unittest_core

- name: Unit Test Coverage
run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --root . --lcov coverage.lcov --txt coverage.txt"
run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --root . --lcov coverage.lcov --txt coverage.txt -e \".*sdk.*\" -e \".*observer.*\" -e \".*log_pb.*\" -e \".*unittest.*\" -e \".*config_server.*\" -e \".*fuse.*\" -e \".*go_pipeline.*\""

- name: Setup Python3.10
uses: actions/setup-python@v5
Expand Down
18 changes: 9 additions & 9 deletions .github/workflows/build-core.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ name: Build Core
on:
pull_request:
paths-ignore:
- 'docs/**'
- 'example_config/**'
- 'docker/**'
- 'k8s_template/**'
- 'changes/**'
- 'licenses/**'
- 'CHANGELOG.md'
- "docs/**"
- "example_config/**"
- "docker/**"
- "k8s_template/**"
- "changes/**"
- "licenses/**"
- "CHANGELOG.md"
push:
branches:
- main
Expand All @@ -37,7 +37,7 @@ jobs:
matrix:
go-version: [1.19]
# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
runner: [ arc-runner-set-ilogtail ]
runner: [arc-runner-set-ilogtail]
fail-fast: true
steps:
- name: prepare ubuntu environment
Expand All @@ -60,7 +60,7 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}

- name: Check out code
uses: actions/checkout@v4
with:
Expand Down
20 changes: 10 additions & 10 deletions .github/workflows/build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ name: Build
on:
pull_request:
paths-ignore:
- 'docs/**'
- 'example_config/**'
- 'docker/**'
- 'k8s_template/**'
- 'changes/**'
- 'licenses/**'
- 'CHANGELOG.md'
- "docs/**"
- "example_config/**"
- "docker/**"
- "k8s_template/**"
- "changes/**"
- "licenses/**"
- "CHANGELOG.md"
push:
branches:
- main
Expand All @@ -37,7 +37,7 @@ jobs:
matrix:
go-version: [1.19]
# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
runner: [ arc-runner-set-ilogtail ]
runner: [arc-runner-set-ilogtail]
fail-fast: true
steps:
- name: prepare ubuntu environment
Expand All @@ -51,7 +51,7 @@ jobs:
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
sudo -E apt-get -qq autoremove --purge
sudo -E apt-get -qq clean

- name: Check disk space
run: |
df -hT $PWD
Expand All @@ -60,7 +60,7 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}

- name: Check out code
uses: actions/checkout@v4
with:
Expand Down
20 changes: 10 additions & 10 deletions .github/workflows/e2e-framework.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ name: E2E Core Test
on:
pull_request:
paths-ignore:
- 'docs/**'
- 'example_config/**'
- 'docker/**'
- 'k8s_template/**'
- 'changes/**'
- 'licenses/**'
- 'CHANGELOG.md'
- "docs/**"
- "example_config/**"
- "docker/**"
- "k8s_template/**"
- "changes/**"
- "licenses/**"
- "CHANGELOG.md"
push:
branches:
- main
Expand All @@ -34,8 +34,8 @@ jobs:
timeout-minutes: 60
strategy:
matrix:
go-version: [ 1.19.10 ]
runner: [ ubuntu-latest ]
go-version: [1.19.10]
runner: [ubuntu-latest]
fail-fast: true
steps:
# Clean up space to prevent action from running out of disk space.
Expand Down Expand Up @@ -81,7 +81,7 @@ jobs:
result:
runs-on: ubuntu-latest
timeout-minutes: 60
needs: [ CI ]
needs: [CI]
steps:
- name: Build Result
run: echo "Just to make the GitHub merge button green"
20 changes: 10 additions & 10 deletions .github/workflows/e2e.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ name: E2E Test
on:
pull_request:
paths-ignore:
- 'docs/**'
- 'example_config/**'
- 'docker/**'
- 'k8s_template/**'
- 'changes/**'
- 'licenses/**'
- 'CHANGELOG.md'
- "docs/**"
- "example_config/**"
- "docker/**"
- "k8s_template/**"
- "changes/**"
- "licenses/**"
- "CHANGELOG.md"
push:
branches:
- main
Expand All @@ -34,8 +34,8 @@ jobs:
timeout-minutes: 60
strategy:
matrix:
go-version: [ 1.19.10 ]
runner: [ ubuntu-latest ]
go-version: [1.19.10]
runner: [ubuntu-latest]
fail-fast: true
steps:
# Clean up space to prevent action from running out of disk space.
Expand Down Expand Up @@ -83,7 +83,7 @@ jobs:
result:
runs-on: ubuntu-latest
timeout-minutes: 60
needs: [ CI ]
needs: [CI]
steps:
- name: Build Result
run: echo "Just to make the GitHub merge button green"
93 changes: 49 additions & 44 deletions core/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,20 @@ endif ()

# Options.
option(BUILD_LOGTAIL "Build Logtail executable and tools" ON)
# Used under the Android environment.
option(BUILD_LOGTAIL_SHARED_LIBRARY "Build Logtail shared library")
option(ENABLE_ENTERPRISE "enable enterprise feature")
cmake_dependent_option(ENABLE_COMPATIBLE_MODE "Build Logtail in compatible mode (for low version Linux)" OFF "LINUX" OFF)
cmake_dependent_option(ENABLE_STATIC_LINK_CRT "Build Logtail by linking CRT statically" OFF "LINUX" OFF)
option(WITHOUTGDB "Build Logtail without gdb")
option(WITHOUTSPL "Build Logtail and UT without SPL")
option(WITHSPL "Build Logtail and UT with SPL" ON)
option(BUILD_LOGTAIL_UT "Build unit test for Logtail")

if (BUILD_LOGTAIL_SHARED_LIBRARY AND WITHSPL)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

BUILD_LOGTAIL_SHARED_LIBRARY 注释说明下应用场景

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

image BUILD_LOGTAIL_SHARED_LIBRARY 是用于生成动态库的开关，是原来就有的

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

代码注释，不是评论注释。说明下什么场景会用

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已修改

message(FATEL_ERROR, "Generating logtail shared library is not supported to be linked with SPL. WITHSPL should be set OFF.")
return()
endif()

if (ENABLE_ENTERPRISE)
message(STATUS "Enable Enterprise Feature.")
add_definitions(-D__ENTERPRISE__)
Expand All @@ -41,7 +47,7 @@ else ()
include(${CMAKE_CURRENT_SOURCE_DIR}/options.cmake)
endif ()

if (WITHOUTSPL)
if (NOT WITHSPL)
add_definitions(-D__EXCLUDE_SPL__)
endif()

Expand Down Expand Up @@ -92,7 +98,8 @@ endif ()
include(${CMAKE_CURRENT_SOURCE_DIR}/utils.cmake)
include(${CMAKE_CURRENT_SOURCE_DIR}/dependencies.cmake)
include(${CMAKE_CURRENT_SOURCE_DIR}/links.cmake)
set(SUBDIR_SOURCE_FILES "")
set(PLUGIN_SOURCE_FILES_CORE "")
set(PLUGIN_SOURCE_FILES_SPL "")
include(${CMAKE_CURRENT_SOURCE_DIR}/common/common.cmake)
include(${CMAKE_CURRENT_SOURCE_DIR}/common/links.cmake)
include(${CMAKE_CURRENT_SOURCE_DIR}/input/input.cmake)
Expand Down Expand Up @@ -127,7 +134,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR})
include_directories("/opt/logtail_spl/include")
if (LINUX)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/observer)
if (NOT WITHOUTSPL)
if (WITHSPL)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/spl)
endif()
endif()
Expand All @@ -138,37 +145,39 @@ endforeach (DIR_NAME)

foreach (DIR_NAME ${SUB_DIRECTORIES_LIST})
file(GLOB TEMP_SOURCE_FILES ${DIR_NAME}/*.c ${DIR_NAME}/*.cc ${DIR_NAME}/*.cpp)
list(APPEND SOURCE_FILES_LIST ${TEMP_SOURCE_FILES})
list(APPEND FRAMEWORK_SOURCE_FILES ${TEMP_SOURCE_FILES})
endforeach (DIR_NAME)

if (ENABLE_ENTERPRISE)
# remove several files in shennong/sdk
list(REMOVE_ITEM SOURCE_FILES_LIST ${CMAKE_CURRENT_SOURCE_DIR}/shennong/sdk/sample.cpp)
list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/shennong/sdk/sample.cpp)
endif()

# remove several files in go_pipeline
list(REMOVE_ITEM SOURCE_FILES_LIST ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.h)
list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/go_pipeline/LogtailPluginAdapter.h)

if(MSVC)
# remove linux event listener
file(GLOB REMOVE_EVENT_LISTENER_SOURCES event_listener/*_Linux.cpp event_listener/*_Linux.h)
list(REMOVE_ITEM SOURCE_FILES_LIST ${REMOVE_EVENT_LISTENER_SOURCES})
set(WINDOWS_SOURCE_FILES ${SOURCE_FILES_LIST} ${SUBDIR_SOURCE_FILES})
list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${REMOVE_EVENT_LISTENER_SOURCES})
elseif(UNIX)
# remove windows event listener
file(GLOB REMOVE_EVENT_LISTENER_SOURCES event_listener/*_Windows.cpp event_listener/*_Windows.h)
list(REMOVE_ITEM SOURCE_FILES_LIST ${REMOVE_EVENT_LISTENER_SOURCES})
list(REMOVE_ITEM FRAMEWORK_SOURCE_FILES ${REMOVE_EVENT_LISTENER_SOURCES})
if (LINUX)
# observer
file(GLOB_RECURSE APPEND_OBSERVER_SOURCES observer/*)
list(APPEND SOURCE_FILES_LIST ${APPEND_OBSERVER_SOURCES})
list(APPEND FRAMEWORK_SOURCE_FILES ${APPEND_OBSERVER_SOURCES})
if (WITHSPL)
set(SRC_FILES ${PLUGIN_SOURCE_FILES_SPL})
endif()
endif()
set(UNIX_SOURCE_FILES ${SOURCE_FILES_LIST} ${SUBDIR_SOURCE_FILES})
endif()
set(SRC_FILES ${SRC_FILES} ${FRAMEWORK_SOURCE_FILES} ${PLUGIN_SOURCE_FILES_CORE})

# Generate SPL library.
if (LINUX)
if (NOT WITHOUTSPL)
if (WITHSPL)
add_subdirectory(spl)
endif()
endif()
Expand All @@ -177,47 +186,35 @@ endif()
if (BUILD_LOGTAIL)
if (ENABLE_ENTERPRISE)
if (UNIX)
add_executable(${LOGTAIL_TARGET} enterprise_logtail.cpp ${UNIX_SOURCE_FILES})
add_executable(${LOGTAIL_TARGET} enterprise_logtail.cpp ${SRC_FILES})
elseif (MSVC)
add_executable(${LOGTAIL_TARGET} enterprise_logtail_windows.cpp ${WINDOWS_SOURCE_FILES})
add_executable(${LOGTAIL_TARGET} enterprise_logtail_windows.cpp ${SRC_FILES})
endif ()
else ()
if (UNIX)
add_executable(${LOGTAIL_TARGET} logtail.cpp ${UNIX_SOURCE_FILES})
add_executable(${LOGTAIL_TARGET} logtail.cpp ${SRC_FILES})
elseif (MSVC)
add_executable(${LOGTAIL_TARGET} logtail_windows.cpp ${WINDOWS_SOURCE_FILES})
add_executable(${LOGTAIL_TARGET} logtail_windows.cpp ${SRC_FILES})
endif ()
endif()
endif()

if (BUILD_LOGTAIL_SHARED_LIBRARY)
if (NOT WITHOUTSPL)
message(ERROR, "generating logtail shared library is not supported with SPL.")
else()
if (ENABLE_ENTERPRISE)
if (UNIX)
add_library(${LOGTAIL_TARGET} SHARED enterprise_logtail.cpp ${UNIX_SOURCE_FILES})
elseif (MSVC)
add_library(${LOGTAIL_TARGET} SHARED enterprise_logtail_windows.cpp ${WINDOWS_SOURCE_FILES})
endif ()
else ()
if (UNIX)
add_library(${LOGTAIL_TARGET} SHARED logtail.cpp ${UNIX_SOURCE_FILES})
elseif (MSVC)
add_library(${LOGTAIL_TARGET} SHARED logtail_windows.cpp ${WINDOWS_SOURCE_FILES})
endif ()
endif()
if (BUILD_LOGTAIL_SHARED_LIBRARY)
if (ENABLE_ENTERPRISE)
if (UNIX)
add_library(${LOGTAIL_TARGET} SHARED enterprise_logtail.cpp ${SRC_FILES})
elseif (MSVC)
add_library(${LOGTAIL_TARGET} SHARED enterprise_logtail_windows.cpp ${SRC_FILES})
endif ()
else ()
if (UNIX)
add_library(${LOGTAIL_TARGET} SHARED logtail.cpp ${SRC_FILES})
elseif (MSVC)
add_library(${LOGTAIL_TARGET} SHARED logtail_windows.cpp ${SRC_FILES})
endif ()
endif()
endif ()

# Logtail UT.
if (BUILD_LOGTAIL_UT)
message(STATUS "Build unittest.")
include(CTest)
enable_testing()
add_subdirectory(unittest)
endif ()

# Generate independent libraries.
# add_subdirectory(helper)
add_subdirectory(go_pipeline)
Expand All @@ -226,8 +223,16 @@ add_subdirectory(common)
# Link libraries.
if(BUILD_LOGTAIL OR BUILD_LOGTAIL_SHARED_LIBRARY)
input_link(${LOGTAIL_TARGET})
processor_link(${LOGTAIL_TARGET})
processor_link(${LOGTAIL_TARGET} ${WITHSPL})
flusher_link(${LOGTAIL_TARGET})
all_link(${LOGTAIL_TARGET})
common_link(${LOGTAIL_TARGET})
endif()
endif()

# Logtail UT.
if (BUILD_LOGTAIL_UT)
message(STATUS "Build unittest.")
include(CTest)
enable_testing()
add_subdirectory(unittest)
endif ()
2 changes: 1 addition & 1 deletion core/common/common.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,4 @@ elseif(UNIX)
endif()

# Set source files to parent
set(SUBDIR_SOURCE_FILES ${SUBDIR_SOURCE_FILES} ${THIS_SOURCE_FILES_LIST})
set(PLUGIN_SOURCE_FILES_CORE ${PLUGIN_SOURCE_FILES_CORE} ${THIS_SOURCE_FILES_LIST})
Loading
Loading