forked from horovod/horovod
-
Notifications
You must be signed in to change notification settings - Fork 0
160 lines (138 loc) · 6.2 KB
/
codeql-analysis.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
name: "CodeQL"
on:
push:
branches: [ master ]
pull_request:
# The branches below must be a subset of the branches above
branches: [ master ]
schedule:
- cron: '32 10 * * 5'
concurrency:
# This controls which concurrent builds to cancel:
# - we do not want any concurrent builds on a branch (pull_request)
# - we do not want concurrent builds on the same commit on master (push)
# - we do not want concurrent builds on the same commit on a tag (push)
# - we allow concurrent runs on the same commit on master and its tag (push)
# - we allow concurrent runs on the same commit on master (push) and a scheduled build (schedule)
#
# A pull_request event only runs on branch commit, a push event only on master and tag commit.
# A schedule event only runs on master HEAD commit.
#
# Expression github.ref means something like refs/heads/master or refs/tags/v0.22.1 or the branch.
# This helps to not cancel concurrent runs on master or a tag that share the same commit.
# Expression github.head_ref refers to the branch of the pull request.
# On master, github.head_ref is empty, so we use the SHA of the commit, this means individual
# commits to master will not be cancelled, while there can only be one concurrent build on a branch.
#
# We include the event name to we allow for concurrent scheduled and master builds.
group: codeql-${{ github.event_name }}-${{ github.ref }}-${{ github.head_ref || github.sha }}
cancel-in-progress: true
jobs:
analyze-python:
name: Analyze Python
runs-on: ubuntu-latest
permissions:
security-events: write
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Install Python dependencies
# install all `install_requires` dependencies but no `extras_require` dependencies
run: |
python setup.py egg_info
pip install $(head -n $(( $(grep -m 1 -n -e "^\[" horovod.egg-info/requires.txt | cut -d : -f 1) - 1 )) horovod.egg-info/requires.txt)
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
with:
languages: python
setup-python-dependencies: false
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
analyze-cpp:
name: Analyze C++
runs-on: ubuntu-latest
permissions:
security-events: write
steps:
- name: Clean up disk space
# deleting these paths frees 38 GB disk space:
# sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
# but this sometimes takes 3-4 minutes
# so we delete only some sub-paths which are known to be quick (10s) and 20 GB
run: |
echo ::group::Disk space before clean up
df -h
echo ::endgroup::
for dir in /usr/share/dotnet/sdk/\*/nuGetPackagesArchive.lzma \
/usr/share/dotnet/shared \
/usr/local/lib/android/sdk/ndk \
/usr/local/lib/android/sdk/build-tools \
/opt/ghc
do
echo ::group::Deleting "$dir"
sudo du -hsc $dir | tail -n1 || true
sudo rm -rf $dir
echo ::endgroup::
done
echo ::group::Disk space after clean up
df -h
echo ::endgroup::
- name: Checkout repository
uses: actions/checkout@v3
with:
submodules: recursive
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
with:
languages: cpp
- name: Add CodeQL to Dockerfile.test.?pu
run: |
# copy CodeQL distribution into docker context
cp -rl "${{ env.CODEQL_DIST }}" .codeql
command=$(cat <<EOF
# Install CodeQL
COPY .codeql ${{ env.CODEQL_DIST }}
RUN ${{ env.CODEQL_DIST }}/codeql version --format=terse
RUN ${{ env.CODEQL_DIST }}/codeql version --format=json
RUN ${{ env.CODEQL_DIST }}/codeql resolve queries cpp-code-scanning.qls --format=bylanguage
RUN ${{ env.CODEQL_DIST }}/codeql resolve languages
RUN ${{ env.CODEQL_DIST }}/codeql resolve qlpacks
EOF
)
sed -i -e "s%^# setup ssh service$%${command//$'\n'/\\n}\n\n# setup ssh service%" Dockerfile.test.?pu
command=$(cat <<EOF
# Setup CodeQL tracing
RUN mkdir -p /home/runner/work/horovod
RUN ln -s /horovod /home/runner/work/horovod/horovod
RUN mkdir -p /home/runner/work/_temp
RUN ${{ env.CODEQL_DIST }}/codeql database init --db-cluster /home/runner/work/_temp/codeql_databases --source-root=/home/runner/work/horovod/horovod --language=cpp --begin-tracing --trace-process-name=Runner.Worker.exe
# Build Horovod (C++)
RUN mkdir -p build/temp
RUN cd build/temp; source /home/runner/work/_temp/codeql_databases/temp/tracingEnvironment/start-tracing.sh; cmake /home/runner/work/horovod/horovod -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_LIBRARY_OUTPUT_DIRECTORY_RELWITHDEBINFO=/home/runner/work/horovod/horovod/build/temp/lib.linux-x86_64-3.8 -DPYTHON_EXECUTABLE:FILEPATH=/usr/bin/python
RUN cd build/temp; source /home/runner/work/_temp/codeql_databases/temp/tracingEnvironment/start-tracing.sh; cmake --build . --config RelWithDebInfo -- VERBOSE=1
EOF
)
sed -i -e "s%^# Install Horovod.$%${command//$'\n'/\\n}\n\n# Install Horovod.%" Dockerfile.test.?pu
# Truncate Dockerfile.test.?pu, after CMake we are done here
for file in Dockerfile.test.?pu
do
head -n $(( $(grep -m 1 -n "# Install Horovod." $file | cut -d : -f 1) - 1 )) $file > tmp
mv tmp $file
done
# Print out changes
git diff
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Build Horovod
shell: bash
run: |
pip install docker-compose
image=$(grep mixed docker-compose.test.yml | sed -e "s/[ :]//g")
docker-compose -f docker-compose.test.yml build ${image}
docker run --name horovod horovod_${image} ls -lah /home/runner/work/_temp/codeql_databases
rm -rf /home/runner/work/_temp/codeql_databases
docker cp horovod:/home/runner/work/_temp/codeql_databases /home/runner/work/_temp/
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2