forked from GafferHQ/gaffer
-
Notifications
You must be signed in to change notification settings - Fork 0
304 lines (263 loc) · 12.6 KB
/
main.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
name: CI
on:
push:
branches:
- main
- '*_maintenance'
- 'macCI'
pull_request:
branches:
- '*'
release:
types: [published]
jobs:
build:
strategy:
# Don't cancel other jobs in the build matrix if one job fails.
fail-fast: false
matrix:
# Rather than generate all permutations of various settings,
# we want to explicitly list each of the variants we want to
# test. We can use `name` to declare the names of our variants,
# and then use `include` to define their settings.
name: [
linux-gcc11,
linux-debug-gcc11,
windows,
macos-arm64,
]
include:
- name: linux-gcc11
os: ubuntu-20.04
buildType: RELEASE
publish: true
containerImage: ghcr.io/gafferhq/build/build:3.0.0
# GitHub container builds run as root. This causes failures for tests that
# assert that filesystem permissions are respected, because root doesn't
# respect permissions. So we run the final test suite as a dedicated
# test user rather than as root.
testRunner: su testUser -c
sconsCacheMegabytes: 400
jobs: 4
- name: linux-debug-gcc11
os: ubuntu-20.04
buildType: DEBUG
publish: false
containerImage: ghcr.io/gafferhq/build/build:3.0.0
testRunner: su testUser -c
testArguments: -excludedCategories performance
# Debug builds are ludicrously big, so we must use a larger cache
# limit. In practice this compresses down to 4-500Mb.
sconsCacheMegabytes: 2500
jobs: 4
- name: windows
os: windows-2022
buildType: RELEASE
publish: true
containerImage:
testRunner: Invoke-Expression
testArguments: -excludedCategories performance GafferTest GafferVDBTest GafferUSDTest GafferSceneTest GafferDispatchTest GafferOSLTest GafferImageTest GafferUITest GafferImageUITest GafferSceneUITest GafferDispatchUITest GafferOSLUITest GafferUSDUITest GafferVDBUITest GafferDelightUITest GafferTractorTest GafferTractorUITest
sconsCacheMegabytes: 800
jobs: 4
- name: macos-arm64
os: macos-14
buildType: RELEASE
publish: false
containerImage:
testRunner: bash -c
testArguments: -excludedCategories performance
sconsCacheMegabytes: 400
jobs: 3
runs-on: ${{ matrix.os }}
container: ${{ matrix.containerImage }}
env:
ARNOLD_LICENSE_ORDER: none # Don't waste time looking for a license that doesn't exist
ARNOLD_FORCE_ABORT_ON_LICENSE_FAIL: 0 # And don't abort because the license isn't found
GAFFER_BUILD_DIR: "./build"
GAFFER_CACHE_DIR: "./sconsCache"
steps:
- uses: actions/checkout@v4
- uses: ilammy/[email protected]
with:
sdk: 10.0.19041.0
- name: Install toolchain (Windows)
run: |
python -m pip install scons
python -m pip install sphinx==4.3.1 sphinx_rtd_theme==1.0.0 myst-parser==0.15.2 docutils==0.17.1 sphinxcontrib-applehelp==1.0.2 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-htmlhelp==2.0.0 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-serializinghtml==1.1.5 sphinxcontrib-qthelp==1.0.3
Invoke-WebRequest -Uri "https://inkscape.org/gallery/item/37363/inkscape-1.2.2_2022-12-09_732a01da63-x64.exe" -OutFile "inkscape.exe"
Start-Process .\inkscape.exe /S -NoNewWindow -Wait
shell: pwsh
if: runner.os == 'Windows'
- name: Install toolchain (Linux)
run: |
Xvfb :99 -screen 0 1280x1024x24 &
metacity --display :99.0 &
useradd -m testUser
echo LD_PRELOAD=libSegFault.so >> $GITHUB_ENV
# The Docker container configures bash shells such that they enable the
# software collections we want. If we could set GitHub's
# `defaults.run.shell` to `bash` then all our build steps would pick up
# this environment automatically. But we can't do that because it
# breaks the build on Windows, and we can't configure a different shell
# per platform because GitHub won't allow it. But we can run _this_
# Linux-only step in bash, and transfer the environment out to be used
# in later steps.
echo $PATH > $GITHUB_PATH
echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH >> $GITHUB_ENV
echo DISPLAY=:99.0 >> $GITHUB_ENV
shell: bash
if: runner.os == 'Linux'
- name: Install toolchain (macOS)
run: |
# Choose the earliest Xcode version available on a macos-14 runner.
sudo xcode-select -s /Applications/Xcode_14.3.1.app/Contents/Developer
sudo pip3 install scons==4.6.0 --break-system-packages
# Install docs dependencies matching the versions specified in the
# linux build container.
sudo pip3 install sphinx==4.3.1 sphinx_rtd_theme==1.0.0 myst-parser==0.15.2 docutils==0.17.1 sphinxcontrib-applehelp==1.0.4 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-htmlhelp==2.0.1 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-serializinghtml==1.1.5 sphinxcontrib-qthelp==1.0.3 --break-system-packages
# Install Inkscape.
brew install --cask inkscape
# Remove CommandLineTools so there is no potential for conflict with Xcode
sudo rm -rf /Library/Developer/CommandLineTools
if: runner.os == 'macOS'
- name: 'Install Python Modules'
run: |
python --version
pip install PyJWT==1.7.1 PyGitHub==1.45
- name: Set Custom Variables
run: |
.github/workflows/main/setBuildVars.py
echo GAFFER_SPHINX=`which sphinx-build` >> $GITHUB_ENV
env:
GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GAFFER_BUILD_VARIANT: ${{ matrix.name }}
shell: bash
- name: Install dependencies
# The `$GITHUB_ENV` shenanigans creates an environment variable
# containing the hash of the archive, for use in the cache key
# below.
run: |
echo GAFFER_DEPENDENCIES_HASH=`python .github/workflows/main/installDependencies.py ${{ matrix.dependenciesURL != '' && format( '--archiveURL {0}', matrix.dependenciesURL ) || '' }} --dependenciesDir ${{ env.GAFFER_BUILD_DIR }} --outputFormat "{archiveDigest}"` >> $GITHUB_ENV
./.github/workflows/main/installDelight.py
echo DELIGHT=$GITHUB_WORKSPACE/3delight >> $GITHUB_ENV
shell: bash
- name: Install Mesa (Windows)
# Installed after dependencies to avoid errors from python related to existing directory `bin`.
# Adapted from Mesa's `systemwidedeploy.cmd` `osmesa` branch.
run: |
curl.exe -L --output mesa.7z --url https://github.com/pal1000/mesa-dist-win/releases/download/22.3.1/mesa3d-22.3.1-release-msvc.7z
& "C:\Program Files\7-Zip\7z.exe" x mesa.7z -omesa
./mesa/systemwidedeploy.cmd 1
echo "GALLIUM_DRIVER=llvmpipe" >> $env:GITHUB_ENV
Set-DisplayResolution -Width 1280 -Height 1024 -Force
shell: pwsh
if: runner.os == 'Windows'
- name: Cache
uses: actions/cache@v4
with:
path: ${{ env.GAFFER_CACHE_DIR }}
key: ${{ runner.os }}-${{ matrix.containerImage }}-${{env.GAFFER_DEPENDENCIES_HASH}}-${{ matrix.buildType }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-${{ matrix.containerImage }}-${{env.GAFFER_DEPENDENCIES_HASH}}-${{ matrix.buildType }}-
- name: Build Gaffer
run: |
scons -j ${{ matrix.jobs }} build BUILD_TYPE=${{ matrix.buildType }} OPTIONS=.github/workflows/main/sconsOptions
env:
PYTHONUTF8: 1
- name: Test
# Tests should complete in well under an hour. If they don't it's most likely because
# of a hang, in which case we'd like to know more quickly than the default 6hr timeout
# allows.
timeout-minutes: 60
run: |
echo "::add-matcher::./.github/workflows/main/problemMatchers/unittest.json"
${{ matrix.testRunner }} "${{ env.GAFFER_BUILD_DIR }}/bin/gaffer test ${{ matrix.testArguments }}"
echo "::remove-matcher owner=unittest::"
- name: Build and test Arnold extension
run: |
import subprocess
import sys
import os
for arnoldVersion in [ "7.2.1.0", "7.3.1.0" ] :
arnoldRoot = os.path.join( os.environ["GITHUB_WORKSPACE"], "arnoldRoot", arnoldVersion )
os.environ["ARNOLD_ROOT"] = arnoldRoot
subprocess.check_call(
[
sys.executable,
".github/workflows/main/installArnold.py",
"--version",
arnoldVersion
]
)
#Build Arnold extension
subprocess.check_call( "scons -j ${{ matrix.jobs }} build BUILD_TYPE=${{ matrix.buildType }} OPTIONS=.github/workflows/main/sconsOptions", shell = True )
if os.name != "nt" :
# Test Arnold extension
print( "::add-matcher::./.github/workflows/main/problemMatchers/unittest.json" )
subprocess.check_call( "${{ matrix.testRunner }} \"" + os.path.join( os.environ["GAFFER_BUILD_DIR"], "bin", "gaffer" ) + " test IECoreArnoldTest GafferArnoldTest GafferArnoldUITest\"", shell = True )
print( "::remove-matcher owner=unittest::" )
# Publish ARNOLD_ROOT to the environment for subsequent steps,
# so we can build the docs for GafferArnold.
with open( os.environ["GITHUB_ENV"], "a" ) as f :
print( "Setting $ARNOLD_ROOT to '%s'" % arnoldRoot )
f.write( 'ARNOLD_ROOT=%s\n' % arnoldRoot )
env:
PYTHONUTF8: 1
ARNOLD_DOWNLOAD_USER: ${{ secrets.ARNOLD_DOWNLOAD_USER }}
ARNOLD_DOWNLOAD_PASSWORD: ${{ secrets.ARNOLD_DOWNLOAD_PASSWORD }}
shell: python
# We can't build GafferArnold for pull requests from forks, because the
# authentication secrets for downloading Arnold aren't available (this is
# a GitHub security restriction). The core development team can still build
# GafferArnold for pull requests by pushing a branch to the main repo and
# making a pull request from there.
if: ${{ env.ARNOLD_DOWNLOAD_USER != '' }}
- name: Build Docs and Package
# Docs builds should be relatively quick. If there is a problem, this
# aborts them in a more timely fashion than the default 6hr timeout.
timeout-minutes: 20
run: |
# Treats warnings-as-errors so we know about broken links
echo "::add-matcher::./.github/workflows/main/problemMatchers/sphinx.json"
scons -j ${{ matrix.jobs }} package BUILD_TYPE=${{ matrix.buildType }} OPTIONS=.github/workflows/main/sconsOptions
echo "::remove-matcher owner=sphinx::"
env:
PYTHONUTF8: 1
if: matrix.publish
- name: Validate
run: |
echo "::add-matcher::./.github/workflows/main/problemMatchers/validateRelease.json"
python ./config/validateRelease.py --archive ${{ env.GAFFER_BUILD_NAME }}.${{ env.PACKAGE_EXTENSION }} ${{ env.GAFFER_VALIDATE_EXTRA_FLAGS }} ${{ secrets.ARNOLD_DOWNLOAD_USER == '' && '--skipPaths arnold' || '' }}
echo "::remove-matcher owner=validateRelease::"
if: matrix.publish
- uses: actions/upload-artifact@v4
with:
name: ${{ env.GAFFER_BUILD_NAME }}
path: ${{ env.GAFFER_BUILD_NAME }}.${{ env.PACKAGE_EXTENSION }}
# Using compression-level 0 avoids compressing our already compressed
# package and results in a significantly faster upload.
compression-level: 0
if: matrix.publish
- name: Publish Release
run: |
python ./config/publishRelease.py --archive ${{ env.GAFFER_BUILD_NAME }}.${{ env.PACKAGE_EXTENSION }} --repo ${{ github.repository }} --releaseId ${{ env.GAFFER_GITHUB_RELEASEID }}
if: matrix.publish && env.GAFFER_GITHUB_RELEASEID != ''
env:
GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Limit cache size
# GitHub has a limit of 5G for all caches in a repository. Because we write new
# files into `./sconsCache` with every build, we must trim its size to avoid
# unbounded growth. In practice, the archives that get uploaded are much smaller
# than the limit we apply here, because they're compressed.
run: python ./.github/workflows/main/limitDirectorySize.py --directory ./sconsCache --megabytes ${{ matrix.sconsCacheMegabytes }} --verbose
- name: Debug Failures
run: |
# Print SCons logs
shopt -s nullglob
for logFile in config.log
do
echo $logFile
cat $logFile
done
if: failure()
shell: bash