forked from sagemath/sage
-
Notifications
You must be signed in to change notification settings - Fork 0
/
.gitlab-ci.yml
195 lines (178 loc) · 7.21 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
# This file configures automatic builds of Sage on [GitLab](https://gitlab.com).
# To make the build time not too excessive, we seed the build cache with
# sagemath/sagemath-dev:develop. When basic SPKGs changed, this does not help
# much and the full build might exceed the set time limit in GitLab. You can
# increase that limit in Settings → CI/CD.
# You can also provision your own private more powerful runner in the same
# place
# https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker-executor;
# or set up your favourite cloud service to provide an on-demand autoscale
# runner. More details below.
# As of early 2018 a run on GitLab CI takes about 45 minutes. We could probably
# save 10 minutes by not building/pushing/testing dev images for branches other
# than master/develop.
# Note that most of the time during CI is spent with pulling and pushing of
# docker images and copying files locally as part of the docker build. At the
# moment there is no reliable way of passing the docker images to the following
# stages without explicit pushing/pulling or similar:
# https://gitlab.com/gitlab-org/gitlab-runner/issues/1107
# The timings mentioned above are typical values. The shared runners provided
# on gitlab.com are sometimes much slower depending on the runner you are
# scheduled on. Sometimes it's slower for no apparent reason, probably just an
# overcommittment of virtual machines on hardware.
# GitLab provides several flavours of shared runners (as of early 2018):
# * runners tagged as "do" (digitalocean.com) provide about 60GB of HDD, two
# cores, but only 2GB of RAM. The RAM is sometimes not sufficient to build
# the documentation.
# * runners tagged as "gce" (Google Compute Engine) provide about 22GB of HDD,
# a single core, 4GB of RAM. Since we are relying on OverlayFS, the disk
# space is not sufficient to build sage from scratch.
# The shared runners are terminated after three hours. Currently, this is often
# insufficient to build sage from scratch.
# If you want to provide your own runners, make sure to tag them as follows:
# * "big" (60GB of disk space are available) to make build-from-clean pass.
image: docker:stable
stages:
- build
- test
- release
variables:
DOCKER_TAG: $CI_COMMIT_REF_NAME
# Builds are very I/O intensive; make sure we have a fast file system.
DOCKER_DRIVER: overlay2
DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:develop
before_script:
# GitLab has no mechanism yet to hide secret variables: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784
# So we roll our own which protects all variables that start with SECRET_
- . .ci/protect-secrets.sh
# Collect debug infos about the system we are running on
- .ci/describe-system.sh
# Set DOCKER_TAG according to the current branch/tag
- . .ci/update-env.sh
# Set MAKEFLAGS and SAGE_NUM_THREADS according to the machine we are running on
- . .ci/setup-make-parallelity.sh
# We use docker-in-docker to build our docker images, i.e., we run a
# docker:dind "service" container and link to it from the container running the
# actual scripts below.
# Our scripts automatically connect to this service (unless you override it by
# setting DOCKER_HOST.) For example, each RUN statement in the Dockerfile
# spawns a docker container inside the docker:dind container to perform the RUN
# command there.
# It can be faster to expose your outer docker daemon by mounting
# /var/run/docker.sock to /var/run/docker.sock and setting DOCKER_HOST in
# Settings -> CI/CD -> Secret variable to unix:///var/run/docker.sock. (The
# speedup is mostly due to sharing layers of intermediate images.) However,
# this is only possible if you provision your own runners. Shared gitlab
# runners, do not bind mount /var/run/docker.sock. Also, docker:dind provides
# better isolation. If you expect many builds to run simultaneously on a host,
# conflicting tags can cause issues with a mounted DOCKER_HOST.
services:
- docker:stable-dind
# Build Sage and its documentation.
# The build starts from the build artifacts of DEFAULT_ARTIFACT_BASE which is
# usually much faster than building from a clean checkout of Sage.
build-from-latest:
stage: build
artifacts:
when: always
paths:
- gitlab-build-docker.log
- html
expire_in: 1 month
script:
- apk --update add coreutils rsync
# The output of the build can get larger than gitlab.com's limit; only
# print the first 1MB (and the last 80 lines.) GitLab's limit is 4MB,
# however, the list of all branches and tags that shows up in the initial
# checkout takes already 1.5 MB:
# https://gitlab.com/gitlab-org/gitlab-runner/issues/4142
- .ci/build-docker.sh | tee gitlab-build-docker.log | .ci/head-tail.sh 1048576
- .ci/push-gitlab.sh sagemath-dev
- .ci/push-gitlab.sh sagemath
- DOCKER_TAG=$CI_COMMIT_SHA .ci/push-gitlab.sh sagemath
except:
- master
- develop
- tags
- web
# Build Sage and its documentation from a clean checkout of Sage.
# Note that this takes several hours. You probably want to run this on your own
# gitlab-runner and increase the standard GitLab time limit for CI runs.
# Some of the shared runners provided by GitLab for free do not have enough
# disk space for this to work. If a build fails with "no space left on device",
# you could just retry it and hope to be scheduled on a machine with more disk
# space, or provision your own runner.
build-from-clean:
extends:
- build-from-latest
artifacts:
when: always
paths:
- gitlab-build-docker.log
- html
expire_in: 99 years
variables:
ARTIFACT_BASE: "source-clean"
only:
- master
- develop
- tags
# Run build-from-clean for a pipeline that has been explicitly created
# through GitLab's web interface.
- web
except: []
tags:
# 60 GB of HDD are available
- big
# This build takes several CPU hours. It is very unlikely that there are any
# actual build errors for a tagged release but the (discounted) cloud
# machines this is running on might be preempted during the long build time.
# So let's try three times before we give up.
retry: 2
test-dev:
stage: test
dependencies: []
script:
- . .ci/pull-gitlab.sh sagemath-dev
- sh .ci/test-dev.sh "$DOCKER_IMAGE"
test-cli:
stage: test
dependencies: []
script:
- . .ci/pull-gitlab.sh sagemath
- sh .ci/test-cli.sh "$DOCKER_IMAGE"
test-jupyter:
stage: test
dependencies: []
script:
- . .ci/pull-gitlab.sh sagemath
- sh .ci/test-jupyter.sh "$DOCKER_IMAGE" docker
# Pushes the built images to Docker Hub if the Settings -> CI/CD -> Secret
# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up.
push-dockerhub:
stage: release
dependencies: []
only:
refs:
- branches
- tags
variables:
- $SECRET_DOCKER_PASS
script:
- . .ci/pull-gitlab.sh sagemath
- sh .ci/push-dockerhub.sh sagemath
# Pushes the built dev images to Docker Hub if the Settings -> CI/CD -> Secret
# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up.
push-dockerhub-dev:
stage: release
dependencies: []
only:
refs:
- master
- develop
- tags
variables:
- $SECRET_DOCKER_PASS
script:
- . .ci/pull-gitlab.sh sagemath-dev
- sh .ci/push-dockerhub.sh sagemath-dev