-
Notifications
You must be signed in to change notification settings - Fork 71
/
in-docker.py
executable file
·215 lines (179 loc) · 8.37 KB
/
in-docker.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
#!/usr/bin/env python
# Copyright 2019 Cloudera Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Script that sets up an environment used in order to perform full (or partial)
toolchain builds. Dockerfiles for images known to work with this script
can be found in the docker/ directory and the tags for the produced images
can be seen in the KNOWN_DOCKER_TAGS variable.
This script uses a combination of docker options and mount points to ensure
that the produced artifacts are owned by the correct user and to allow each
container to work with its own copy of the files.
* The source dir gets copied (using git ls-files) at the start of the build
* The build and check directories get mounted per-container.
The resulting artifacts of the build and the logs can be found under
docker_build/$img/build and docker_build/$img/check respectively.
One directory is created per docker-tag. Under this directory the familiar
build, check, and source directories can be found.
--docker-args arguments are passed in verbatim to docker. So for example:
in-docker.py --docker-args="-u root --env FOO=BAR" impala-toolchain-centos6 -- bash -c 'whoami; echo $FOO'
root
BAR
To get an interactive terminal:
in-docker.py --docker-args="-t" impala-toolchain-centos6 -- bash
"""
import argparse
import errno
import logging
import os
import shlex
import subprocess
import sys
import textwrap
LOG = logging.getLogger()
# Maps docker images to BUILD_TARGET_LABELs which is ultimately included
# in the path for each built package. The mapping that follows is also present
# in bin/bootstrap_toolchain.py, which depends on these strings.
KNOWN_DOCKER_TAGS = {'impala-toolchain-redhat7': 'ec2-package-centos-7',
'impala-toolchain-redhat8': 'ec2-package-centos-8',
'impala-toolchain-redhat9': 'ec2-package-rocky-9',
'impala-toolchain-sles12': 'ec2-package-sles-12',
'impala-toolchain-sles15': 'ec2-package-sles-15',
'impala-toolchain-ubuntu1604': 'ec2-package-ubuntu-16-04',
'impala-toolchain-ubuntu1804': 'ec2-package-ubuntu-18-04',
'impala-toolchain-ubuntu2004': 'ec2-package-ubuntu-20-04',
'impala-toolchain-ubuntu2204': 'ec2-package-ubuntu-22-04'}
__SOURCE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
TARGET_DIR = '/mnt'
DOCKER_CMD = ['docker',
'create',
'-i',
'-w', TARGET_DIR,
'-u', '%s:%s' % (os.geteuid(), os.getgid()),
'-v', '/etc/passwd:/etc/passwd:ro',
'-v', '/etc/group:/etc/group:ro',
'-v', '{SOURCE_DIR}:{TARGET_DIR}'.format(SOURCE_DIR=__SOURCE_DIR, TARGET_DIR=TARGET_DIR)]
def parse_args():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--docker-args', default='', help=textwrap.dedent('''\
Arguments that will be passed in verbatim to `docker create`.
Note that since argparse will attempt to parse all options here, it\'s usually
required to quote this argument as follows:
--docker-args="--arg1 --arg2 --etc"
Refer to docker-create(1) for information about the supported arguments.
'''))
parser.add_argument('DOCKER_IMAGE', help='Docker image to use, tags of images known to work are: %s' % ', '.join(KNOWN_DOCKER_TAGS))
parser.add_argument('COMMAND', nargs='+')
return parser.parse_args()
def mkdir_p(d):
try:
os.makedirs(d)
except OSError as e:
if e.errno != errno.EEXIST:
raise
def prepare_source_dir():
# Since we mount our entire SOURCE_DIR, create these directories ourselves to prevent root owned directories.
# mkdir foo
# docker run -v $(pwd):/mnt -v $(pwd)/foo:/mnt/baz -w /mnt ubuntu find . -user root; find . -user root;
# ./baz
for d in ['build', 'ccache', 'check', 'source']:
mkdir_p(d)
def create_mounts_cmd(distro_build_dir):
cmd = []
for d in ['build', 'check', 'source']:
srcdir = os.path.join(distro_build_dir, d)
tgtdir = os.path.join(TARGET_DIR, d)
# Create the mountpoints first. Otherwise docker creates root owned directories.
mkdir_p(srcdir)
assert ':' not in srcdir and ':' not in tgtdir, ': in source or targetdir'
cmd += ['-v', '%s:%s' % (srcdir, tgtdir)]
return cmd
def passthrough_env(image):
env_vars = ['AWS_ACCESS_KEY_ID',
'AWS_SECRET_ACCESS_KEY',
'AWS_SESSION_TOKEN',
'BUILD_TARGET_LABEL',
'CLEAN',
'CLEAN_TMP_AFTER_BUILD',
'DEBUG',
'FAIL_ON_PUBLISH',
'KUDU_GITHUB_URL',
'KUDU_VERSION',
'PRODUCTION',
'PUBLISH_DEPENDENCIES',
'PUBLISH_DEPENDENCIES_S3',
'PUBLISH_DEPENDENCIES_ARTIFACTORY',
'SYSTEM_GCC',
'SYSTEM_CMAKE',
'S3_BUCKET',
'S3_MIRROR_BUCKET',
'TOOLCHAIN_BUILD_ID']
if 'BUILD_TARGET_LABEL' not in os.environ:
# Discard docker registry prefix, if it exists.
matches = filter(image.endswith, KNOWN_DOCKER_TAGS)
if matches:
assert len(matches) == 1
os.environ['BUILD_TARGET_LABEL'] = KNOWN_DOCKER_TAGS[matches[0]]
for program in os.listdir('source'):
env_vars.append(program.upper() + '_VERSION')
ret = []
for e in env_vars:
if e in os.environ:
ret += ['-e', '%s=%s' % (e, os.environ[e])]
return ret
def copy_source_dir(distro_build_dir):
git = subprocess.Popen(['git', 'ls-tree', '--full-tree', '-r', '--name-only', 'HEAD', 'source/'], stdout=subprocess.PIPE)
xargs = subprocess.Popen(['xargs', '-I{}', 'cp', '--parents', '{}', distro_build_dir], stdin=git.stdout)
xargs.communicate()
if git.poll() or xargs.poll():
raise Exception('Error copying source directory into container mount.')
def add_ccache_opts():
if os.environ.get('USE_CCACHE', '1').lower() not in ('1', 'true'):
return []
# In order to make it easier to handle, we keep a single ccache directory for all containers
ccache_src = os.environ.get('CCACHE_DIR', os.path.join(__SOURCE_DIR, 'build_docker/ccache'))
ccache_tgt = os.path.join(TARGET_DIR, 'ccache')
mkdir_p(ccache_src)
return ['-e', 'USE_CCACHE=1',
'-v', '{ccache_src}:{ccache_tgt}'.format(ccache_src=ccache_src, ccache_tgt=ccache_tgt),
'-e', 'CCACHE_DIR={ccache_tgt}'.format(ccache_tgt=ccache_tgt)]
def main():
args = parse_args()
logging.basicConfig(level=logging.INFO, format='%(message)s')
prepare_source_dir()
build_dir = os.environ.get('BUILD_DIR', os.path.join(__SOURCE_DIR, 'build_docker'))
distro_build_dir = os.path.join(build_dir, args.DOCKER_IMAGE.replace('/', '_'))
cmd = DOCKER_CMD
cmd += create_mounts_cmd(distro_build_dir)
cmd += add_ccache_opts()
cmd += passthrough_env(args.DOCKER_IMAGE)
cmd += shlex.split(args.docker_args)
cmd += [args.DOCKER_IMAGE]
cmd += args.COMMAND
copy_source_dir(distro_build_dir)
container_id = None
try:
LOG.info('Running:\n%s' % textwrap.fill(' '.join(cmd), 120, break_on_hyphens=False, break_long_words=False))
container_id = subprocess.check_output(cmd).strip()
subprocess.check_call(['docker', 'start', '--attach', '--interactive', container_id])
except subprocess.CalledProcessError as e:
if e.output:
LOG.error(e.output + '\n')
sys.exit(e.returncode)
finally:
if container_id:
subprocess.check_output(['docker', 'stop', container_id])
subprocess.check_output(['docker', 'rm', container_id])
if __name__ == '__main__':
main()