forked from opendatahub-io/notebooks
-
Notifications
You must be signed in to change notification settings - Fork 7
219 lines (187 loc) · 8.57 KB
/
build-notebooks-TEMPLATE.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
# inspired by
# https://github.com/thesuperzapper/kubeflow/blob/master/.github/workflows/example_notebook_servers_publish_TEMPLATE.yaml
---
name: Build & Publish Notebook Servers (TEMPLATE)
"on":
workflow_call:
inputs:
# https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
# https://docs.github.com/en/actions/learn-github-actions/contexts
target:
required: true
description: "make target to build"
type: string
github:
required: true
description: "top workflow's `github`"
type: string
jobs:
build:
runs-on: ubuntu-22.04
env:
# GitHub image registry used for storing $(CONTAINER_ENGINE)'s cache
CACHE: "ghcr.io/${{ github.repository }}/workbench-images/build-cache"
# Targets (and their folder) that should be scanned using FS instead of IMAGE scan due to resource constraints
TRIVY_SCAN_FS_JSON: '{}'
steps:
- uses: actions/checkout@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Free up additional disk space
# https://docs.github.com/en/actions/learn-github-actions/expressions
if: "${{ contains(inputs.target, 'rocm') || contains(inputs.target, 'cuda') || contains(inputs.target, 'intel') ||
contains(inputs.target, 'pytorch') || contains(inputs.target, 'tensorflow') }}"
run: |
set -x
df -h
sudo apt-get update
sudo apt-get remove -y '^dotnet-.*'
sudo apt-get remove -y '^llvm-.*'
sudo apt-get remove -y 'php.*'
sudo apt-get remove -y '^mongodb-.*'
sudo apt-get autoremove -y
sudo apt-get clean
sudo rm -rf /usr/local/.ghcup &
sudo rm -rf /usr/local/lib/android &
sudo rm -rf /usr/local/share/boost &
sudo rm -rf /usr/local/lib/node_modules &
sudo rm -rf /usr/share/dotnet &
sudo rm -rf /opt/ghc &
sudo rm -rf /opt/hostedtoolcache/CodeQL &
sudo docker image prune --all --force &
wait
df -h
- name: Mount lvm overlay for podman builds
run: |
df -h
free -h
bash ./ci/cached-builds/gha_lvm_overlay.sh
df -h
free -h
# https://github.com/containers/buildah/issues/2521#issuecomment-884779112
- name: Workaround https://github.com/containers/podman/issues/22152#issuecomment-2027705598
run: sudo apt-get -qq remove podman crun
- uses: actions/cache@v4
id: cached-linuxbrew
with:
path: /home/linuxbrew/.linuxbrew
key: linuxbrew
- name: Install podman
if: steps.cached-linuxbrew.outputs.cache-hit != 'true'
run: |
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
/home/linuxbrew/.linuxbrew/bin/brew install podman
- name: Add linuxbrew to PATH
run: echo "/home/linuxbrew/.linuxbrew/bin/" >> $GITHUB_PATH
- name: Configure Podman
run: |
set -x
mkdir -p $HOME/.config/containers/
cp ci/cached-builds/containers.conf $HOME/.config/containers/containers.conf
cp ci/cached-builds/storage.conf $HOME/.config/containers/storage.conf
# should at least reset storage when touching storage.conf
podman system reset --force
mkdir -p $HOME/.local/share/containers/storage/tmp
# start systemd user service
# since `brew services start podman` is buggy, let's do our own brew-compatible service
mkdir -p "${HOME}/.config/systemd/user/"
cp ci/cached-builds/homebrew.podman.service "${HOME}/.config/systemd/user/homebrew.podman.service"
systemctl --user daemon-reload
systemctl --user start homebrew.podman.service
echo "PODMAN_SOCK=/run/user/${UID}/podman/podman.sock" >> $GITHUB_ENV
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push
- name: "push|schedule: make ${{ inputs.target }}"
run: |
SANITIZED_REF_NAME=$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g')
export IMAGE_TAG="${SANITIZED_REF_NAME}_${{ github.sha }}"
make ${{ inputs.target }}
if: ${{ fromJson(inputs.github).event_name == 'push' || fromJson(inputs.github).event_name == 'schedule' }}
env:
IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images"
CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }} --cache-to ${{ env.CACHE }}"
# dependent images were already built and pushed, so just let podman pull it
BUILD_DEPENDENT_IMAGES: "no"
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
- name: "pull_request: make ${{ inputs.target }}"
run: |
make ${{ inputs.target }}
if: "${{ fromJson(inputs.github).event_name == 'pull_request' }}"
env:
IMAGE_TAG: "${{ github.sha }}"
IMAGE_REGISTRY: "localhost:5000/workbench-images"
CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }}"
# We don't have access to image registry, so disable pushing
PUSH_IMAGES: "no"
- name: "Show podman images information"
run: podman images --digests
- name: "pull_request|schedule: resolve target if Trivy scan should run"
id: resolve-target
if: ${{ fromJson(inputs.github).event_name == 'pull_request' || fromJson(inputs.github).event_name == 'schedule' }}
env:
EVENT_NAME: ${{ fromJson(inputs.github).event_name }}
HAS_TRIVY_LABEL: ${{ contains(fromJson(inputs.github).event.pull_request.labels.*.name, 'trivy-scan') }}
FS_SCAN_FOLDER: ${{ fromJson(env.TRIVY_SCAN_FS_JSON)[inputs.target] }}
run: |
if [[ "$EVENT_NAME" == "pull_request" && "$HAS_TRIVY_LABEL" == "true" ]]; then
if [[ -n "$FS_SCAN_FOLDER" ]]; then
TARGET="$FS_SCAN_FOLDER"
TYPE="fs"
else
TARGET="localhost:5000/workbench-images:${{ inputs.target }}-${{ github.sha }}"
TYPE="image"
fi
elif [[ "$EVENT_NAME" == "schedule" ]]; then
if [[ -n "$FS_SCAN_FOLDER" ]]; then
TARGET="$FS_SCAN_FOLDER"
TYPE="fs"
else
TARGET="ghcr.io/${{ github.repository }}/workbench-images:${{ inputs.target }}-${{ github.ref_name }}_${{ github.sha }}"
TYPE="image"
fi
fi
if [[ -n "$TARGET" ]]; then
echo "target=$TARGET" >> $GITHUB_OUTPUT
echo "type=$TYPE" >> $GITHUB_OUTPUT
echo "Trivy scan will run on $TARGET ($TYPE)"
else
echo "Trivy scan won't run"
fi
- name: Run Trivy vulnerability scanner
if: ${{ steps.resolve-target.outputs.target }}
run: |
TRIVY_VERSION=0.53.0
REPORT_FOLDER=${{ github.workspace }}/report
REPORT_FILE=trivy-report.md
REPORT_TEMPLATE=trivy-markdown.tpl
mkdir -p $REPORT_FOLDER
cp ci/$REPORT_TEMPLATE $REPORT_FOLDER
SCAN_TARGET=${{ steps.resolve-target.outputs.target }}
SCAN_TYPE=${{ steps.resolve-target.outputs.type }}
echo "Scanning $SCAN_TARGET ($SCAN_TYPE)"
if [[ "$SCAN_TYPE" == "image" ]]; then
SCAN_ARGS="--image-src podman --podman-host /var/run/podman/podman.sock"
PODMAN_ARGS="-v ${PODMAN_SOCK}:/var/run/podman/podman.sock"
elif [[ "$SCAN_TYPE" == "fs" ]]; then
WORKSPACE_FOLDER="/workspace"
SCAN_TARGET="$WORKSPACE_FOLDER/$SCAN_TARGET"
PODMAN_ARGS="-v ${{ github.workspace }}:$WORKSPACE_FOLDER"
fi
# have trivy access podman socket,
# https://github.com/aquasecurity/trivy/issues/580#issuecomment-666423279
podman run --rm \
$PODMAN_ARGS \
-v ${REPORT_FOLDER}:/report \
docker.io/aquasec/trivy:$TRIVY_VERSION \
$SCAN_TYPE \
$SCAN_ARGS \
--scanners vuln --ignore-unfixed \
--exit-code 0 --timeout 30m \
--format template --template "@/report/$REPORT_TEMPLATE" -o /report/$REPORT_FILE \
$SCAN_TARGET
cat $REPORT_FOLDER/$REPORT_FILE >> $GITHUB_STEP_SUMMARY
- run: df -h
if: "${{ !cancelled() }}"