-
-
Notifications
You must be signed in to change notification settings - Fork 2.5k
400 lines (353 loc) · 15.5 KB
/
linux_build.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
name: Linux Builds
on:
push:
paths-ignore:
- 'doc/**'
- 'docker/**'
branches-ignore:
- 'backport**'
- 'dependabot**'
pull_request:
paths-ignore:
- 'doc/**'
- 'docker/**'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
linux-build:
# Store the components of the container name as environment variables:
# ${CONTAINER_REGISTRY}/${CONTAINER_REGISTRY_USER}/${CONTAINER_NAME}:${CONTAINER_TAG}
#
# Additionally, CACHE_CONTAINER_TAG may be used as as source for the
# Docker build cache. So if the Dockerfile in a feature branch is
# unchanged relative to master, a full container rebuild should not
# be required.
env:
CONTAINER_REGISTRY: ${{ vars.gdal_container_registry || 'ghcr.io' }}
CONTAINER_REGISTRY_USER: ${{ vars.gdal_container_registry_user || github.repository_owner }}
CONTAINER_NAME: gdal-deps
CONTAINER_TAG: ${{ matrix.container }}-${{ github.base_ref || github.ref_name }}
CACHE_CONTAINER_TAG: ${{ matrix.container }}-master
permissions:
packages: write
strategy:
fail-fast: false
matrix:
# Matrix variables:
#
# * name : readable summary of configuration, used for display
# * id : used as a ccache key, and to create a build subdirectory
# * container : build environment container and path to build script
# * use_avx2 : if true, determine arch at runtime and use in ccache key
# * build_script : name of custom build script, if any. Will be executed
# inside container, from build subdirectory.
# * before_test_script : name of script to run before tests, if any.
# Will be executed outside container, from
# workspace root. Can be used to start docker
# containers as services for testing.
# * test_script : name of custom test script, if any. Will be executed
# inside container, from build subdirectory.
# * travis_branch : value of TRAVIS_BRANCH environment variable,
# used for test skipping
include:
- name: Alpine, gcc
id: alpine
container: alpine
build_script: build.sh
os: ubuntu-22.04
- name: Alpine, numpy 2
id: alpine_numpy2
container: alpine_numpy2
build_script: build.sh
os: ubuntu-22.04
- name: Alpine, clang 32-bit
id: alpine_32bit
container: alpine_32bit
build_script: build.sh
test_script: test.sh
travis_branch: alpine_32bit
os: ubuntu-22.04
- name: Fedora Rawhide, clang++
id: fedora_rawhide
travis_branch: fedora_rawhide
container: fedora_rawhide
build_script: build.sh
os: ubuntu-22.04
- name: Ubuntu 24.04, gcc
id: ubuntu_24.04
travis_branch: ubuntu_2404
container: ubuntu_24.04
before_test_script: services.sh
build_script: build.sh
test_script: test.sh
os: ubuntu-22.04
- name: Ubuntu 22.04, gcc
id: ubuntu_22.04
travis_branch: ubuntu_2204
container: ubuntu_22.04
before_test_script: services.sh
build_script: build.sh
test_script: test.sh
os: ubuntu-22.04
- name: Ubuntu 22.04, clang ASAN
id: asan
travis_branch: sanitize
container: ubuntu_22.04
build_script: build.sh
test_script: test.sh
os: ubuntu-22.04
- name: Ubuntu 20.04, gcc
id: ubuntu_20.04
travis_branch: ubuntu_2004
container: ubuntu_20.04
use_avx2: true
build_script: build.sh
test_script: test.sh
os: ubuntu-22.04
- name: Ubuntu 20.04, coverage
id: coverage
travis_branch: ubuntu_2004
container: ubuntu_20.04
before_test_script: services.sh
build_script: build.sh
test_script: test.sh
os: ubuntu-22.04
- name: Ubuntu 20.04, benchmarks
id: benchmarks
travis_branch: ubuntu_2004
container: ubuntu_20.04
build_script: build.sh
test_script: test.sh
os: ubuntu-22.04
- name: Ubuntu 20.04, Intel compiler
id: icc
container: icc
build_script: build.sh
os: ubuntu-22.04
name: ${{ matrix.name }}
runs-on: ${{ matrix.os }}
defaults:
run:
# bash is needed to use ${CONTAINER_REGISTRY_USER,,}, which forces the
# username to lower-case as required by docker.
shell: bash
steps:
- name: Set variables
# This logic needs to be kept in sync between linux_build.yml and converity_scan.yml
run: |
CONTAINER_TAG_CLEAN=$(echo ${CONTAINER_TAG} | tr -d -c "[:alnum:].-")
echo "CONTAINER_TAG_CLEAN=${CONTAINER_TAG_CLEAN}"
echo "CONTAINER_TAG_CLEAN=${CONTAINER_TAG_CLEAN}" >> ${GITHUB_ENV}
CACHE_CONTAINER_TAG_CLEAN=$(echo ${CACHE_CONTAINER_TAG} | tr -d -c "[:alnum:].-")
echo "CACHE_CONTAINER_TAG_CLEAN=${CACHE_CONTAINER_TAG_CLEAN}"
echo "CACHE_CONTAINER_TAG_CLEAN=${CACHE_CONTAINER_TAG_CLEAN}" >> ${GITHUB_ENV}
echo "CONTAINER_NAME_FULL=${CONTAINER_REGISTRY}/${CONTAINER_REGISTRY_USER,,}/${CONTAINER_NAME}:${CONTAINER_TAG_CLEAN}" >>${GITHUB_ENV}
# Work around segfaults in ASan/MSan jobs
# Cf https://github.com/libjpeg-turbo/libjpeg-turbo/commit/2dfe6c0fe9e18671105e94f7cbf044d4a1d157e6
# and https://github.com/actions/runner-images/issues/9491
- name: Set up build
run: |
sudo sysctl vm.mmap_rnd_bits=28
- name: Checkout
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
- name: Login to Docker Hub
if: env.CONTAINER_REGISTRY == 'docker.io'
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GHCR
if: env.CONTAINER_REGISTRY == 'ghcr.io'
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
# Pull build environment in forks or pull requests, unless [skip cache] is included in the commit message
- name: Pull build environment
if: "(github.repository_owner != 'OSGeo' || github.event_name == 'pull_request') && !contains(github.event.head_commit.message, '[skip cache]')"
run: |
docker pull ${CONTAINER_REGISTRY}/osgeo/${CONTAINER_NAME}:${CONTAINER_TAG_CLEAN} || true
docker pull ${CONTAINER_REGISTRY}/osgeo/${CONTAINER_NAME}:${CACHE_CONTAINER_TAG_CLEAN} || true
docker pull ${CONTAINER_NAME_FULL} || true
echo "DOCKER_BUILD_CACHE_FROM=--cache-from ${CONTAINER_REGISTRY}/osgeo/${CONTAINER_NAME}:${CONTAINER_TAG_CLEAN} --cache-from ${CONTAINER_REGISTRY}/osgeo/${CONTAINER_NAME}:${CACHE_CONTAINER_TAG_CLEAN} --cache-from ${CONTAINER_NAME_FULL}" >>${GITHUB_ENV}
- name: Prepare build context
run: |
mkdir docker-build-context
cp autotest/requirements.txt docker-build-context
- name: Update build environment
env:
DOCKER_BUILDKIT: 1
run: |
# FIXME: for some reason, the fedora rawhide container pushed by
# master job is corrupted (looks like it contains an outdated layer
# symlinking libssl.so.3 to an older version of the actual file),
# once it is pushed. But in the job that generates it,
# compilation & tests work fine. It looks like some weird caching
# issue
if test "${{ matrix.container }}" = "fedora_rawhide"; then
DOCKER_BUILD_CACHE_FROM=""
else
BUILD_ARG_INLINE_CACHE="--build-arg BUILDKIT_INLINE_CACHE=1"
fi
docker build \
${BUILD_ARG_INLINE_CACHE} \
${DOCKER_BUILD_CACHE_FROM} \
-t ${CONTAINER_NAME_FULL} \
-f .github/workflows/${{ matrix.container }}/Dockerfile.ci \
docker-build-context
# Get the architecture so we can use it as part of the cache key,
# but only if we are going to use avx2 in the build. If we are not,
# including the arch will cause unnecessary cache misses.
- name: Get Architecture
id: get-arch
if: matrix.use_avx2
run: |
export ARCH=$(cc -march=native -### -E - < /dev/null 2>&1 | sed -ne 's/.*cc1 .*-march=\([^ "]*\)[ "].*/\1/p')
echo "Architecture: $ARCH"
echo "arch=$ARCH" >> $GITHUB_OUTPUT
# cache the .ccache directory
# key it on the runner os, build type, deps, and arch
# It's especially important to include arch in the key because we
# may get runtime errors with -mavx2 from objects built on a
# different architecture.
- name: Restore build cache
id: restore-cache
uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
with:
path: ${{ github.workspace }}/.ccache
key: ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }}-${{ github.run_id }}
restore-keys: |
${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }}
${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}
- name: Prepare ccache
run: |
mkdir -p ${{ github.workspace }}/.ccache
chmod -R a+rw ${{ github.workspace }}/.ccache
docker run --rm \
-v ${{ github.workspace }}/.ccache:/.ccache \
-u $(id -u ${USER}):$(id -g ${USER}) \
${CONTAINER_NAME_FULL} \
sh -c "ccache -M 1G && ccache -sp && ccache -z"
# FIXME the default BUILD_CMD here isn't working...we get an error
# about the quotes not matching.
- name: Build
run: |
if test -f ".github/workflows/${{ matrix.id }}/${{ matrix.build_script }}"; then
BUILD_CMD="$(pwd)/.github/workflows/${{ matrix.id }}/${{ matrix.build_script }}"
else
BUILD_CMD="sh -c 'cmake .. && make -j$(nproc)'"
fi
mkdir -p build-${{ matrix.id }}
docker run --name gdal-build \
--rm \
-e "GDAL_SOURCE_DIR=$(pwd)" \
-u $(id -u ${USER}):$(id -g ${USER}) \
-v $(pwd):$(pwd):rw \
-v ${{ github.workspace }}/.ccache:/.ccache:rw \
--workdir $(pwd)/build-${{ matrix.id }} \
${CONTAINER_NAME_FULL} \
${BUILD_CMD}
- name: Summarize ccache
run: |
docker run --rm \
-v ${{ github.workspace }}/.ccache:/.ccache \
-u $(id -u ${USER}):$(id -g ${USER}) \
${CONTAINER_NAME_FULL} \
ccache -s
- name: Save build cache
uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
with:
path: ${{ github.workspace }}/.ccache
key: ${{ steps.restore-cache.outputs.cache-primary-key }}
- name: Start test services
if: matrix.before_test_script
run: |
.github/workflows/${{ matrix.id }}/${{ matrix.before_test_script }}
# --security-opt seccomp=unconfined, so that the userfaulfd syscall is availabledocker run \
- name: Run tests
env:
TRAVIS: yes
TRAVIS_BRANCH: ${{ matrix.travis_branch }}
BUILD_NAME: ${{ matrix.travis_branch }}
run: |
if test -f ".github/workflows/${{ matrix.id }}/${{ matrix.test_script }}"; then
TEST_CMD="$(pwd)/.github/workflows/${{ matrix.id }}/${{ matrix.test_script }}"
else
TEST_CMD="ctest -V -j $(nproc)"
fi
if test "${{ matrix.id }}" = "benchmarks"; then
if test -f /sys/devices/system/cpu/intel_pstate/no_turbo; then
echo "Disable TurboBoost"
echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo
fi
fi
# For cache
mkdir .gdal
docker run \
-e CI \
-e GITHUB_WORKFLOW \
-e TRAVIS \
-e TRAVIS_BRANCH \
-e BUILD_NAME \
-e "GDAL_SOURCE_DIR=$(pwd)" \
-u $(id -u ${USER}):$(id -g ${USER}) \
--security-opt seccomp=unconfined \
--add-host=host.docker.internal:host-gateway \
--rm \
-v $(pwd)/.gdal:/.gdal \
-v $(pwd):$(pwd) \
--workdir $(pwd)/build-${{ matrix.id }} \
${CONTAINER_NAME_FULL} \
${TEST_CMD}
- name: Coveralls
uses: coverallsapp/github-action@643bc377ffa44ace6394b2b5d0d3950076de9f63 # v2.3.0
if: ${{ matrix.id == 'coverage' }}
with:
format: lcov
file: build-coverage/gdal_filtered.info
- name: Push build environment
if: github.event_name == 'push'
continue-on-error: true
env:
DOCKER_BUILDKIT: 1
run: |
docker push ${CONTAINER_NAME_FULL}
- name: Upload coverage artifacts
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
if: ${{ matrix.id == 'coverage' }}
with:
name: coverage_index.html
path: build-${{ matrix.id }}/coverage_html/index.html
- name: Upload coverage artifacts
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
if: ${{ matrix.id == 'coverage' }}
with:
name: HTML
path: build-${{ matrix.id }}/coverage_html/*
- name: Deploy ssh key (for coverage)
if: ${{ matrix.id == 'coverage' && github.event_name == 'push' && github.ref_name == 'master' && github.repository == 'OSGeo/gdal' }}
shell: bash -l {0}
run: |
mkdir $HOME/.ssh && echo "${{ secrets.GDAL_TEST_COVERAGE_RESULTS_SSH_KEY }}" > $HOME/.ssh/id_rsa
chmod 700 $HOME/.ssh && chmod 600 $HOME/.ssh/id_rsa
ssh-keyscan -t rsa github.com >> $HOME/.ssh/known_hosts
eval `ssh-agent -s`
ssh-add $HOME/.ssh/id_rsa
- name: Deploy to https://github.com/OSGeo/gdal-test-coverage-results
if: ${{ matrix.id == 'coverage' && github.event_name == 'push' && github.ref_name == 'master' && github.repository == 'OSGeo/gdal' }}
shell: bash -l {0}
run: |
set -x
set -e
mkdir -p output_html/coverage_html
cp -r $GITHUB_WORKSPACE/build-${{ matrix.id }}/coverage_html/* output_html/coverage_html/
cd output_html
git init
git config user.email "[email protected]"
git config user.name "GDAL test coverage results bot"
git remote add origin [email protected]:gdalautotest-coverage-results/gdalautotest-coverage-results.github.io
git remote -v
echo "Results of coverage of GDAL autotest See https://gdalautotest-coverage-results.github.io/coverage_html/index.html" > README.md
git add -A
git commit -m "Update with OSGeo/gdal commit $GITHUB_SHA"
git push -f origin master