Skip to content

Commit 4404c73

Browse files
c-dilksc-dilks
authored
ci: warm the maven cache to mitigate cloudflare blocks (#1537)
1 parent 49c8d85 commit 4404c73

1 file changed

Lines changed: 95 additions & 17 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 95 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ env:
2222
CCDB_CONNECTION: 'sqlite:///${{ github.workspace }}/coatjava/etc/data/sqlite/ccdb.sqlite'
2323
RCDB_CONNECTION: 'sqlite:///${{ github.workspace }}/coatjava/etc/data/sqlite/rcdb.sqlite'
2424
nthreads: 1
25+
# Maven 3.9+ reads MAVEN_ARGS automatically, including inside build-coatjava.sh:
26+
# --no-snapshot-updates : don't re-check remote repos (j4ml) for newer SNAPSHOTs;
27+
# use whatever `warm_maven_cache` resolved
28+
# retryHandler.count : built-in HTTP retries for transient 403/429/5xx
29+
MAVEN_ARGS: --no-snapshot-updates -Daether.connector.http.retryHandler.count=5
2530

2631
jobs:
2732

@@ -52,10 +57,58 @@ jobs:
5257
git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data
5358
cp validation-data/raw/rg-d/clas_018779.evio.00001 .
5459
60+
# Resolve all Maven dependencies ONCE (with retries and a single download
61+
# thread, to avoid tripping Cloudflare), save them to a cache, and let every
62+
# other job restore that cache instead of hitting the remote repos.
63+
#
64+
# The cache key is: m2-<ISO year-week>-<hash of all pom.xml files>
65+
# - a pom.xml change gives a new key, so dependencies are re-resolved
66+
# - the week component forces a weekly refresh, so SNAPSHOTs don't go stale
67+
# - to force a refresh sooner, delete the `m2-*` caches in the Actions UI
68+
warm_maven_cache:
69+
runs-on: ubuntu-latest
70+
outputs:
71+
key: ${{ steps.key.outputs.key }}
72+
steps:
73+
- uses: actions/checkout@v7
74+
- name: compute cache key
75+
id: key
76+
run: echo "key=m2-$(date +%G-%V)-${{ hashFiles('**/pom.xml') }}" | tee -a $GITHUB_OUTPUT
77+
- uses: actions/cache/restore@v6
78+
id: restore
79+
with:
80+
path: ~/.m2/repository
81+
key: ${{ steps.key.outputs.key }}
82+
restore-keys: |
83+
m2-
84+
- uses: actions/setup-java@v6.0.1
85+
with:
86+
java-version: ${{ env.JAVA_VERSION }}
87+
distribution: ${{ env.java_distribution }}
88+
- name: resolve dependencies (with retries)
89+
if: ${{ steps.restore.outputs.cache-hit != 'true' }}
90+
uses: nick-fields/retry@v4.0.0
91+
env:
92+
# override the global flags: here we DO want SNAPSHOTs refreshed (-U)
93+
MAVEN_ARGS: -U -Daether.connector.http.retryHandler.count=5
94+
with:
95+
timeout_minutes: 30
96+
max_attempts: 5
97+
retry_wait_seconds: 90
98+
command: mvn -B -ntp -Dmaven.artifact.threads=1 -DexcludeReactor=true dependency:go-offline
99+
- name: save maven cache
100+
# only on success, so a half-populated cache is never saved under the real key
101+
if: ${{ success() && steps.restore.outputs.cache-hit != 'true' }}
102+
uses: actions/cache/save@v6
103+
with:
104+
path: ~/.m2/repository
105+
key: ${{ steps.key.outputs.key }}
106+
55107
# build
56108
#############################################################################
57109

58110
build_macos:
111+
needs: [ warm_maven_cache ]
59112
strategy:
60113
fail-fast: true
61114
runs-on: macos-latest
@@ -65,7 +118,11 @@ jobs:
65118
with:
66119
java-version: ${{ env.JAVA_VERSION }}
67120
distribution: ${{ env.java_distribution }}
68-
cache: maven
121+
- uses: actions/cache/restore@v6
122+
with:
123+
path: ~/.m2/repository
124+
key: ${{ needs.warm_maven_cache.outputs.key }}
125+
fail-on-cache-miss: true
69126
- name: bump version to tag if tag trigger
70127
if: ${{ github.ref_type == 'tag' }}
71128
run: libexec/version-bump.sh ${{ github.ref_name }}
@@ -86,6 +143,7 @@ jobs:
86143
clara.tar.gz
87144
88145
build:
146+
needs: [ warm_maven_cache ]
89147
strategy:
90148
fail-fast: true
91149
runs-on: ubuntu-latest
@@ -95,7 +153,11 @@ jobs:
95153
with:
96154
java-version: ${{ env.JAVA_VERSION }}
97155
distribution: ${{ env.java_distribution }}
98-
cache: maven
156+
- uses: actions/cache/restore@v6
157+
with:
158+
path: ~/.m2/repository
159+
key: ${{ needs.warm_maven_cache.outputs.key }}
160+
fail-on-cache-miss: true
99161
- name: bump version to tag if tag trigger
100162
if: ${{ github.ref_type == 'tag' }}
101163
run: libexec/version-bump.sh ${{ github.ref_name }}
@@ -116,7 +178,14 @@ jobs:
116178
clara.tar.gz
117179
118180
dependency_info:
181+
# Informational job. It deliberately does a clean (uncached) resolve to see where
182+
# dependencies really come from, so it always hits the remote repos.
183+
# - run after `warm_maven_cache`, so it doesn't add to the parallel burst
184+
# - retry the build step
185+
needs: [ warm_maven_cache ]
119186
runs-on: ubuntu-latest
187+
env:
188+
MAVEN_ARGS: -Daether.connector.http.retryHandler.count=5 # no -nsu: want a true fresh resolve
120189
steps:
121190
- uses: actions/checkout@v7
122191
- name: Set up JDK
@@ -126,8 +195,12 @@ jobs:
126195
distribution: ${{ env.java_distribution }}
127196
# NOTE: do not use `cache: maven` here, so stale `_remote.repositories` data aren't read
128197
- name: build
129-
run: |
130-
./build-coatjava.sh --no-aux --no-progress --quiet -T${{ env.nthreads }}
198+
uses: nick-fields/retry@v4.0.0
199+
with:
200+
timeout_minutes: 30
201+
max_attempts: 3
202+
retry_wait_seconds: 120
203+
command: ./build-coatjava.sh --no-aux --no-progress --quiet -T${{ env.nthreads }}
131204
- name: collect info
132205
run: libexec/dependency-info.rb
133206
- name: print dependency trees
@@ -148,8 +221,7 @@ jobs:
148221
run: echo versions="[\"${{ env.JAVA_VERSION }}\",\"${{ env.JAVA_VERSION_LATEST }}\"]" | tee -a $GITHUB_OUTPUT
149222

150223
unit_tests:
151-
needs:
152-
- java_matrix
224+
needs: [ java_matrix, warm_maven_cache ]
153225
strategy:
154226
fail-fast: false
155227
matrix:
@@ -161,7 +233,11 @@ jobs:
161233
with:
162234
java-version: ${{ matrix.JAVA_VERSION }}
163235
distribution: ${{ env.java_distribution }}
164-
cache: maven
236+
- uses: actions/cache/restore@v6
237+
with:
238+
path: ~/.m2/repository
239+
key: ${{ needs.warm_maven_cache.outputs.key }}
240+
fail-on-cache-miss: true
165241
- name: unit tests
166242
run: ./build-coatjava.sh --lfs --unittests --no-progress -T${{ env.nthreads }}
167243
- name: collect jacoco report
@@ -176,7 +252,7 @@ jobs:
176252
retention-days: 1
177253

178254
spotbugs:
179-
needs: [ build ]
255+
needs: [ build, warm_maven_cache ]
180256
runs-on: ubuntu-latest
181257
steps:
182258
- uses: actions/checkout@v7
@@ -185,7 +261,11 @@ jobs:
185261
with:
186262
java-version: ${{ env.JAVA_VERSION }}
187263
distribution: ${{ env.java_distribution }}
188-
cache: maven
264+
- uses: actions/cache/restore@v6
265+
with:
266+
path: ~/.m2/repository
267+
key: ${{ needs.warm_maven_cache.outputs.key }}
268+
fail-on-cache-miss: true
189269
- uses: actions/download-artifact@v8
190270
with:
191271
name: build_ubuntu-latest
@@ -204,7 +284,6 @@ jobs:
204284
with:
205285
java-version: ${{ env.JAVA_VERSION }}
206286
distribution: ${{ env.java_distribution }}
207-
cache: maven
208287
- uses: actions/download-artifact@v8
209288
with:
210289
name: build_ubuntu-latest
@@ -233,7 +312,6 @@ jobs:
233312
with:
234313
java-version: ${{ env.JAVA_VERSION }}
235314
distribution: ${{ env.java_distribution }}
236-
cache: maven
237315
- uses: actions/download-artifact@v8
238316
with:
239317
name: build_ubuntu-latest
@@ -291,7 +369,6 @@ jobs:
291369
with:
292370
java-version: ${{ env.JAVA_VERSION }}
293371
distribution: ${{ env.java_distribution }}
294-
cache: maven
295372
- uses: actions/download-artifact@v8
296373
with:
297374
name: build_${{ matrix.runner }}
@@ -319,7 +396,6 @@ jobs:
319396
with:
320397
java-version: ${{ env.JAVA_VERSION }}
321398
distribution: ${{ env.java_distribution }}
322-
cache: maven
323399
- uses: actions/download-artifact@v8
324400
with:
325401
name: build_macos
@@ -344,7 +420,6 @@ jobs:
344420
with:
345421
java-version: ${{ env.JAVA_VERSION }}
346422
distribution: ${{ env.java_distribution }}
347-
cache: maven
348423
- name: setup groovy
349424
uses: wtfjoke/setup-groovy@v3
350425
with:
@@ -367,7 +442,6 @@ jobs:
367442
with:
368443
java-version: ${{ env.JAVA_VERSION }}
369444
distribution: ${{ env.java_distribution }}
370-
cache: maven
371445
- uses: actions/download-artifact@v8
372446
with:
373447
name: test_clara_result
@@ -386,7 +460,7 @@ jobs:
386460

387461
generate_documentation:
388462
runs-on: ubuntu-latest
389-
needs: [ unit_tests ]
463+
needs: [ unit_tests, warm_maven_cache ]
390464
steps:
391465
### zensical
392466
- uses: actions/checkout@v7
@@ -408,7 +482,11 @@ jobs:
408482
with:
409483
java-version: ${{ env.JAVA_VERSION }}
410484
distribution: ${{ env.java_distribution }}
411-
cache: maven
485+
- uses: actions/cache/restore@v6
486+
with:
487+
path: ~/.m2/repository
488+
key: ${{ needs.warm_maven_cache.outputs.key }}
489+
fail-on-cache-miss: true
412490
- name: build coatjava javadocs # javadoc:aggregate output dir cannot be controlled, so assume the latest "standard" path and `mv` it
413491
run: |
414492
libexec/build-javadocs.sh

0 commit comments

Comments
 (0)