Skip to content

Commit e8202e4

Browse files
rust-bors[bot]rust-bors[bot]
authored
Auto merge of #144257 - ChrisDenton:free-diskspace, r=<try>
WIP r? `@ghost` try-job: x86_64-msvc-1
2 parents 9748d87 + 9d60c13 commit e8202e4

File tree

6 files changed

+318
-264
lines changed

6 files changed

+318
-264
lines changed

‎.github/workflows/ci.yml

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,12 +117,18 @@ jobs:
117117
with:
118118
fetch-depth: 2
119119

120+
# Always print disk usage, even for jobs that don't free up disk space.
121+
- name: print disk usage
122+
run: |
123+
echo "disk usage:"
124+
df -h
125+
120126
# Free up disk space on Linux by removing preinstalled components that
121127
# we do not need. We do this to enable some of the less resource
122128
# intensive jobs to run on free runners, which however also have
123129
# less disk space.
124-
- name: free up disk space
125-
run: src/ci/scripts/free-disk-space.sh
130+
- name: free up disk space start
131+
run: src/ci/scripts/free-disk-space.sh start
126132
if: matrix.free_disk
127133

128134
# Rust Log Analyzer can't currently detect the PR number of a GitHub
@@ -218,6 +224,10 @@ jobs:
218224
run: |
219225
cd src/ci/citool
220226
CARGO_INCREMENTAL=0 CARGO_TARGET_DIR=../../../build/citool cargo build
227+
228+
- name: free up disk space report
229+
run: src/ci/scripts/free-disk-space.sh report
230+
if: matrix.free_disk
221231

222232
- name: run the build
223233
run: |

‎src/bootstrap/download-ci-llvm-stamp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
Change this file to make users of the `download-ci-llvm` configuration download
22
a new version of LLVM from CI, even if the LLVM submodule hasn’t changed.
33

4-
Last change is for: https://github.com/rust-lang/rust/pull/139931
4+
Last change is for: ????

‎src/ci/github-actions/jobs.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,15 @@ runners:
3131
<<: *base-job
3232

3333
- &job-windows
34-
os: windows-2022
34+
os: windows-2025
35+
free_disk: true
3536
<<: *base-job
3637

3738
# NOTE: windows-2025 has less disk space available than windows-2022,
3839
# because the D drive is missing.
3940
- &job-windows-25
4041
os: windows-2025
42+
free_disk: true
4143
<<: *base-job
4244

4345
- &job-windows-8c

‎src/ci/scripts/free-disk-space-linux.sh

Lines changed: 266 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,266 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
# Free disk space on Linux GitHub action runners
5+
# Script inspired by https://github.com/jlumbroso/free-disk-space
6+
7+
isX86() {
8+
local arch
9+
arch=$(uname -m)
10+
if [ "$arch" = "x86_64" ]; then
11+
return 0
12+
else
13+
return 1
14+
fi
15+
}
16+
17+
# Check if we're on a GitHub hosted runner.
18+
# In aws codebuild, the variable RUNNER_ENVIRONMENT is "self-hosted".
19+
isGitHubRunner() {
20+
# `:-` means "use the value of RUNNER_ENVIRONMENT if it exists, otherwise use an empty string".
21+
if [[ "${RUNNER_ENVIRONMENT:-}" == "github-hosted" ]]; then
22+
return 0
23+
else
24+
return 1
25+
fi
26+
}
27+
28+
# print a line of the specified character
29+
printSeparationLine() {
30+
for ((i = 0; i < 80; i++)); do
31+
printf "%s" "$1"
32+
done
33+
printf "\n"
34+
}
35+
36+
# compute available space
37+
# REF: https://unix.stackexchange.com/a/42049/60849
38+
# REF: https://stackoverflow.com/a/450821/408734
39+
getAvailableSpace() {
40+
df -a | awk 'NR > 1 {avail+=$4} END {print avail}'
41+
}
42+
43+
# make Kb human readable (assume the input is Kb)
44+
# REF: https://unix.stackexchange.com/a/44087/60849
45+
formatByteCount() {
46+
numfmt --to=iec-i --suffix=B --padding=7 "${1}000"
47+
}
48+
49+
# macro to output saved space
50+
printSavedSpace() {
51+
# Disk space before the operation
52+
local before=${1}
53+
local title=${2:-}
54+
55+
local after
56+
after=$(getAvailableSpace)
57+
local saved=$((after - before))
58+
59+
if [ "$saved" -lt 0 ]; then
60+
echo "::warning::Saved space is negative: $saved. Using '0' as saved space."
61+
saved=0
62+
fi
63+
64+
echo ""
65+
printSeparationLine "*"
66+
if [ -n "${title}" ]; then
67+
echo "=> ${title}: Saved $(formatByteCount "$saved")"
68+
else
69+
echo "=> Saved $(formatByteCount "$saved")"
70+
fi
71+
printSeparationLine "*"
72+
echo ""
73+
}
74+
75+
# macro to print output of df with caption
76+
printDF() {
77+
local caption=${1}
78+
79+
printSeparationLine "="
80+
echo "${caption}"
81+
echo ""
82+
echo "$ df -h"
83+
echo ""
84+
df -h
85+
printSeparationLine "="
86+
}
87+
88+
removeUnusedFilesAndDirs() {
89+
local to_remove=(
90+
"/usr/share/java"
91+
)
92+
93+
if isGitHubRunner; then
94+
to_remove+=(
95+
"/usr/local/aws-sam-cli"
96+
"/usr/local/doc/cmake"
97+
"/usr/local/julia"*
98+
"/usr/local/lib/android"
99+
"/usr/local/share/chromedriver-"*
100+
"/usr/local/share/chromium"
101+
"/usr/local/share/cmake-"*
102+
"/usr/local/share/edge_driver"
103+
"/usr/local/share/emacs"
104+
"/usr/local/share/gecko_driver"
105+
"/usr/local/share/icons"
106+
"/usr/local/share/powershell"
107+
"/usr/local/share/vcpkg"
108+
"/usr/local/share/vim"
109+
"/usr/share/apache-maven-"*
110+
"/usr/share/gradle-"*
111+
"/usr/share/kotlinc"
112+
"/usr/share/miniconda"
113+
"/usr/share/php"
114+
"/usr/share/ri"
115+
"/usr/share/swift"
116+
117+
# binaries
118+
"/usr/local/bin/azcopy"
119+
"/usr/local/bin/bicep"
120+
"/usr/local/bin/ccmake"
121+
"/usr/local/bin/cmake-"*
122+
"/usr/local/bin/cmake"
123+
"/usr/local/bin/cpack"
124+
"/usr/local/bin/ctest"
125+
"/usr/local/bin/helm"
126+
"/usr/local/bin/kind"
127+
"/usr/local/bin/kustomize"
128+
"/usr/local/bin/minikube"
129+
"/usr/local/bin/packer"
130+
"/usr/local/bin/phpunit"
131+
"/usr/local/bin/pulumi-"*
132+
"/usr/local/bin/pulumi"
133+
"/usr/local/bin/stack"
134+
135+
# Haskell runtime
136+
"/usr/local/.ghcup"
137+
138+
# Azure
139+
"/opt/az"
140+
"/usr/share/az_"*
141+
)
142+
143+
if [ -n "${AGENT_TOOLSDIRECTORY:-}" ]; then
144+
# Environment variable set by GitHub Actions
145+
to_remove+=(
146+
"${AGENT_TOOLSDIRECTORY}"
147+
)
148+
else
149+
echo "::warning::AGENT_TOOLSDIRECTORY is not set. Skipping removal."
150+
fi
151+
else
152+
# Remove folders and files present in AWS CodeBuild
153+
to_remove+=(
154+
# binaries
155+
"/usr/local/bin/ecs-cli"
156+
"/usr/local/bin/eksctl"
157+
"/usr/local/bin/kubectl"
158+
159+
"${HOME}/.gradle"
160+
"${HOME}/.dotnet"
161+
"${HOME}/.goenv"
162+
"${HOME}/.phpenv"
163+
164+
)
165+
fi
166+
167+
for element in "${to_remove[@]}"; do
168+
if [ ! -e "$element" ]; then
169+
# The file or directory doesn't exist.
170+
# Maybe it was removed in a newer version of the runner or it's not present in a
171+
# specific architecture (e.g. ARM).
172+
echo "::warning::Directory or file $element does not exist, skipping."
173+
fi
174+
done
175+
176+
# Remove all files and directories at once to save time.
177+
sudo rm -rf "${to_remove[@]}"
178+
}
179+
180+
execAndMeasureSpaceChange() {
181+
local operation=${1} # Function to execute
182+
local title=${2}
183+
184+
local before
185+
before=$(getAvailableSpace)
186+
$operation
187+
188+
printSavedSpace "$before" "$title"
189+
}
190+
191+
# Remove large packages
192+
# REF: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh
193+
cleanPackages() {
194+
local packages=(
195+
'^aspnetcore-.*'
196+
'^dotnet-.*'
197+
'^llvm-.*'
198+
'^mongodb-.*'
199+
'firefox'
200+
'libgl1-mesa-dri'
201+
'mono-devel'
202+
'php.*'
203+
)
204+
205+
if isGitHubRunner; then
206+
packages+=(
207+
azure-cli
208+
)
209+
210+
if isX86; then
211+
packages+=(
212+
'google-chrome-stable'
213+
'google-cloud-cli'
214+
'google-cloud-sdk'
215+
'powershell'
216+
)
217+
fi
218+
else
219+
packages+=(
220+
'google-chrome-stable'
221+
)
222+
fi
223+
224+
sudo apt-get -qq remove -y --fix-missing "${packages[@]}"
225+
226+
sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed"
227+
sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed"
228+
}
229+
230+
# Remove Docker images.
231+
# Ubuntu 22 runners have docker images already installed.
232+
# They aren't present in ubuntu 24 runners.
233+
cleanDocker() {
234+
echo "=> Removing the following docker images:"
235+
sudo docker image ls
236+
echo "=> Removing docker images..."
237+
sudo docker image prune --all --force || true
238+
}
239+
240+
# Remove Swap storage
241+
cleanSwap() {
242+
sudo swapoff -a || true
243+
sudo rm -rf /mnt/swapfile || true
244+
free -h
245+
}
246+
247+
# Display initial disk space stats
248+
249+
AVAILABLE_INITIAL=$(getAvailableSpace)
250+
251+
printDF "BEFORE CLEAN-UP:"
252+
echo ""
253+
254+
execAndMeasureSpaceChange cleanPackages "Unused packages"
255+
execAndMeasureSpaceChange cleanDocker "Docker images"
256+
execAndMeasureSpaceChange cleanSwap "Swap storage"
257+
execAndMeasureSpaceChange removeUnusedFilesAndDirs "Unused files and directories"
258+
259+
# Output saved space statistic
260+
echo ""
261+
printDF "AFTER CLEAN-UP:"
262+
263+
echo ""
264+
echo ""
265+
266+
printSavedSpace "$AVAILABLE_INITIAL" "Total saved"

‎src/ci/scripts/free-disk-space-windows.ps1

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Free disk space on Windows GitHub action runners.
2+
3+
$ErrorActionPreference = 'Stop'
4+
5+
# Exclude directory from Defender to make this faster
6+
Add-MpPreference -ExclusionPath C:\
7+
8+
$available = $(Get-Volume C).SizeRemaining
9+
10+
$dirs = 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\Llvm',
11+
'C:\rtools45', 'C:\ghcup', 'C:\Program Files (x86)\Android',
12+
'C:\Program Files\Google\Chrome', 'C:\Program Files (x86)\Microsoft\Edge',
13+
'C:\Program Files\Mozilla Firefox', 'C:\Program Files\MySQL', 'C:\Julia',
14+
'C:\Program Files\MongoDB', 'C:\Program Files\Azure Cosmos DB Emulator',
15+
'C:\Program Files\PostgreSQL', 'C:\Program Files\Unity Hub', 'C:\ProgramData\chocolatey',
16+
'C:\Strawberry', 'C:\hostedtoolcache\windows\Java_Temurin-Hotspot_jdk'
17+
18+
Remove-Item -Recurse -Force -ErrorAction Continue $dirs
19+
20+
$saved = ($available - $(Get-Volume C).SizeRemaining) / 1gb
21+
Write-Host "total space saved $saved GB"

0 commit comments

Comments
 (0)