Skip to content

Commit f7bb8b0

Browse files
samrosesamrose
authored
feat: mirror fallbacks (#1903)
* feat: mirror fallbacks * fix: remove tier 3 * chore: no release cut for this change
1 parent 336f49e commit f7bb8b0

File tree

2 files changed

+176
-11
lines changed

2 files changed

+176
-11
lines changed

‎ebssurrogate/scripts/chroot-bootstrap-nix.sh‎

Lines changed: 81 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,94 @@ export APT_OPTIONS="-oAPT::Install-Recommends=false \
1414
-oAPT::Install-Suggests=false \
1515
-oAcquire::Languages=none"
1616

17-
if [ $(dpkg --print-architecture) = "amd64" ];
18-
then
17+
if [ $(dpkg --print-architecture) = "amd64" ];
18+
then
1919
ARCH="amd64";
2020
else
2121
ARCH="arm64";
2222
fi
2323

24+
# Mirror fallback function for resilient apt-get update
25+
function apt_update_with_fallback {
26+
local sources_file="/etc/apt/sources.list"
27+
local max_attempts=2
28+
local attempt=1
29+
30+
# Detect the current region from sources.list (it's already been substituted)
31+
# Extract the region from existing sources.list entries
32+
local current_region=$(grep -oP '(?<=http://)[^.]+(?=\.clouds\.ports\.ubuntu\.com)' "${sources_file}" | head -1 || echo "")
33+
34+
# Define mirror tiers (in priority order)
35+
local -a mirror_tiers=(
36+
"${current_region}.clouds.ports.ubuntu.com" # Tier 1: Regional CDN (as set in sources.list)
37+
"ports.ubuntu.com" # Tier 2: Global pool
38+
)
39+
40+
# If we couldn't detect current region, skip tier 1
41+
if [ -z "${current_region}" ]; then
42+
echo "Warning: Could not determine region from sources.list, skipping regional CDN"
43+
mirror_tiers=("${mirror_tiers[@]:1}") # Remove first element
44+
fi
45+
46+
for mirror in "${mirror_tiers[@]}"; do
47+
echo "========================================="
48+
echo "Attempting apt-get update with mirror: ${mirror}"
49+
echo "Attempt ${attempt} of ${max_attempts}"
50+
echo "========================================="
51+
52+
# Update sources.list to use current mirror
53+
sed -i "s|http://[^/]*/ubuntu-ports/|http://${mirror}/ubuntu-ports/|g" "${sources_file}"
54+
55+
# Show what we're using
56+
echo "Current sources.list configuration:"
57+
grep -E '^deb ' "${sources_file}" | head -3
58+
59+
# Attempt update with timeout (5 minutes)
60+
if timeout 300 apt-get $APT_OPTIONS update 2>&1; then
61+
echo "========================================="
62+
echo "✓ Successfully updated apt cache using mirror: ${mirror}"
63+
echo "========================================="
64+
return 0
65+
else
66+
local exit_code=$?
67+
echo "========================================="
68+
echo "✗ Failed to update using mirror: ${mirror}"
69+
echo "Exit code: ${exit_code}"
70+
echo "========================================="
71+
72+
# Clean partial downloads
73+
apt-get clean
74+
rm -rf /var/lib/apt/lists/*
75+
76+
# Exponential backoff before next attempt
77+
if [ ${attempt} -lt ${max_attempts} ]; then
78+
local sleep_time=$((attempt * 5))
79+
echo "Waiting ${sleep_time} seconds before trying next mirror..."
80+
sleep ${sleep_time}
81+
fi
82+
fi
83+
84+
attempt=$((attempt + 1))
85+
done
86+
87+
echo "========================================="
88+
echo "ERROR: All mirror tiers failed after ${max_attempts} attempts"
89+
echo "========================================="
90+
return 1
91+
}
92+
2493

2594

2695
function update_install_packages {
2796
source /etc/os-release
2897

29-
# Update APT with new sources
98+
# Update APT with new sources (using fallback mechanism)
3099
cat /etc/apt/sources.list
31-
apt-get $APT_OPTIONS update && apt-get $APT_OPTIONS --yes dist-upgrade
100+
if ! apt_update_with_fallback; then
101+
echo "FATAL: Failed to update package lists with any mirror tier"
102+
exit 1
103+
fi
104+
apt-get $APT_OPTIONS --yes dist-upgrade
32105

33106
# Do not configure grub during package install
34107
if [ "${ARCH}" = "amd64" ]; then
@@ -59,7 +132,10 @@ function update_install_packages {
59132

60133
# Install OpenSSH and other packages
61134
sudo add-apt-repository --yes universe
62-
apt-get update
135+
if ! apt_update_with_fallback; then
136+
echo "FATAL: Failed to update package lists after adding universe repository"
137+
exit 1
138+
fi
63139
apt-get install -y --no-install-recommends \
64140
openssh-server \
65141
git \

‎ebssurrogate/scripts/surrogate-bootstrap-nix.sh‎

Lines changed: 95 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,86 @@ set -o errexit
1010
set -o pipefail
1111
set -o xtrace
1212

13-
if [ $(dpkg --print-architecture) = "amd64" ];
14-
then
13+
if [ $(dpkg --print-architecture) = "amd64" ];
14+
then
1515
ARCH="amd64";
1616
else
17-
ARCH="arm64";
17+
ARCH="arm64";
1818
fi
1919

20+
# Mirror fallback function for resilient apt-get update
21+
function apt_update_with_fallback {
22+
local sources_file="/etc/apt/sources.list"
23+
local max_attempts=2
24+
local attempt=1
25+
26+
# Get EC2 region if not already set
27+
if [ -z "${REGION}" ]; then
28+
REGION=$(curl --silent --fail http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -E 's|[a-z]+$||g' || echo "")
29+
fi
30+
31+
# Define mirror tiers (in priority order)
32+
local -a mirror_tiers=(
33+
"${REGION}.clouds.ports.ubuntu.com" # Tier 1: Regional CDN
34+
"ports.ubuntu.com" # Tier 2: Global pool
35+
)
36+
37+
# If we couldn't get REGION, skip tier 1
38+
if [ -z "${REGION}" ]; then
39+
echo "Warning: Could not determine EC2 region, skipping regional CDN"
40+
mirror_tiers=("${mirror_tiers[@]:1}") # Remove first element
41+
fi
42+
43+
for mirror in "${mirror_tiers[@]}"; do
44+
echo "========================================="
45+
echo "Attempting apt-get update with mirror: ${mirror}"
46+
echo "Attempt ${attempt} of ${max_attempts}"
47+
echo "========================================="
48+
49+
# Update sources.list to use current mirror
50+
# Replace the region-specific mirror URL
51+
sed -i "s|http://[^/]*/ubuntu-ports/|http://${mirror}/ubuntu-ports/|g" "${sources_file}"
52+
# Also update any security sources
53+
sed -i "s|http://ports.ubuntu.com/ubuntu-ports|http://${mirror}/ubuntu-ports|g" "${sources_file}"
54+
55+
# Show what we're using
56+
echo "Current sources.list configuration:"
57+
grep -E '^deb ' "${sources_file}" | head -3
58+
59+
# Attempt update with timeout (5 minutes)
60+
if timeout 300 apt-get update 2>&1; then
61+
echo "========================================="
62+
echo "✓ Successfully updated apt cache using mirror: ${mirror}"
63+
echo "========================================="
64+
return 0
65+
else
66+
local exit_code=$?
67+
echo "========================================="
68+
echo "✗ Failed to update using mirror: ${mirror}"
69+
echo "Exit code: ${exit_code}"
70+
echo "========================================="
71+
72+
# Clean partial downloads
73+
apt-get clean
74+
rm -rf /var/lib/apt/lists/*
75+
76+
# Exponential backoff before next attempt
77+
if [ ${attempt} -lt ${max_attempts} ]; then
78+
local sleep_time=$((attempt * 5))
79+
echo "Waiting ${sleep_time} seconds before trying next mirror..."
80+
sleep ${sleep_time}
81+
fi
82+
fi
83+
84+
attempt=$((attempt + 1))
85+
done
86+
87+
echo "========================================="
88+
echo "ERROR: All mirror tiers failed after ${max_attempts} attempts"
89+
echo "========================================="
90+
return 1
91+
}
92+
2093
function waitfor_boot_finished {
2194
export DEBIAN_FRONTEND=noninteractive
2295

@@ -30,12 +103,28 @@ function waitfor_boot_finished {
30103

31104
function install_packages {
32105
# Setup Ansible on host VM
33-
apt-get update && sudo apt-get install software-properties-common -y
34-
add-apt-repository --yes --update ppa:ansible/ansible && sudo apt-get install ansible -y
106+
if ! apt_update_with_fallback; then
107+
echo "FATAL: Failed to update package lists on host VM"
108+
exit 1
109+
fi
110+
111+
sudo apt-get install software-properties-common -y
112+
add-apt-repository --yes --update ppa:ansible/ansible
113+
114+
if ! apt_update_with_fallback; then
115+
echo "FATAL: Failed to update package lists after adding Ansible PPA"
116+
exit 1
117+
fi
118+
119+
sudo apt-get install ansible -y
35120
ansible-galaxy collection install community.general
36121

37122
# Update apt and install required packages
38-
apt-get update
123+
if ! apt_update_with_fallback; then
124+
echo "FATAL: Failed to update package lists before installing tools"
125+
exit 1
126+
fi
127+
39128
apt-get install -y \
40129
gdisk \
41130
e2fsprogs \

0 commit comments

Comments
 (0)