Skip to content

Commit 5907152

Browse files
sudo87Locharla, Sandeep
sudo87
authored and
Locharla, Sandeep
committed
Add cloud image downloader script (apache#11918)
1 parent 618b6d0 commit 5907152

File tree

1 file changed

+259
-0
lines changed

1 file changed

+259
-0
lines changed

‎tools/utils/cloud-image-downloader.sh‎

Lines changed: 259 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
#!/usr/bin/env bash
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
#-------------------------------------------------------------------------------
20+
# Configuration
21+
#-------------------------------------------------------------------------------
22+
# This section contains the variables you might want to change.
23+
24+
# The temporary directory where files will be downloaded.
25+
# It's a good practice to create a unique temporary directory for each script run.
26+
TEMP_DIR=$(mktemp -d)
27+
28+
# The BASE destination directory for the downloaded image files.
29+
# Subdirectories for each distro will be created inside this one.
30+
# Make sure this directory exists before running the script.
31+
# Must be executed by the cloudstack user on machine hosting the public download site.
32+
# It will be publicly available at https://download.cloudstack.org/templates/cloud-images/
33+
DEST_DIR="${HOME}/repository/templates/cloud-images"
34+
35+
# The directory where log files will be stored.
36+
# Make sure this directory exists.
37+
LOG_DIR="${HOME}/log/cloud-image-downloader"
38+
LOG_FILE="${LOG_DIR}/cloud-image-downloader_$(date +%Y%m%d_%H%M%S).log"
39+
LOG_RETENTION_DAYS=30
40+
41+
LOGGER_TAG="cloud-image-downloader"
42+
LOGGER_FACILITY="user"
43+
LOGGER_AVAILABLE=false
44+
45+
log_message() {
46+
local priority=$1
47+
shift
48+
local message="$*"
49+
local timestamp=$(date +'%Y-%m-%d %H:%M:%S')
50+
51+
# Log to file
52+
echo "${timestamp} [${priority}] ${message}" | tee -a "${LOG_FILE}"
53+
54+
# Log to syslog using logger utility
55+
if [ "${LOGGER_AVAILABLE}" = true ]; then
56+
logger -t "${LOGGER_TAG}" -p "${LOGGER_FACILITY}.${priority}" -- "${message}"
57+
fi
58+
}
59+
60+
log_info() {
61+
log_message "info" "$@"
62+
}
63+
64+
log_warn() {
65+
log_message "warning" "$@"
66+
}
67+
68+
log_error() {
69+
log_message "err" "$@"
70+
}
71+
72+
cleanup_old_logs() {
73+
log_info "Cleaning up log files older than ${LOG_RETENTION_DAYS} days..."
74+
75+
if [ ! -d "$LOG_DIR" ]; then
76+
log_warn "Log directory does not exist: $LOG_DIR"
77+
return
78+
fi
79+
80+
local deleted_count=0
81+
82+
# Find and delete log files older than retention period
83+
while IFS= read -r -d '' log_file; do
84+
rm -f "$log_file"
85+
deleted_count=$((deleted_count + 1))
86+
done < <(find "$LOG_DIR" -name "*.log" -type f -mtime +${LOG_RETENTION_DAYS} -print0 2>/dev/null)
87+
88+
if [ $deleted_count -gt 0 ]; then
89+
log_info "Deleted $deleted_count old log file(s)"
90+
else
91+
log_info "No old log files to delete"
92+
fi
93+
}
94+
95+
#-------------------------------------------------------------------------------
96+
# Image Definitions
97+
#-------------------------------------------------------------------------------
98+
# To add a new image, you must add an entry to BOTH arrays below.
99+
100+
# 1. Add the destination filename and the download URL.
101+
declare -A IMAGE_URLS=(
102+
["Rocky-9-GenericCloud.latest.x86_64.qcow2"]="https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2"
103+
["Rocky-9-GenericCloud.latest.aarch64.qcow2"]="https://dl.rockylinux.org/pub/rocky/9/images/aarch64/Rocky-9-GenericCloud.latest.aarch64.qcow2"
104+
["Rocky-8-GenericCloud.latest.x86_64.qcow2"]="https://dl.rockylinux.org/pub/rocky/8/images/x86_64/Rocky-8-GenericCloud.latest.x86_64.qcow2"
105+
["Rocky-8-GenericCloud.latest.aarch64.qcow2"]="https://dl.rockylinux.org/pub/rocky/8/images/aarch64/Rocky-8-GenericCloud.latest.aarch64.qcow2"
106+
["openSUSE-Leap-15.5-Minimal-VM.x86_64-Cloud.qcow2"]="https://download.opensuse.org/distribution/leap/15.5/appliances/openSUSE-Leap-15.5-Minimal-VM.x86_64-Cloud.qcow2"
107+
["openSUSE-Leap-15.5-Minimal-VM.aarch64-Cloud.qcow2"]="https://download.opensuse.org/distribution/leap/15.5/appliances/openSUSE-Leap-15.5-Minimal-VM.aarch64-Cloud.qcow2"
108+
["debian-12-genericcloud-amd64.qcow2"]="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2"
109+
["debian-12-genericcloud-arm64.qcow2"]="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-arm64.qcow2"
110+
["ubuntu-24.04-server-cloudimg-amd64.img"]="https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img"
111+
["ubuntu-24.04-server-cloudimg-arm64.img"]="https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img"
112+
["ubuntu-22.04-server-cloudimg-amd64.img"]="https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img"
113+
["ubuntu-22.04-server-cloudimg-arm64.img"]="https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-arm64.img"
114+
["ubuntu-20.04-server-cloudimg-amd64.img"]="https://cloud-images.ubuntu.com/releases/20.04/release/ubuntu-20.04-server-cloudimg-amd64.img"
115+
["ubuntu-20.04-server-cloudimg-arm64.img"]="https://cloud-images.ubuntu.com/releases/20.04/release/ubuntu-20.04-server-cloudimg-arm64.img"
116+
["OL9U5_x86_64-kvm-b259.qcow2"]="https://yum.oracle.com/templates/OracleLinux/OL9/u5/x86_64/OL9U5_x86_64-kvm-b259.qcow2"
117+
["OL9U5_aarch64-kvm-b126.qcow2"]="https://yum.oracle.com/templates/OracleLinux/OL9/u5/aarch64/OL9U5_aarch64-kvm-b126.qcow2"
118+
["OL8U10_x86_64-kvm-b258.qcow2"]="https://yum.oracle.com/templates/OracleLinux/OL8/u10/x86_64/OL8U10_x86_64-kvm-b258.qcow2"
119+
["OL8U10_aarch64-kvm-b122.qcow2"]="https://yum.oracle.com/templates/OracleLinux/OL8/u10/aarch64/OL8U10_aarch64-kvm-b122.qcow2"
120+
)
121+
122+
# 2. Add the destination filename and its corresponding distribution subdirectory name.
123+
declare -A IMAGE_DISTROS=(
124+
["Rocky-9-GenericCloud.latest.x86_64.qcow2"]="rockylinux"
125+
["Rocky-9-GenericCloud.latest.aarch64.qcow2"]="rockylinux"
126+
["Rocky-8-GenericCloud.latest.x86_64.qcow2"]="rockylinux"
127+
["Rocky-8-GenericCloud.latest.aarch64.qcow2"]="rockylinux"
128+
["openSUSE-Leap-15.5-Minimal-VM.x86_64-Cloud.qcow2"]="opensuse"
129+
["openSUSE-Leap-15.5-Minimal-VM.aarch64-Cloud.qcow2"]="opensuse"
130+
["debian-12-genericcloud-amd64.qcow2"]="debian"
131+
["debian-12-genericcloud-arm64.qcow2"]="debian"
132+
["ubuntu-24.04-server-cloudimg-amd64.img"]="ubuntu"
133+
["ubuntu-24.04-server-cloudimg-arm64.img"]="ubuntu"
134+
["ubuntu-22.04-server-cloudimg-amd64.img"]="ubuntu"
135+
["ubuntu-22.04-server-cloudimg-arm64.img"]="ubuntu"
136+
["ubuntu-20.04-server-cloudimg-amd64.img"]="ubuntu"
137+
["ubuntu-20.04-server-cloudimg-arm64.img"]="ubuntu"
138+
["OL9U5_x86_64-kvm-b259.qcow2"]="oraclelinux"
139+
["OL9U5_aarch64-kvm-b126.qcow2"]="oraclelinux"
140+
["OL8U10_x86_64-kvm-b258.qcow2"]="oraclelinux"
141+
["OL8U10_aarch64-kvm-b122.qcow2"]="oraclelinux"
142+
)
143+
144+
#-------------------------------------------------------------------------------
145+
# Cleanup Handler
146+
#-------------------------------------------------------------------------------
147+
148+
cleanup_on_exit() {
149+
local exit_code=$?
150+
if [ -d "$TEMP_DIR" ]; then
151+
rm -rf "$TEMP_DIR"
152+
log_info "Temporary directory $TEMP_DIR removed."
153+
fi
154+
155+
if [ $exit_code -ne 0 ]; then
156+
log_error "Script exited with error code: $exit_code"
157+
fi
158+
}
159+
160+
trap cleanup_on_exit EXIT INT TERM
161+
162+
#-------------------------------------------------------------------------------
163+
# Main Script Logic
164+
#-------------------------------------------------------------------------------
165+
166+
if command -v logger &> /dev/null; then
167+
LOGGER_AVAILABLE=true
168+
fi
169+
170+
# Ensure base destination and log directories exist
171+
mkdir -p "$DEST_DIR"
172+
mkdir -p "$LOG_DIR"
173+
174+
# Clean up old logs first
175+
cleanup_old_logs
176+
177+
log_info "Starting image download process."
178+
log_info "Temporary directory: $TEMP_DIR"
179+
log_info "Base destination directory: $DEST_DIR"
180+
log_info "Log file: $LOG_FILE"
181+
182+
# Inform about logger status
183+
if [ "${LOGGER_AVAILABLE}" = true ]; then
184+
log_info "Syslog logging enabled (tag: ${LOGGER_TAG})"
185+
else
186+
log_warn "Syslog logging disabled - logger utility not found"
187+
fi
188+
189+
# Loop through the image URLs
190+
for filename in "${!IMAGE_URLS[@]}"; do
191+
url="${IMAGE_URLS[$filename]}"
192+
distro="${IMAGE_DISTROS[$filename]}"
193+
194+
# Check if a distro is defined for the file
195+
if [ -z "$distro" ]; then
196+
log_error "No distribution directory defined for $filename. Skipping."
197+
continue
198+
fi
199+
200+
distro_dest_dir="${DEST_DIR}/${distro}"
201+
temp_filepath="${TEMP_DIR}/${filename}"
202+
dest_filepath="${distro_dest_dir}/${filename}"
203+
204+
log_info "--------------------------------------------------"
205+
log_info "Starting download for: $filename"
206+
log_info "URL: $url"
207+
208+
# Download the file to the temporary directory
209+
wget --progress=bar:force:noscroll -O "$temp_filepath" "$url"
210+
download_status=$?
211+
212+
if [ $download_status -ne 0 ]; then
213+
# Handle download failure
214+
log_error "Failed to download $filename from $url. wget exit code: $download_status"
215+
else
216+
# Handle download success
217+
log_info "Successfully downloaded $filename to temporary location."
218+
219+
# Ensure the specific distro directory exists
220+
log_info "Ensuring destination directory exists: $distro_dest_dir"
221+
mkdir -p "$distro_dest_dir"
222+
223+
# Move the file to the destination directory, replacing any existing file
224+
log_info "Moving $filename to $dest_filepath"
225+
mv -f "$temp_filepath" "$dest_filepath"
226+
move_status=$?
227+
228+
if [ $move_status -ne 0 ]; then
229+
log_error "Failed to move $filename to $dest_filepath. mv exit code: $move_status"
230+
else
231+
log_info "Successfully moved $filename."
232+
fi
233+
fi
234+
done
235+
236+
log_info "Generate checksum"
237+
# Create md5 checksum
238+
checksum_file="md5sum.txt"
239+
sha512_checksum_file="sha512sum.txt"
240+
241+
cd "$DEST_DIR"
242+
find . -type f ! -iname '*.txt' -exec md5sum {} \; > "$checksum_file"
243+
checksum_status=$?
244+
if [ $checksum_status -ne 0 ]; then
245+
log_error "Failed to create md5 checksum. md5sum exit code: $checksum_status"
246+
else
247+
log_info "Successfully created checksum file: $checksum_file"
248+
fi
249+
250+
find . -type f ! -iname '*.txt' -exec sha512sum {} \; > "$sha512_checksum_file"
251+
sha512_checksum_status=$?
252+
if [ $sha512_checksum_status -ne 0 ]; then
253+
log_error "Failed to create sha512 checksum. sha512sum exit code: $sha512_checksum_status"
254+
else
255+
log_info "Successfully created checksum file: $sha512_checksum_file"
256+
fi
257+
258+
log_info "--------------------------------------------------"
259+
log_info "Image download process finished."

0 commit comments

Comments
 (0)