Skip to content

Commit 61ef7d4

Browse files
committed
build_toolchains: Break dep loop
1 parent fc92672 commit 61ef7d4

File tree

4 files changed

+191
-84
lines changed

4 files changed

+191
-84
lines changed

build_library/break_dep_loop.sh

+137
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
# Goo to attempt to resolve dependency loops on individual packages.
2+
# If this becomes insufficient we will need to move to a full multi-stage
3+
# bootstrap process like we do with the SDK via catalyst.
4+
#
5+
# Called like:
6+
#
7+
# break_dep_loop [-v] [PKG_USE_PAIR]…
8+
#
9+
# Pass -v for verbose output.
10+
#
11+
# PKG_USE_PAIR consists of two arguments: a package name (for example:
12+
# sys-fs/lvm2), and a comma-separated list of USE flags to clear (for
13+
# example: udev,systemd).
14+
#
15+
# Env vars:
16+
#
17+
# BDL_ROOT, BDL_PORTAGEQ, BDL_EQUERY, BDL_EMERGE, BDL_INFO
18+
break_dep_loop() {
19+
local bdl_root=${BDL_ROOT:-/}
20+
local bdl_portageq=${BDL_PORTAGEQ:-portageq}
21+
local bdl_equery=${BDL_EQUERY:-equery}
22+
local bdl_emerge=${BDL_EMERGE:-emerge}
23+
local bdl_info=${BDL_INFO:-echo}
24+
local conf_dir="${bdl_root%/}/etc/portage"
25+
local flag_file="${conf_dir}/package.use/break_dep_loop"
26+
local force_flag_file="${conf_dir}/profile/package.use.force/break_dep_loop"
27+
28+
local verbose=
29+
if [[ ${1:-} = '-v' ]]; then
30+
verbose=x
31+
shift
32+
fi
33+
34+
# Be sure to clean up use flag hackery from previous failed runs
35+
sudo rm -f "${flag_file}" "${force_flag_file}"
36+
37+
if [[ ${#} -eq 0 ]]; then
38+
return 0
39+
fi
40+
41+
function bdl_call() {
42+
local output_var_name=${1}; shift
43+
if [[ ${output_var_name} = '-' ]]; then
44+
local throw_away
45+
output_var_name=throw_away
46+
fi
47+
local -n output_ref=${output_var_name}
48+
if [[ -n ${verbose} ]]; then
49+
"${bdl_info}" "${*@Q}"
50+
fi
51+
local -i rv=0
52+
output_ref=$("${@}") || rv=${?}
53+
if [[ -n ${verbose} ]]; then
54+
"${bdl_info}" "output: ${output_ref}"
55+
"${bdl_info}" "exit status: ${rv}"
56+
fi
57+
return ${rv}
58+
}
59+
60+
# Temporarily compile/install packages with flags disabled. If a binary
61+
# package is available use it regardless of its version or use flags.
62+
local pkg use_flags disabled_flags
63+
local -a flags
64+
local -a pkgs args flag_file_entries pkg_summaries
65+
local -A per_pkg_flags=()
66+
while [[ $# -gt 1 ]]; do
67+
pkg=${1}
68+
use_flags=${2}
69+
shift 2
70+
71+
mapfile -t flags <<<"${use_flags//,/$'\n'}"
72+
disabled_flags="${flags[*]/#/-}"
73+
74+
pkgs+=( "${pkg}" )
75+
per_pkg_flags["${pkg}"]=${use_flags}
76+
flag_file_entries+=( "${pkg} ${disabled_flags}" )
77+
args+=( "--buildpkg-exclude=${pkg}" )
78+
pkg_summaries+=( "${pkg}[${disabled_flags}]" )
79+
done
80+
unset pkg use_flags disabled_flags flags
81+
82+
# If packages are already installed we have nothing to do
83+
local pkg any_package_uninstalled=
84+
for pkg in "${pkgs[@]}"; do
85+
if ! bdl_call - "${bdl_portageq}" has_version "${bdl_root}" "${pkg}"; then
86+
any_package_uninstalled=x
87+
break
88+
fi
89+
done
90+
if [[ -z ${any_package_uninstalled} ]]; then
91+
if [[ -n ${verbose} ]]; then
92+
"${bdl_info}" "all packages (${pkgs[*]}) are installed already, skipping"
93+
fi
94+
return 0
95+
fi
96+
unset pkg any_package_uninstalled
97+
98+
# Likewise, nothing to do if the flags aren't actually enabled.
99+
local pkg any_flag_enabled= equery_output flag flags_str
100+
local -a flags grep_args
101+
for pkg in "${pkgs[@]}"; do
102+
bdl_call equery_output "${bdl_equery}" -q uses "${pkg}"
103+
flags_str=${per_pkg_flags["${pkg}"]}
104+
mapfile -t flags <<<"${flags_str//,/$'\n'}"
105+
for flag in "${flags[@]}"; do
106+
grep_args+=( -e "${flag/#/+}" )
107+
done
108+
if bdl_call - grep --quiet --line-regexp --fixed-strings "${grep_args[@]}" <<<"${equery_output}"; then
109+
any_flag_enabled=x
110+
break
111+
fi
112+
done
113+
if [[ -z ${any_flag_enabled} ]]; then
114+
if [[ -n ${verbose} ]]; then
115+
"${bdl_info}" "all packages (${pkgs[*]}) has all the desired USE flags already disabled, skipping"
116+
fi
117+
return 0
118+
fi
119+
unset pkg any_flag_enabled equery_output flag flags_str flags grep_args
120+
121+
"${bdl_info}" "Merging ${pkg_summaries[*]}"
122+
sudo mkdir -p "${flag_file%/*}" "${force_flag_file%/*}"
123+
printf '%s\n' "${flag_file_entries[@]}" | sudo tee "${flag_file}" >/dev/null
124+
cp -a "${flag_file}" "${force_flag_file}"
125+
if [[ -n ${verbose} ]]; then
126+
"${bdl_info}" "contents of ${flag_file@Q}:"
127+
"${bdl_info}" "$(<"${flag_file}")"
128+
"${bdl_info}" "${bdl_emerge}" --rebuild-if-unbuilt=n "${args[@]}" "${pkgs[@]}"
129+
fi
130+
# rebuild-if-unbuilt is disabled to prevent portage from needlessly
131+
# rebuilding zlib for some unknown reason, in turn triggering more rebuilds.
132+
"${bdl_emerge}" \
133+
--rebuild-if-unbuilt=n \
134+
"${args[@]}" "${pkgs[@]}"
135+
sudo rm -f "${flag_file}" "${force_flag_file}"
136+
unset bdl_call
137+
}

build_library/catalyst_toolchains.sh

+40
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
set -e
44
source /tmp/chroot-functions.sh
55
source /tmp/toolchain_util.sh
6+
source /tmp/break_dep_loop.sh
67

78
# A note on packages:
89
# The default PKGDIR is /usr/portage/packages
@@ -32,6 +33,45 @@ build_target_toolchain() {
3233
cp -at "${ROOT}" "${SYSROOT}"/lib*
3334
cp -at "${ROOT}"/usr "${SYSROOT}"/usr/include "${SYSROOT}"/usr/lib*
3435

36+
local -a args_for_bdl=()
37+
if [[ -n ${clst_VERBOSE} ]]; then
38+
args_for_bdl+=(-v)
39+
fi
40+
function btt_bdl_portageq() {
41+
ROOT=${ROOT} SYSROOT=${ROOT} PORTAGE_CONFIGROOT=${ROOT} portageq "${@}"
42+
}
43+
function btt_bdl_equery() {
44+
ROOT=${ROOT} SYSROOT=${ROOT} PORTAGE_CONFIGROOT=${ROOT} equery "${@}"
45+
}
46+
function btt_bdl_emerge() {
47+
PORTAGE_CONFIGROOT="$ROOT" run_merge --root="$ROOT" --sysroot="$ROOT" "${@}"
48+
}
49+
# Breaking the following loops here:
50+
#
51+
# cryptsetup[udev] -> libudev[systemd] -> systemd[cryptsetup] -> cryptsetup
52+
# lvm2[udev] -> libudev[systemd] -> systemd[cryptsetup] -> cryptsetup -> lvm2
53+
# systemd requires udev, so needs to be disabled too
54+
# lvm2[lvm] -> systemd[cryptsetup] -> cryptsetup -> lvm2
55+
# thin requires lvm, so needs to be disabled too
56+
# systemd[cryptsetup] -> cryptsetup -> tmpfiles[systemd] -> systemd
57+
# util-linux[audit] -> audit[python] -> python -> util-linux
58+
# util-linux[cryptsetup] -> cryptsetup -> util-linux
59+
# util-linux[selinux] -> libselinux[python] -> python -> util-linux
60+
# util-linux[systemd] -> systemd -> util-linux
61+
# util-linux[udev] -> libudev[systemd] -> systemd -> util-linux
62+
args_for_bdl+=(
63+
sys-apps/systemd cryptsetup
64+
sys-apps/util-linux audit,cryptsetup,selinux,systemd,udev
65+
#sys-fs/cryptsetup udev
66+
#sys-fs/lvm2 lvm,systemd,thin,udev
67+
)
68+
BDL_ROOT=${ROOT} \
69+
BDL_PORTAGEQ=btt_bdl_portageq \
70+
BDL_EQUERY=btt_bdl_equery \
71+
BDL_EMERGE=btt_bdl_emerge \
72+
break_dep_loop "${args_for_bdl[@]}"
73+
unset btt_bdl_portageq btt_bdl_equery btt_bdl_emerge
74+
3575
# --root is required because run_merge overrides ROOT=
3676
PORTAGE_CONFIGROOT="$ROOT" \
3777
run_merge -u --root="$ROOT" --sysroot="$ROOT" "${TOOLCHAIN_PKGS[@]}"

build_packages

+11-82
Original file line numberDiff line numberDiff line change
@@ -173,89 +173,12 @@ if [[ ${#WORKON_PKGS[@]} -gt 0 ]]; then
173173
)
174174
fi
175175

176-
# Goo to attempt to resolve dependency loops on individual packages.
177-
# If this becomes insufficient we will need to move to a full multi-stage
178-
# bootstrap process like we do with the SDK via catalyst.
179-
#
180-
# Called like:
181-
#
182-
# break_dep_loop [PKG_USE_PAIR]…
183-
#
184-
# PKG_USE_PAIR consists of two arguments: a package name (for example:
185-
# sys-fs/lvm2), and a comma-separated list of USE flags to clear (for
186-
# example: udev,systemd).
187-
break_dep_loop() {
188-
local -a pkgs
189-
local -a all_flags
190-
local -a args
191-
local flag_file="${BOARD_ROOT}/etc/portage/package.use/break_dep_loop"
192-
local -a flag_file_entries
193-
local -a pkg_summaries
194-
195-
# Be sure to clean up use flag hackery from previous failed runs
196-
sudo rm -f "${flag_file}"
197-
198-
if [[ $# -eq 0 ]]; then
199-
return 0
200-
fi
201-
202-
# Temporarily compile/install packages with flags disabled. If a binary
203-
# package is available use it regardless of its version or use flags.
204-
local pkg
205-
local -a flags
206-
local disabled_flags
207-
while [[ $# -gt 0 ]]; do
208-
pkg="${1}"
209-
pkgs+=("${pkg}")
210-
flags=( ${2//,/ } )
211-
all_flags+=("${flags[@]}")
212-
disabled_flags="${flags[@]/#/-}"
213-
flag_file_entries+=("${pkg} ${disabled_flags}")
214-
args+=("--buildpkg-exclude=${pkg}")
215-
pkg_summaries+=("${pkg}[${disabled_flags}]")
216-
shift 2
217-
done
218-
219-
# If packages are already installed we have nothing to do
220-
local any_package_uninstalled=0
221-
for pkg in "${pkgs[@]}"; do
222-
if ! portageq-"${BOARD}" has_version "${BOARD_ROOT}" "${pkgs[@]}"; then
223-
any_package_uninstalled=1
224-
break
225-
fi
226-
done
227-
if [[ ${any_package_uninstalled} -eq 0 ]]; then
228-
return 0
229-
fi
230-
231-
# Likewise, nothing to do if the flags aren't actually enabled.
232-
local any_flag_enabled=0
233-
for pkg in "${pkgs[@]}"; do
234-
if pkg_use_enabled "${pkg}" "${all_flags[@]}"; then
235-
any_flag_enabled=1
236-
break
237-
fi
238-
done
239-
if [[ ${any_flag_enabled} -eq 0 ]]; then
240-
return 0
241-
fi
242-
243-
info "Merging ${pkg_summaries[@]}"
244-
sudo mkdir -p "${flag_file%/*}"
245-
sudo_clobber "${flag_file}" <<<"${flag_file_entries[0]}"
246-
local entry
247-
for entry in "${flag_file_entries[@]:1}"; do
248-
sudo_append "${flag_file}" <<<"${entry}"
249-
done
250-
# rebuild-if-unbuilt is disabled to prevent portage from needlessly
251-
# rebuilding zlib for some unknown reason, in turn triggering more rebuilds.
252-
sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" \
253-
--rebuild-if-unbuilt=n \
254-
"${args[@]}" "${pkgs[@]}"
255-
sudo rm -f "${flag_file}"
256-
}
257-
258176
if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
177+
. "${BUILD_LIBRARY_DIR}/break_dep_loop.sh" || exit 1
178+
179+
function bp_bdl_emerge() {
180+
sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "${@}"
181+
}
259182
# Breaking the following loops here:
260183
#
261184
# util-linux[udev] -> virtual/udev -> systemd -> util-linux
@@ -266,12 +189,18 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
266189
# lvm2[systemd] -> systemd[cryptsetup] -> cryptsetup -> lvm2
267190
# systemd[cryptsetup] -> cryptsetup[udev] -> virtual/udev -> systemd
268191
# curl[http2] -> nghttp2[systemd] -> systemd[curl] -> curl
192+
BDL_ROOT=${BOARD_ROOT} \
193+
BDL_PORTAGEQ=portageq-"${BOARD}" \
194+
BDL_EQUERY=equery-"${BOARD}" \
195+
BDL_EMERGE=bp_bdl_emerge \
196+
BDL_INFO=info \
269197
break_dep_loop sys-apps/util-linux udev,systemd,cryptsetup \
270198
sys-fs/cryptsetup udev \
271199
sys-fs/lvm2 udev,systemd \
272200
sys-apps/systemd cryptsetup,tpm \
273201
net-misc/curl http2 \
274202
net-libs/nghttp2 systemd
203+
unset bp_bdl_emerge
275204
fi
276205

277206
if [[ "${FLAGS_only_resolve_circular_deps}" -eq "${FLAGS_TRUE}" ]]; then

build_toolchains

+3-2
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,10 @@ catalyst_init "$@"
4848

4949
ROOT_OVERLAY="${TEMPDIR}/stage4-${ARCH}-$FLAGS_version-overlay"
5050

51-
# toolchain_util.sh is required by catalyst_toolchains.sh
51+
# toolchain_util.sh and break_dep_loop.sh are required by
52+
# catalyst_toolchains.sh
5253
mkdir -p "${ROOT_OVERLAY}/tmp"
53-
cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
54+
cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${BUILD_LIBRARY_DIR}/break_dep_loop.sh" "${ROOT_OVERLAY}/tmp"
5455
create_provenance_overlay "${ROOT_OVERLAY}"
5556

5657
catalyst_build

0 commit comments

Comments
 (0)