Skip to content

Commit dda15ba

Browse files
jonkeaneclaude
andcommitted
GH-55478: [R] Trim the vendored Boost to the libraries Thrift needs
The generic trim leaves a 29 MB Boost tarball. tools/trim/boost.sh keeps only the Boost libraries that build_boost() in ThirdpartyToolchain.cmake includes (minus ones the R build doesn't use, e.g. multiprecision) and their dependencies, as resolved by Boost's own CMake, and drops their tests, docs and examples. This brings the vendored Boost down to ~5 MB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent bdd7fbf commit dda15ba

2 files changed

Lines changed: 206 additions & 0 deletions

File tree

‎r/.Rbuildignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,4 @@ STYLE.md
3333
^bootstrap.R$
3434
air.toml
3535
^tools/vendored-downloads$
36+
^tools/trim$

‎r/tools/trim/boost.sh‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
# Trim the Boost CMake source tarball (boost-X.Y.Z-cmake.tar.gz, ~140 MB)
21+
# down to the Boost libraries that libarrow's bundled Boost build
22+
# (build_boost() in cpp/cmake_modules/ThirdpartyToolchain.cmake) uses when
23+
# the R package builds libarrow, so it is small enough to ship in the R
24+
# package (~5 MB).
25+
#
26+
# Usage: tools/trim/boost.sh IN_TARBALL OUT_TARBALL
27+
#
28+
# Called by tools/vendor_dependencies.sh (`make sync-cpp`) after tools/cpp
29+
# has been synced. Needs cmake and a C++ compiler, because Boost's own CMake
30+
# is used to work out which libraries are needed.
31+
#
32+
# How it works:
33+
#
34+
# 1. Read the libraries build_boost() adds to BOOST_INCLUDE_LIBRARIES from
35+
# ThirdpartyToolchain.cmake, minus the ones that the R build never needs
36+
# (EXCLUDED_LIBRARIES below).
37+
# 2. Configure Boost with that BOOST_INCLUDE_LIBRARIES. Boost's CMake adds
38+
# the transitive dependencies and logs each library it adds.
39+
# 3. Keep only the top-level files, tools/cmake and those libraries, without
40+
# their tests, docs, examples, etc.
41+
#
42+
# Set BOOST_INCLUDE_LIBRARIES (semicolon-separated) in the environment to
43+
# skip step 1 and use that list instead.
44+
45+
set -euo pipefail
46+
47+
if [ "$#" -ne 2 ]; then
48+
echo "Usage: $0 IN_TARBALL OUT_TARBALL" >&2
49+
exit 1
50+
fi
51+
52+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
53+
IN="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
54+
OUT="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")"
55+
56+
# Libraries that build_boost() can add to BOOST_INCLUDE_LIBRARIES but that
57+
# the R package's libarrow build never needs. If you add a library here,
58+
# say why.
59+
#
60+
# - multiprecision: only needed when ARROW_BOOST_NEED_MULTIPRECISION is
61+
# true, i.e. with ARROW_TESTING, ARROW_GANDIVA, ARROW_USE_UBSAN or a
62+
# compiler without a native 128-bit integer (ARROW_USE_NATIVE_INT128).
63+
# The R package builds none of those, and gcc/clang on the 64-bit
64+
# platforms that nixlibs.R builds for all have __int128 (32-bit targets
65+
# don't). Including it would more than double the size of the trimmed
66+
# tarball (~5 MB, 54 libraries -> ~11 MB, 58 libraries). If a build that
67+
# needs it uses this tarball, configuring libarrow fails in build_boost()
68+
# because the boost_multiprecision target doesn't exist.
69+
# - beast, variant, xpressive: only for ARROW_FLIGHT_SQL_ODBC.
70+
EXCLUDED_LIBRARIES=(multiprecision beast variant xpressive)
71+
72+
# Print the library names that build_boost() adds to BOOST_INCLUDE_LIBRARIES
73+
# with `set(BOOST_INCLUDE_LIBRARIES ...)` or
74+
# `list(APPEND BOOST_INCLUDE_LIBRARIES ...)`, one per line. Variable
75+
# references such as ${ARROW_BOOST_COMPONENTS} are skipped: those are only
76+
# non-empty when building tests, integration tests, benchmarks or ODBC.
77+
boost_libraries_from_cmake() {
78+
awk '
79+
/^function\(build_boost\)/ { inside = 1; next }
80+
inside && /^endfunction\(\)/ { exit }
81+
inside {
82+
sub(/#.*/, "")
83+
gsub(/\(/, " ( ")
84+
gsub(/\)/, " ) ")
85+
for (i = 1; i <= NF; i++) tok[++n] = $i
86+
}
87+
END {
88+
for (i = 1; i < n; i++) {
89+
if (tok[i + 1] != "(") continue
90+
cmd = tolower(tok[i])
91+
j = i + 2
92+
if (cmd == "set" && tok[j] == "BOOST_INCLUDE_LIBRARIES") {
93+
j += 1
94+
} else if (cmd == "list" && tok[j] == "APPEND" &&
95+
tok[j + 1] == "BOOST_INCLUDE_LIBRARIES") {
96+
j += 2
97+
} else {
98+
continue
99+
}
100+
for (; j <= n && tok[j] != ")"; j++) {
101+
if (tok[j] !~ /^\$/) print tok[j]
102+
}
103+
}
104+
}' "$1"
105+
}
106+
107+
if [ -z "${BOOST_INCLUDE_LIBRARIES:-}" ]; then
108+
# tools/cpp is the synced copy of cpp/ (`make sync-cpp`); fall back to the
109+
# repository's cpp/ so this can also be run by hand from a git checkout.
110+
TOOLCHAIN=""
111+
for dir in "${SCRIPT_DIR}/../cpp" "${SCRIPT_DIR}/../../../cpp"; do
112+
if [ -f "${dir}/cmake_modules/ThirdpartyToolchain.cmake" ]; then
113+
TOOLCHAIN="${dir}/cmake_modules/ThirdpartyToolchain.cmake"
114+
break
115+
fi
116+
done
117+
if [ -z "${TOOLCHAIN}" ]; then
118+
echo "Can't find ThirdpartyToolchain.cmake; run \`make sync-cpp\` first" >&2
119+
exit 1
120+
fi
121+
122+
ALL_LIBRARIES="$(boost_libraries_from_cmake "${TOOLCHAIN}")"
123+
if [ -z "${ALL_LIBRARIES}" ]; then
124+
echo "No BOOST_INCLUDE_LIBRARIES found in build_boost() in ${TOOLCHAIN}." >&2
125+
echo "Has build_boost() changed? Update ${BASH_SOURCE[0]}." >&2
126+
exit 1
127+
fi
128+
for lib in "${EXCLUDED_LIBRARIES[@]}"; do
129+
if ! printf '%s\n' "${ALL_LIBRARIES}" | grep -qxF "${lib}"; then
130+
echo "Warning: '${lib}' is in EXCLUDED_LIBRARIES but build_boost() no" \
131+
"longer uses it; remove it from ${BASH_SOURCE[0]}" >&2
132+
fi
133+
done
134+
BOOST_INCLUDE_LIBRARIES="$(printf '%s\n' "${ALL_LIBRARIES}" |
135+
grep -vxF "$(printf '%s\n' "${EXCLUDED_LIBRARIES[@]}")" |
136+
paste -s -d ';' -)"
137+
fi
138+
echo "Trimming Boost to BOOST_INCLUDE_LIBRARIES=${BOOST_INCLUDE_LIBRARIES} and their dependencies"
139+
140+
WORK="$(mktemp -d "${TMPDIR:-/tmp}/arrow-trim-boost.XXXXXX")"
141+
trap 'rm -rf "${WORK}"' EXIT
142+
143+
mkdir "${WORK}/src" "${WORK}/trimmed"
144+
tar -xzf "${IN}" -C "${WORK}/src"
145+
# The tarball has a single top-level directory, e.g. boost-1.92.0
146+
SRC_NAME="$(ls "${WORK}/src")"
147+
SRC="${WORK}/src/${SRC_NAME}"
148+
if [ ! -f "${SRC}/tools/cmake/include/BoostRoot.cmake" ]; then
149+
echo "${IN} doesn't look like a Boost CMake source tarball" >&2
150+
exit 1
151+
fi
152+
153+
# Let Boost's CMake resolve the transitive dependencies. With
154+
# --log-level=VERBOSE, BoostRoot.cmake logs "Adding Boost library 'x'" for
155+
# each requested library and "Adding Boost dependency 'x'" for each
156+
# dependency (including 'headers', which is always added).
157+
# BUILD_TESTING=OFF matches prepare_fetchcontent(), so test-only
158+
# dependencies are not pulled in.
159+
cmake -S "${SRC}" -B "${WORK}/build" \
160+
--log-level=VERBOSE \
161+
-DBUILD_TESTING=OFF \
162+
-DBOOST_INCLUDE_LIBRARIES="${BOOST_INCLUDE_LIBRARIES}" \
163+
> "${WORK}/cmake.log" 2>&1 || {
164+
cat "${WORK}/cmake.log" >&2
165+
echo "Configuring Boost failed" >&2
166+
exit 1
167+
}
168+
if grep -q "has not been found" "${WORK}/cmake.log"; then
169+
grep "has not been found" "${WORK}/cmake.log" >&2
170+
echo "A library in BOOST_INCLUDE_LIBRARIES doesn't exist in ${SRC_NAME}" >&2
171+
exit 1
172+
fi
173+
# (bash 3.2 on macOS has no mapfile)
174+
KEEP_LIBRARIES=()
175+
while IFS= read -r lib; do
176+
KEEP_LIBRARIES+=("${lib}")
177+
done < <(sed -n "s/.*Adding Boost [a-z]* '\([^']*\)'.*/\1/p" "${WORK}/cmake.log" | LC_ALL=C sort -u)
178+
if [ "${#KEEP_LIBRARIES[@]}" -eq 0 ]; then
179+
echo "Found no \"Adding Boost ...\" messages in Boost's CMake output." >&2
180+
echo "Has BoostRoot.cmake changed? Update ${BASH_SOURCE[0]}." >&2
181+
exit 1
182+
fi
183+
echo "Keeping ${#KEEP_LIBRARIES[@]} Boost libraries: ${KEEP_LIBRARIES[*]}"
184+
185+
# Copy what the bundled build needs into ${WORK}/trimmed/${SRC_NAME}:
186+
# the top-level files (CMakeLists.txt, LICENSE_1_0.txt, ...), tools/cmake
187+
# (BoostRoot.cmake etc.) and the libraries, but not the top-level doc/,
188+
# more/, status/ or the other tools/.
189+
DEST="${WORK}/trimmed/${SRC_NAME}"
190+
mkdir -p "${DEST}/tools"
191+
find "${SRC}" -maxdepth 1 -type f -exec cp -p {} "${DEST}/" \;
192+
cp -pR "${SRC}/tools/cmake" "${DEST}/tools/"
193+
for lib in "${KEEP_LIBRARIES[@]}"; do
194+
# lib can be a nested library such as numeric/conversion
195+
mkdir -p "$(dirname "${DEST}/libs/${lib}")"
196+
cp -pR "${SRC}/libs/${lib}" "${DEST}/libs/${lib}"
197+
# Only remove these at the top level of each library: deeper down they
198+
# can be header directories (e.g. include/boost/math/tools/).
199+
for dir in test doc example examples bench benchmark perf performance meta tools; do
200+
rm -rf "${DEST}/libs/${lib:?}/${dir}"
201+
done
202+
done
203+
204+
# COPYFILE_DISABLE stops macOS tar from adding AppleDouble (._*) files.
205+
(cd "${WORK}/trimmed" && COPYFILE_DISABLE=1 tar -czf "${OUT}" "${SRC_NAME}")

0 commit comments

Comments
 (0)