|
| 1 | +#!/usr/bin/env bash |
| 2 | +# |
| 3 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 4 | +# or more contributor license agreements. See the NOTICE file |
| 5 | +# distributed with this work for additional information |
| 6 | +# regarding copyright ownership. The ASF licenses this file |
| 7 | +# to you under the Apache License, Version 2.0 (the |
| 8 | +# "License"); you may not use this file except in compliance |
| 9 | +# with the License. You may obtain a copy of the License at |
| 10 | +# |
| 11 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 12 | +# |
| 13 | +# Unless required by applicable law or agreed to in writing, |
| 14 | +# software distributed under the License is distributed on an |
| 15 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 16 | +# KIND, either express or implied. See the License for the |
| 17 | +# specific language governing permissions and limitations |
| 18 | +# under the License. |
| 19 | + |
| 20 | +# Trim the Boost CMake source tarball (boost-X.Y.Z-cmake.tar.gz, ~140 MB) |
| 21 | +# down to the Boost libraries that libarrow's bundled Boost build |
| 22 | +# (build_boost() in cpp/cmake_modules/ThirdpartyToolchain.cmake) uses when |
| 23 | +# the R package builds libarrow, so it is small enough to ship in the R |
| 24 | +# package (~5 MB). |
| 25 | +# |
| 26 | +# Usage: tools/trim/boost.sh IN_TARBALL OUT_TARBALL |
| 27 | +# |
| 28 | +# Called by tools/vendor_dependencies.sh (`make sync-cpp`) after tools/cpp |
| 29 | +# has been synced. Needs cmake and a C++ compiler, because Boost's own CMake |
| 30 | +# is used to work out which libraries are needed. |
| 31 | +# |
| 32 | +# How it works: |
| 33 | +# |
| 34 | +# 1. Read the libraries build_boost() adds to BOOST_INCLUDE_LIBRARIES from |
| 35 | +# ThirdpartyToolchain.cmake, minus the ones that the R build never needs |
| 36 | +# (EXCLUDED_LIBRARIES below). |
| 37 | +# 2. Configure Boost with that BOOST_INCLUDE_LIBRARIES. Boost's CMake adds |
| 38 | +# the transitive dependencies and logs each library it adds. |
| 39 | +# 3. Keep only the top-level files, tools/cmake and those libraries, without |
| 40 | +# their tests, docs, examples, etc. |
| 41 | +# |
| 42 | +# Set BOOST_INCLUDE_LIBRARIES (semicolon-separated) in the environment to |
| 43 | +# skip step 1 and use that list instead. |
| 44 | + |
| 45 | +set -euo pipefail |
| 46 | + |
| 47 | +if [ "$#" -ne 2 ]; then |
| 48 | + echo "Usage: $0 IN_TARBALL OUT_TARBALL" >&2 |
| 49 | + exit 1 |
| 50 | +fi |
| 51 | + |
| 52 | +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 53 | +IN="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" |
| 54 | +OUT="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")" |
| 55 | + |
| 56 | +# Libraries that build_boost() can add to BOOST_INCLUDE_LIBRARIES but that |
| 57 | +# the R package's libarrow build never needs. If you add a library here, |
| 58 | +# say why. |
| 59 | +# |
| 60 | +# - multiprecision: only needed when ARROW_BOOST_NEED_MULTIPRECISION is |
| 61 | +# true, i.e. with ARROW_TESTING, ARROW_GANDIVA, ARROW_USE_UBSAN or a |
| 62 | +# compiler without a native 128-bit integer (ARROW_USE_NATIVE_INT128). |
| 63 | +# The R package builds none of those, and gcc/clang on the 64-bit |
| 64 | +# platforms that nixlibs.R builds for all have __int128 (32-bit targets |
| 65 | +# don't). Including it would more than double the size of the trimmed |
| 66 | +# tarball (~5 MB, 54 libraries -> ~11 MB, 58 libraries). If a build that |
| 67 | +# needs it uses this tarball, configuring libarrow fails in build_boost() |
| 68 | +# because the boost_multiprecision target doesn't exist. |
| 69 | +# - beast, variant, xpressive: only for ARROW_FLIGHT_SQL_ODBC. |
| 70 | +EXCLUDED_LIBRARIES=(multiprecision beast variant xpressive) |
| 71 | + |
| 72 | +# Print the library names that build_boost() adds to BOOST_INCLUDE_LIBRARIES |
| 73 | +# with `set(BOOST_INCLUDE_LIBRARIES ...)` or |
| 74 | +# `list(APPEND BOOST_INCLUDE_LIBRARIES ...)`, one per line. Variable |
| 75 | +# references such as ${ARROW_BOOST_COMPONENTS} are skipped: those are only |
| 76 | +# non-empty when building tests, integration tests, benchmarks or ODBC. |
| 77 | +boost_libraries_from_cmake() { |
| 78 | + awk ' |
| 79 | + /^function\(build_boost\)/ { inside = 1; next } |
| 80 | + inside && /^endfunction\(\)/ { exit } |
| 81 | + inside { |
| 82 | + sub(/#.*/, "") |
| 83 | + gsub(/\(/, " ( ") |
| 84 | + gsub(/\)/, " ) ") |
| 85 | + for (i = 1; i <= NF; i++) tok[++n] = $i |
| 86 | + } |
| 87 | + END { |
| 88 | + for (i = 1; i < n; i++) { |
| 89 | + if (tok[i + 1] != "(") continue |
| 90 | + cmd = tolower(tok[i]) |
| 91 | + j = i + 2 |
| 92 | + if (cmd == "set" && tok[j] == "BOOST_INCLUDE_LIBRARIES") { |
| 93 | + j += 1 |
| 94 | + } else if (cmd == "list" && tok[j] == "APPEND" && |
| 95 | + tok[j + 1] == "BOOST_INCLUDE_LIBRARIES") { |
| 96 | + j += 2 |
| 97 | + } else { |
| 98 | + continue |
| 99 | + } |
| 100 | + for (; j <= n && tok[j] != ")"; j++) { |
| 101 | + if (tok[j] !~ /^\$/) print tok[j] |
| 102 | + } |
| 103 | + } |
| 104 | + }' "$1" |
| 105 | +} |
| 106 | + |
| 107 | +if [ -z "${BOOST_INCLUDE_LIBRARIES:-}" ]; then |
| 108 | + # tools/cpp is the synced copy of cpp/ (`make sync-cpp`); fall back to the |
| 109 | + # repository's cpp/ so this can also be run by hand from a git checkout. |
| 110 | + TOOLCHAIN="" |
| 111 | + for dir in "${SCRIPT_DIR}/../cpp" "${SCRIPT_DIR}/../../../cpp"; do |
| 112 | + if [ -f "${dir}/cmake_modules/ThirdpartyToolchain.cmake" ]; then |
| 113 | + TOOLCHAIN="${dir}/cmake_modules/ThirdpartyToolchain.cmake" |
| 114 | + break |
| 115 | + fi |
| 116 | + done |
| 117 | + if [ -z "${TOOLCHAIN}" ]; then |
| 118 | + echo "Can't find ThirdpartyToolchain.cmake; run \`make sync-cpp\` first" >&2 |
| 119 | + exit 1 |
| 120 | + fi |
| 121 | + |
| 122 | + ALL_LIBRARIES="$(boost_libraries_from_cmake "${TOOLCHAIN}")" |
| 123 | + if [ -z "${ALL_LIBRARIES}" ]; then |
| 124 | + echo "No BOOST_INCLUDE_LIBRARIES found in build_boost() in ${TOOLCHAIN}." >&2 |
| 125 | + echo "Has build_boost() changed? Update ${BASH_SOURCE[0]}." >&2 |
| 126 | + exit 1 |
| 127 | + fi |
| 128 | + for lib in "${EXCLUDED_LIBRARIES[@]}"; do |
| 129 | + if ! printf '%s\n' "${ALL_LIBRARIES}" | grep -qxF "${lib}"; then |
| 130 | + echo "Warning: '${lib}' is in EXCLUDED_LIBRARIES but build_boost() no" \ |
| 131 | + "longer uses it; remove it from ${BASH_SOURCE[0]}" >&2 |
| 132 | + fi |
| 133 | + done |
| 134 | + BOOST_INCLUDE_LIBRARIES="$(printf '%s\n' "${ALL_LIBRARIES}" | |
| 135 | + grep -vxF "$(printf '%s\n' "${EXCLUDED_LIBRARIES[@]}")" | |
| 136 | + paste -s -d ';' -)" |
| 137 | +fi |
| 138 | +echo "Trimming Boost to BOOST_INCLUDE_LIBRARIES=${BOOST_INCLUDE_LIBRARIES} and their dependencies" |
| 139 | + |
| 140 | +WORK="$(mktemp -d "${TMPDIR:-/tmp}/arrow-trim-boost.XXXXXX")" |
| 141 | +trap 'rm -rf "${WORK}"' EXIT |
| 142 | + |
| 143 | +mkdir "${WORK}/src" "${WORK}/trimmed" |
| 144 | +tar -xzf "${IN}" -C "${WORK}/src" |
| 145 | +# The tarball has a single top-level directory, e.g. boost-1.92.0 |
| 146 | +SRC_NAME="$(ls "${WORK}/src")" |
| 147 | +SRC="${WORK}/src/${SRC_NAME}" |
| 148 | +if [ ! -f "${SRC}/tools/cmake/include/BoostRoot.cmake" ]; then |
| 149 | + echo "${IN} doesn't look like a Boost CMake source tarball" >&2 |
| 150 | + exit 1 |
| 151 | +fi |
| 152 | + |
| 153 | +# Let Boost's CMake resolve the transitive dependencies. With |
| 154 | +# --log-level=VERBOSE, BoostRoot.cmake logs "Adding Boost library 'x'" for |
| 155 | +# each requested library and "Adding Boost dependency 'x'" for each |
| 156 | +# dependency (including 'headers', which is always added). |
| 157 | +# BUILD_TESTING=OFF matches prepare_fetchcontent(), so test-only |
| 158 | +# dependencies are not pulled in. |
| 159 | +cmake -S "${SRC}" -B "${WORK}/build" \ |
| 160 | + --log-level=VERBOSE \ |
| 161 | + -DBUILD_TESTING=OFF \ |
| 162 | + -DBOOST_INCLUDE_LIBRARIES="${BOOST_INCLUDE_LIBRARIES}" \ |
| 163 | + > "${WORK}/cmake.log" 2>&1 || { |
| 164 | + cat "${WORK}/cmake.log" >&2 |
| 165 | + echo "Configuring Boost failed" >&2 |
| 166 | + exit 1 |
| 167 | +} |
| 168 | +if grep -q "has not been found" "${WORK}/cmake.log"; then |
| 169 | + grep "has not been found" "${WORK}/cmake.log" >&2 |
| 170 | + echo "A library in BOOST_INCLUDE_LIBRARIES doesn't exist in ${SRC_NAME}" >&2 |
| 171 | + exit 1 |
| 172 | +fi |
| 173 | +# (bash 3.2 on macOS has no mapfile) |
| 174 | +KEEP_LIBRARIES=() |
| 175 | +while IFS= read -r lib; do |
| 176 | + KEEP_LIBRARIES+=("${lib}") |
| 177 | +done < <(sed -n "s/.*Adding Boost [a-z]* '\([^']*\)'.*/\1/p" "${WORK}/cmake.log" | LC_ALL=C sort -u) |
| 178 | +if [ "${#KEEP_LIBRARIES[@]}" -eq 0 ]; then |
| 179 | + echo "Found no \"Adding Boost ...\" messages in Boost's CMake output." >&2 |
| 180 | + echo "Has BoostRoot.cmake changed? Update ${BASH_SOURCE[0]}." >&2 |
| 181 | + exit 1 |
| 182 | +fi |
| 183 | +echo "Keeping ${#KEEP_LIBRARIES[@]} Boost libraries: ${KEEP_LIBRARIES[*]}" |
| 184 | + |
| 185 | +# Copy what the bundled build needs into ${WORK}/trimmed/${SRC_NAME}: |
| 186 | +# the top-level files (CMakeLists.txt, LICENSE_1_0.txt, ...), tools/cmake |
| 187 | +# (BoostRoot.cmake etc.) and the libraries, but not the top-level doc/, |
| 188 | +# more/, status/ or the other tools/. |
| 189 | +DEST="${WORK}/trimmed/${SRC_NAME}" |
| 190 | +mkdir -p "${DEST}/tools" |
| 191 | +find "${SRC}" -maxdepth 1 -type f -exec cp -p {} "${DEST}/" \; |
| 192 | +cp -pR "${SRC}/tools/cmake" "${DEST}/tools/" |
| 193 | +for lib in "${KEEP_LIBRARIES[@]}"; do |
| 194 | + # lib can be a nested library such as numeric/conversion |
| 195 | + mkdir -p "$(dirname "${DEST}/libs/${lib}")" |
| 196 | + cp -pR "${SRC}/libs/${lib}" "${DEST}/libs/${lib}" |
| 197 | + # Only remove these at the top level of each library: deeper down they |
| 198 | + # can be header directories (e.g. include/boost/math/tools/). |
| 199 | + for dir in test doc example examples bench benchmark perf performance meta tools; do |
| 200 | + rm -rf "${DEST}/libs/${lib:?}/${dir}" |
| 201 | + done |
| 202 | +done |
| 203 | + |
| 204 | +# COPYFILE_DISABLE stops macOS tar from adding AppleDouble (._*) files. |
| 205 | +(cd "${WORK}/trimmed" && COPYFILE_DISABLE=1 tar -czf "${OUT}" "${SRC_NAME}") |
0 commit comments