Skip to content

Commit 292ea0e

Browse files
fanquakefanquake
committed
Merge bitcoin/bitcoin#33677: ci: Retry image building once on failure
5555bce ci: Document why IN_GETOPT_BIN env var is needed on macOS (MarcoFalke) fabe516 ci: Export the container id in python script (MarcoFalke) fa6aa9f ci: Retry image building once on failure (MarcoFalke) fa4dbe0 ci: Allow overwriting check option in run() helper (MarcoFalke) fa8e4de ci: Use os.environ[key] access when value must be set (MarcoFalke) Pull request description: This should fix bitcoin/bitcoin#33640. It also contains a few refactor cleanups, which are explained in the corresponding commits. ACKs for top commit: l0rinc: Code review reACK 5555bce kevkevinpal: ACK [5555bce](bitcoin/bitcoin@5555bce) davidgumberg: crACK bitcoin/bitcoin@5555bce Tree-SHA512: f1ea95b0650e57d6a9f97c575a11ee461832c0715c3d1a24dbfe12ccc5366f295639d4c4827f1d01da460ddf00917ecaa627e7dbd12e405770db6c53c3778a9c
2 parents dd82c6c + 5555bce commit 292ea0e

File tree

2 files changed

+92
-75
lines changed

2 files changed

+92
-75
lines changed

‎ci/test/02_run_container.py‎

Lines changed: 92 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,19 @@
33
# Distributed under the MIT software license, see the accompanying
44
# file COPYING or https://opensource.org/license/mit/.
55

6+
from pathlib import Path
67
import os
78
import shlex
89
import subprocess
910
import sys
11+
import time
1012

1113

1214
def run(cmd, **kwargs):
1315
print("+ " + shlex.join(cmd), flush=True)
16+
kwargs.setdefault("check", True)
1417
try:
15-
return subprocess.run(cmd, check=True, **kwargs)
18+
return subprocess.run(cmd, **kwargs)
1619
except Exception as e:
1720
sys.exit(e)
1821

@@ -36,8 +39,8 @@ def main():
3639
# Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME
3740
# to allow support starting multiple runs simultaneously by the same user.
3841
env_file = "/tmp/env-{u}-{c}".format(
39-
u=os.getenv("USER"),
40-
c=os.getenv("CONTAINER_NAME"),
42+
u=os.environ["USER"],
43+
c=os.environ["CONTAINER_NAME"],
4144
)
4245
with open(env_file, "w", encoding="utf8") as file:
4346
for k, v in os.environ.items():
@@ -64,7 +67,92 @@ def main():
6467
cmd_build += [os.environ["BASE_READ_ONLY_DIR"]]
6568

6669
print(f"Building {os.environ['CONTAINER_NAME']} image tag to run in")
67-
run(cmd_build)
70+
if run(cmd_build, check=False).returncode != 0:
71+
print(f"Retry building {os.environ['CONTAINER_NAME']} image tag after failure")
72+
time.sleep(3)
73+
run(cmd_build)
74+
75+
for suffix in ["ccache", "depends", "depends_sources", "previous_releases"]:
76+
run(["docker", "volume", "create", f"{os.environ['CONTAINER_NAME']}_{suffix}"], check=False)
77+
78+
CI_CCACHE_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_ccache,dst={os.environ['CCACHE_DIR']}"
79+
CI_DEPENDS_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends,dst={os.environ['DEPENDS_DIR']}/built"
80+
CI_DEPENDS_SOURCES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends_sources,dst={os.environ['DEPENDS_DIR']}/sources"
81+
CI_PREVIOUS_RELEASES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_previous_releases,dst={os.environ['PREVIOUS_RELEASES_DIR']}"
82+
CI_BUILD_MOUNT = []
83+
84+
if os.getenv("DANGER_CI_ON_HOST_FOLDERS"):
85+
# ensure the directories exist
86+
for create_dir in [
87+
os.environ["CCACHE_DIR"],
88+
f"{os.environ['DEPENDS_DIR']}/built",
89+
f"{os.environ['DEPENDS_DIR']}/sources",
90+
os.environ["PREVIOUS_RELEASES_DIR"],
91+
os.environ["BASE_BUILD_DIR"], # Unset by default, must be defined externally
92+
]:
93+
Path(create_dir).mkdir(parents=True, exist_ok=True)
94+
95+
CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
96+
CI_DEPENDS_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/built,dst={os.environ['DEPENDS_DIR']}/built"
97+
CI_DEPENDS_SOURCES_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/sources,dst={os.environ['DEPENDS_DIR']}/sources"
98+
CI_PREVIOUS_RELEASES_MOUNT = f"type=bind,src={os.environ['PREVIOUS_RELEASES_DIR']},dst={os.environ['PREVIOUS_RELEASES_DIR']}"
99+
CI_BUILD_MOUNT = [f"--mount=type=bind,src={os.environ['BASE_BUILD_DIR']},dst={os.environ['BASE_BUILD_DIR']}"]
100+
101+
if os.getenv("DANGER_CI_ON_HOST_CCACHE_FOLDER"):
102+
if not os.path.isdir(os.environ["CCACHE_DIR"]):
103+
print(f"Error: Directory '{os.environ['CCACHE_DIR']}' must be created in advance.")
104+
sys.exit(1)
105+
CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
106+
107+
run(["docker", "network", "create", "--ipv6", "--subnet", "1111:1111::/112", "ci-ip6net"], check=False)
108+
109+
if os.getenv("RESTART_CI_DOCKER_BEFORE_RUN"):
110+
print("Restart docker before run to stop and clear all containers started with --rm")
111+
run(["podman", "container", "rm", "--force", "--all"]) # Similar to "systemctl restart docker"
112+
113+
# Still prune everything in case the filtered pruning doesn't work, or if labels were not set
114+
# on a previous run. Belt and suspenders approach, should be fine to remove in the future.
115+
# Prune images used by --external containers (e.g. build containers) when
116+
# using podman.
117+
print("Prune all dangling images")
118+
run(["podman", "image", "prune", "--force", "--external"])
119+
120+
print(f"Prune all dangling {CI_IMAGE_LABEL} images")
121+
# When detecting podman-docker, `--external` should be added.
122+
run(["docker", "image", "prune", "--force", "--filter", f"label={CI_IMAGE_LABEL}"])
123+
124+
cmd_run = ["docker", "run", "--rm", "--interactive", "--detach", "--tty"]
125+
cmd_run += [
126+
"--cap-add=LINUX_IMMUTABLE",
127+
*shlex.split(os.getenv("CI_CONTAINER_CAP", "")),
128+
f"--mount=type=bind,src={os.environ['BASE_READ_ONLY_DIR']},dst={os.environ['BASE_READ_ONLY_DIR']},readonly",
129+
f"--mount={CI_CCACHE_MOUNT}",
130+
f"--mount={CI_DEPENDS_MOUNT}",
131+
f"--mount={CI_DEPENDS_SOURCES_MOUNT}",
132+
f"--mount={CI_PREVIOUS_RELEASES_MOUNT}",
133+
*CI_BUILD_MOUNT,
134+
f"--env-file={env_file}",
135+
f"--name={os.environ['CONTAINER_NAME']}",
136+
"--network=ci-ip6net",
137+
f"--platform={os.environ['CI_IMAGE_PLATFORM']}",
138+
os.environ["CONTAINER_NAME"],
139+
]
140+
141+
container_id = run(
142+
cmd_run,
143+
stdout=subprocess.PIPE,
144+
text=True,
145+
).stdout.strip()
146+
os.environ["CI_CONTAINER_ID"] = container_id
147+
148+
# GNU getopt is required for the CI_RETRY_EXE script
149+
if os.getenv("CI_OS_NAME") == "macos":
150+
prefix = run(
151+
["brew", "--prefix", "gnu-getopt"],
152+
stdout=subprocess.PIPE,
153+
text=True,
154+
).stdout.strip()
155+
os.environ["IN_GETOPT_BIN"] = f"{prefix}/bin/getopt"
68156

69157
run(["./ci/test/02_run_container.sh"]) # run the remainder
70158

‎ci/test/02_run_container.sh‎

Lines changed: 0 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -5,76 +5,10 @@
55
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
66

77
export LC_ALL=C.UTF-8
8-
export CI_IMAGE_LABEL="bitcoin-ci-test"
98

109
set -o errexit -o pipefail -o xtrace
1110

1211
if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
13-
docker volume create "${CONTAINER_NAME}_ccache" || true
14-
docker volume create "${CONTAINER_NAME}_depends" || true
15-
docker volume create "${CONTAINER_NAME}_depends_sources" || true
16-
docker volume create "${CONTAINER_NAME}_previous_releases" || true
17-
18-
CI_CCACHE_MOUNT="type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR"
19-
CI_DEPENDS_MOUNT="type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built"
20-
CI_DEPENDS_SOURCES_MOUNT="type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources"
21-
CI_PREVIOUS_RELEASES_MOUNT="type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR"
22-
CI_BUILD_MOUNT=""
23-
24-
if [ "$DANGER_CI_ON_HOST_FOLDERS" ]; then
25-
# ensure the directories exist
26-
mkdir -p "${CCACHE_DIR}"
27-
mkdir -p "${DEPENDS_DIR}/built"
28-
mkdir -p "${DEPENDS_DIR}/sources"
29-
mkdir -p "${PREVIOUS_RELEASES_DIR}"
30-
mkdir -p "${BASE_BUILD_DIR}" # Unset by default, must be defined externally
31-
32-
CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=$CCACHE_DIR"
33-
CI_DEPENDS_MOUNT="type=bind,src=${DEPENDS_DIR}/built,dst=$DEPENDS_DIR/built"
34-
CI_DEPENDS_SOURCES_MOUNT="type=bind,src=${DEPENDS_DIR}/sources,dst=$DEPENDS_DIR/sources"
35-
CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR"
36-
CI_BUILD_MOUNT="--mount type=bind,src=${BASE_BUILD_DIR},dst=${BASE_BUILD_DIR}"
37-
fi
38-
39-
if [ "$DANGER_CI_ON_HOST_CCACHE_FOLDER" ]; then
40-
if [ ! -d "${CCACHE_DIR}" ]; then
41-
echo "Error: Directory '${CCACHE_DIR}' must be created in advance."
42-
exit 1
43-
fi
44-
CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=${CCACHE_DIR}"
45-
fi
46-
47-
docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true
48-
49-
if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
50-
echo "Restart docker before run to stop and clear all containers started with --rm"
51-
podman container rm --force --all # Similar to "systemctl restart docker"
52-
53-
# Still prune everything in case the filtered pruning doesn't work, or if labels were not set
54-
# on a previous run. Belt and suspenders approach, should be fine to remove in the future.
55-
# Prune images used by --external containers (e.g. build containers) when
56-
# using podman.
57-
echo "Prune all dangling images"
58-
podman image prune --force --external
59-
fi
60-
echo "Prune all dangling $CI_IMAGE_LABEL images"
61-
# When detecting podman-docker, `--external` should be added.
62-
docker image prune --force --filter "label=$CI_IMAGE_LABEL"
63-
64-
# shellcheck disable=SC2086
65-
CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \
66-
--mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \
67-
--mount "${CI_CCACHE_MOUNT}" \
68-
--mount "${CI_DEPENDS_MOUNT}" \
69-
--mount "${CI_DEPENDS_SOURCES_MOUNT}" \
70-
--mount "${CI_PREVIOUS_RELEASES_MOUNT}" \
71-
${CI_BUILD_MOUNT} \
72-
--env-file /tmp/env-$USER-$CONTAINER_NAME \
73-
--name "$CONTAINER_NAME" \
74-
--network ci-ip6net \
75-
--platform="${CI_IMAGE_PLATFORM}" \
76-
"$CONTAINER_NAME")
77-
export CI_CONTAINER_ID
7812
export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}"
7913
else
8014
echo "Running on host system without docker wrapper"
@@ -83,11 +17,6 @@ else
8317
mkdir -p "${PREVIOUS_RELEASES_DIR}"
8418
fi
8519

86-
if [ "$CI_OS_NAME" == "macos" ]; then
87-
IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt"
88-
export IN_GETOPT_BIN
89-
fi
90-
9120
CI_EXEC () {
9221
$CI_EXEC_CMD_PREFIX bash -c "export PATH=\"/path_with space:${BINS_SCRATCH_DIR}:${BASE_ROOT_DIR}/ci/retry:\$PATH\" && cd \"${BASE_ROOT_DIR}\" && $*"
9322
}

0 commit comments

Comments
 (0)