From 8f8cc091ca899db03293f0070368605fedb13c57 Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:36:05 -0700 Subject: [PATCH] HDDS-15824. Wait for OPEN RATIS THREE pipeline before writing keys in debug-tools acceptance test The container.db backup flow in replicas-test.sh (added by HDDS-14225) stops all datanodes for a consistent backup, restarts them, and waits only for wait_for_datanode HEALTHY before running debug/ozone-debug-tests.robot. That wait confirms the SCM heartbeat state but not that write pipelines have been re-established, so the "Write keys" suite setup can fail with a non-zero exit (255) when it writes RATIS THREE and EC keys before a pipeline is available, causing the whole Ozone-Debug-Tests suite to fail as flaky. Wait for at least one OPEN RATIS THREE pipeline (bounded at 180s) after the datanode restart, before writing keys. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/main/compose/common/replicas-test.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh index 31cc0c1f9b6e..5dd3dd17d6fa 100755 --- a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh +++ b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh @@ -66,6 +66,23 @@ for dn_container in ${datanodes}; do wait_for_datanode "${dn_container}" HEALTHY 60 done +# A datanode reporting HEALTHY only means SCM sees its heartbeat again; the +# write pipelines it belonged to are not necessarily re-established yet. Wait +# for an OPEN RATIS THREE pipeline before running the debug test, whose +# "Write keys" suite setup writes RATIS THREE and EC keys and otherwise fails +# with a non-zero exit when no pipeline is available after the restart above. +echo "Waiting for an OPEN RATIS THREE pipeline before writing keys" +SECONDS=0 +until [[ "$(docker-compose exec -T ${SCM} bash -c \ + "ozone admin pipeline list --state OPEN --type RATIS --replication THREE --json" 2>/dev/null \ + | jq -r 'length')" -ge 1 ]]; do + if [[ $SECONDS -ge 180 ]]; then + echo "Timed out waiting for an OPEN RATIS THREE pipeline" >&2 + exit 1 + fi + sleep 2 +done + execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests.robot # get block locations for key