Skip to content

Commit 63f44e9

Browse files
fix(database-ha): improve startup and replication logging
1 parent b7053c1 commit 63f44e9

11 files changed

Lines changed: 307 additions & 47 deletions

File tree

‎checks/README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@
3030
| [check-php-legacy-backport.py](check-php-legacy-backport.py) | 旧 PHP 源码包校验、补丁应用与构建失败处理 | Python 3、Shell、patch;可传入 PHP 7.3.33 源码包 |
3131
| [check-php-pecl.sh](check-php-pecl.sh) | 容器内下载、编译、加载 APCu 并验证缓存读写 | Bash、Docker、已构建的 PHP 镜像、网络 |
3232
| [check-php-legacy-runtime.py](check-php-legacy-runtime.py) | PHP CLI、FastCGI、OOM 恢复、扩展及上传行为 | Python 3、Docker、已构建的 PHP 镜像 |
33-
| [check-mysql-cluster.sh](check-mysql-cluster.sh) | MySQL 默认客户端、备份恢复、全量与增量复制、只读及重启恢复 | Bash、Docker、已构建的 MySQL 主从镜像 |
34-
| [check-postgresql-cluster.sh](check-postgresql-cluster.sh) | PostgreSQL 流复制、历史认证规则保留、配置、目录权限及数据版本保护 | Bash、Docker、已构建的 PostgreSQL 主从镜像 |
35-
| [check-redis-cluster.sh](check-redis-cluster.sh) | Redis 主从复制、复制延迟等待、配置优先级、密码及 RDB/AOF 恢复 | Bash、Docker、已构建的 Redis 主从镜像 |
33+
| [check-mysql-cluster.sh](check-mysql-cluster.sh) | MySQL 默认客户端、备份恢复、全量与增量复制、只读、重启恢复及容器日志 | Bash、Docker、已构建的 MySQL 主从镜像 |
34+
| [check-postgresql-cluster.sh](check-postgresql-cluster.sh) | PostgreSQL 流复制、历史认证规则保留、配置、目录权限、数据版本保护及容器日志 | Bash、Docker、已构建的 PostgreSQL 主从镜像 |
35+
| [check-redis-cluster.sh](check-redis-cluster.sh) | Redis 主从复制、复制延迟等待、配置优先级、密码、RDB/AOF 恢复及容器日志 | Bash、Docker、已构建的 Redis 主从镜像 |
3636

3737
## 何时执行
3838

‎checks/check-mysql-cluster.sh‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,34 @@ wait_count() {
5757
return 1
5858
}
5959

60+
check_logs() {
61+
local container="$1" role="$2" phase="$3"
62+
shift 3
63+
local log_file="$test_dir/$role-$phase.log" message started_at missing='required MySQL startup logs'
64+
started_at="$(docker inspect --format '{{.State.StartedAt}}' "$container")"
65+
local deadline=$((SECONDS + 60))
66+
while (( SECONDS < deadline )); do
67+
docker logs --since "$started_at" "$container" > "$log_file" 2>&1
68+
missing=''
69+
for message in "$@"; do
70+
if ! grep -Fq -- "[INFO] MySQL cluster ($role): $message" "$log_file"; then
71+
missing="$message"
72+
break
73+
fi
74+
done
75+
if [[ -z "$missing" ]] && ! grep -Eq "\\[INFO\\] MySQL cluster \\($role\\): Starting MySQL $role with server ID [0-9]+" "$log_file"; then
76+
missing='MySQL startup stage'
77+
fi
78+
if [[ -z "$missing" ]] && ! grep -Eq '\[MY-010931\].*ready for connections.*port: 3306([[:space:]]|$)' "$log_file"; then
79+
missing='MySQL engine readiness on port 3306'
80+
fi
81+
[[ -n "$missing" ]] || return 0
82+
sleep 1
83+
done
84+
echo "Log check timed out after 60s: $container ($phase), waiting for $missing" >&2
85+
return 1
86+
}
87+
6088
run_replica() {
6189
docker run -d --name "$replica" --network "$network" \
6290
-e MYSQL_REPLICATION_MODE=slave -e MYSQL_REPLICATION_USER=replicator \
@@ -94,6 +122,8 @@ docker run -d --name "$master" --network "$network" \
94122
-v "$master_volume:/bitnami/mysql/data" \
95123
-v "$test_dir/my.cnf:/opt/bitnami/mysql/conf/my.cnf" "$image" >/dev/null
96124
wait_healthy "$master"
125+
check_logs "$master" master initial \
126+
'Configuring root authentication' 'Creating replication account' 'Initialization complete'
97127
[[ "$(docker exec "$master" id -u)" == 1001 ]]
98128
docker exec -e MYSQL_PWD="$root_password" "$master" mysqladmin --user=root ping >/dev/null
99129
[[ "$(query "$master" "SELECT plugin FROM mysql.user WHERE User='root' AND Host='localhost'")" == mysql_native_password ]]
@@ -107,6 +137,10 @@ docker exec -i -e MYSQL_PWD="$root_password" "$master" mysql --user=root ha_rest
107137
run_replica
108138
wait_healthy "$replica"
109139
wait_count 3
140+
check_logs "$replica" slave initial \
141+
'Configuring root authentication' "Waiting for master $master:3306" 'Master connection established' \
142+
'Creating initial dump from master' 'Restoring initial dump' "Configuring replication from $master:3306" \
143+
'Replication configuration applied' 'Initialization complete'
110144
[[ "$(query "$replica" 'SELECT content FROM ha_test.records WHERE id=2')" == 'existing data' ]]
111145
[[ "$(query "$replica" 'SELECT @@super_read_only')" == 1 ]]
112146
if query "$replica" "INSERT INTO ha_test.records VALUES (99,'must fail')" >/dev/null 2>&1; then
@@ -123,6 +157,7 @@ docker rm -v "$replica" >/dev/null
123157
run_replica
124158
wait_healthy "$replica"
125159
wait_count 5
160+
check_logs "$replica" slave persisted 'Using existing data'
126161
[[ "$(query "$replica" 'SELECT @@server_id')" == "$server_id" ]]
127162
if docker run --rm -e MYSQL_REPLICATION_MODE=slave -e MYSQL_REPLICATION_USER=replicator \
128163
-e MYSQL_REPLICATION_PASSWORD="$replication_password" -e MYSQL_ROOT_PASSWORD="$root_password" \
@@ -135,6 +170,7 @@ grep -q 'different replication role' "$test_dir/wrong-role.log"
135170
docker exec "$master" rm /bitnami/mysql/data/.1panel-replication-mode
136171
docker restart "$master" >/dev/null
137172
wait_healthy "$master"
173+
check_logs "$master" master persisted 'Using existing data'
138174
query "$master" "INSERT INTO ha_test.records VALUES (6,'after master restart');" >/dev/null
139175
wait_count 6
140176
[[ "$(query "$replica" "SELECT COUNT(*) FROM performance_schema.replication_applier_status WHERE SERVICE_STATE='ON'")" == 1 ]]
@@ -151,4 +187,14 @@ if docker run --rm -e MYSQL_REPLICATION_MODE=master -e MYSQL_REPLICATION_USER=re
151187
exit 1
152188
fi
153189
grep -q 'previous initialization did not finish' "$test_dir/unfinished-init.log"
190+
for log_file in "$test_dir/"*.log; do
191+
for password in "$root_password" "$replication_password" unused-on-dump-replica; do
192+
escaped_password="${password//\\/\\\\}"
193+
escaped_password="$(printf '%s' "$escaped_password" | sed "s/'/''/g")"
194+
if grep -Fq -- "$password" "$log_file" || grep -Fq -- "$escaped_password" "$log_file"; then
195+
echo "Configured password found in container output: $(basename "$log_file")" >&2
196+
exit 1
197+
fi
198+
done
199+
done
154200
echo "MySQL cluster runtime checks passed: $image"

‎checks/check-postgresql-cluster.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,46 @@ wait_rows() {
6060
return 1
6161
}
6262

63+
check_log_secrets() {
64+
if [[ "$1" == *"$password"* || "$1" == *"$replication_password"* ]]; then
65+
printf >&2 'PostgreSQL logs exposed a configured password\n'
66+
return 1
67+
fi
68+
}
69+
70+
check_startup_logs() {
71+
local container="$1"
72+
shift
73+
local started_at logs message attempt
74+
started_at="$(docker inspect --format '{{.State.StartedAt}}' "$container")"
75+
for (( attempt=0; attempt<90; attempt++ )); do
76+
logs="$(docker logs --since "$started_at" "$container" 2>&1)"
77+
check_log_secrets "$logs"
78+
if awk '
79+
/\[INFO\] PostgreSQL cluster \((master|slave)\): Starting PostgreSQL on port / { started = 1; ready = 0; next }
80+
started && /database system is ready to accept/ { ready = 1 }
81+
END { exit !ready }
82+
' <<< "$logs"; then
83+
break
84+
fi
85+
sleep 1
86+
done
87+
if (( attempt == 90 )); then
88+
printf >&2 'PostgreSQL engine readiness log is missing after the current startup: %s\n' "$container"
89+
return 1
90+
fi
91+
if ! grep -Eq '^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z \[INFO\] PostgreSQL cluster \((master|slave)\): ' <<< "$logs"; then
92+
printf >&2 'Structured PostgreSQL startup log is missing: %s\n' "$container"
93+
return 1
94+
fi
95+
for message in "$@"; do
96+
if ! grep -Fq "$message" <<< "$logs"; then
97+
printf >&2 'PostgreSQL startup log is missing for %s: %s\n' "$container" "$message"
98+
return 1
99+
fi
100+
done
101+
}
102+
63103
docker image inspect "$image" >/dev/null
64104
major="$(docker run --rm "$image" bash -ec 'printf "%s" "$PG_MAJOR"')"
65105
case "$major" in
@@ -87,6 +127,7 @@ docker run -d --name "$master" --network "$network" --network-alias pg-master \
87127
-e "POSTGRESQL_REPLICATION_PASSWORD=$replication_password" \
88128
-v "${volumes[0]}:/bitnami/postgresql" -v "${volumes[1]}:/bitnami/postgresql/conf" "$image" >/dev/null
89129
wait_healthy "$master"
130+
check_startup_logs "$master" 'Initializing master database' 'Configuring primary replication account' 'Startup configuration complete'
90131
[[ "$(sql "$master" 55432 'SHOW shared_buffers')" == 32MB ]]
91132
[[ "$(docker exec "$master" id -u)" == 1001 ]]
92133
[[ "$(sql "$master" 55432 "SELECT current_setting('server_version_num')::integer / 10000")" == "$major" ]]
@@ -109,6 +150,7 @@ docker exec "$master" bash -ec '
109150
'
110151
docker restart "$master" >/dev/null
111152
wait_healthy "$master"
153+
check_startup_logs "$master" 'Using existing master data; skipping initialization and base backup' 'Configuring primary replication account'
112154
[[ "$(sql "$master" 55432 'SELECT system_identifier FROM pg_control_system()')" == "$master_id" ]]
113155
[[ "$(docker exec -e "PGPASSWORD=$password" "$master" psql -h 127.0.0.1 -p 55432 -U postgres -d postgres -Atqc 'SELECT 1')" == 1 ]]
114156
if docker exec -e "PGPASSWORD=$password" "$master" psql -h "$master_ip" -p 55432 -U postgres -d postgres -Atqc 'SELECT 1' >/dev/null 2>&1; then
@@ -123,6 +165,7 @@ docker run -d --name "$slave" --network "$network" \
123165
-e "POSTGRESQL_REPLICATION_PASSWORD=$replication_password" \
124166
-v "${volumes[2]}:/bitnami/postgresql" -v "${volumes[3]}:/bitnami/postgresql/conf" "$image" >/dev/null
125167
wait_healthy "$slave"
168+
check_startup_logs "$slave" 'Cloning standby database from primary; waiting for base backup' 'Standby base backup complete' 'Configuring standby replication connection'
126169
[[ "$(sql "$slave" 55433 'SELECT pg_is_in_recovery()')" == t ]]
127170
[[ "$(sql "$slave" 55433 'SHOW shared_buffers')" == 48MB ]]
128171
[[ "$(sql "$slave" 55433 'SELECT system_identifier FROM pg_control_system()')" == "$master_id" ]]
@@ -145,9 +188,11 @@ docker stop "$slave" >/dev/null
145188
sql "$master" 55432 "INSERT INTO cluster_probe VALUES (3, 'offline-replica');" >/dev/null
146189
docker start "$slave" >/dev/null
147190
wait_healthy "$slave"
191+
check_startup_logs "$slave" 'Using existing slave data; skipping initialization and base backup' 'Configuring standby replication connection'
148192
wait_rows 3
149193
docker restart "$master" >/dev/null
150194
wait_healthy "$master"
195+
check_startup_logs "$master" 'Using existing master data; skipping initialization and base backup' 'Configuring primary replication account'
151196
sql "$master" 55432 "INSERT INTO cluster_probe VALUES (4, 'restarted-primary');" >/dev/null
152197
wait_rows 4
153198
[[ "$(docker exec "$slave" stat -c '%u:%g:%a' /bitnami/postgresql/conf/replication.pgpass)" == 1001:1001:600 ]]
@@ -173,6 +218,10 @@ if docker run --name "$prefix-failed" --network "$network" \
173218
printf >&2 'Clone unexpectedly succeeded against a closed port\n'
174219
exit 1
175220
fi
221+
failed_started_at="$(docker inspect --format '{{.State.StartedAt}}' "$prefix-failed")"
222+
failed_logs="$(docker logs --since "$failed_started_at" "$prefix-failed" 2>&1)"
223+
check_log_secrets "$failed_logs"
224+
grep -Fq '[ERROR] PostgreSQL cluster (slave): Standby base backup failed' <<< "$failed_logs"
176225
[[ "$(docker run --rm -v "${volumes[7]}:/conf" "$image" cat /conf/user-file)" == 'preserve configuration' ]]
177226
[[ -z "$(docker run --rm -v "${volumes[6]}:/bitnami/postgresql" "$image" find /bitnami/postgresql -maxdepth 1 -name '.clone.*' -print)" ]]
178227

@@ -187,6 +236,7 @@ fi
187236
[[ "$(docker run --rm -v "${volumes[0]}:/bitnami/postgresql" "$image" cat /bitnami/postgresql/data/PG_VERSION)" == "$major" ]]
188237
docker start "$master" >/dev/null
189238
wait_healthy "$master"
239+
check_startup_logs "$master" 'Using existing master data; skipping initialization and base backup'
190240
[[ "$(sql "$master" 55432 'SELECT system_identifier FROM pg_control_system()')" == "$master_id" ]]
191241
[[ "$(sql "$master" 55432 'SELECT count(*) FROM cluster_probe')" == 4 ]]
192242

@@ -204,6 +254,7 @@ docker run -d --user root --name "$master" --network "$network" --network-alias
204254
-e "POSTGRESQL_REPLICATION_PASSWORD=$replication_password" \
205255
-v "${volumes[0]}:/bitnami/postgresql" -v "${volumes[1]}:/bitnami/postgresql/conf" "$image" >/dev/null
206256
wait_healthy "$master"
257+
check_startup_logs "$master" 'Preparing directory permissions and switching to the postgres user' 'Using existing master data; skipping initialization and base backup' '[WARN] PostgreSQL cluster (master): Authentication configuration is missing'
207258
[[ "$(docker exec "$master" stat -c '%u:%g' /bitnami/postgresql)" == 1001:1001 ]]
208259
[[ "$(docker exec "$master" stat -c '%u' /proc/1)" == 1001 ]]
209260
[[ "$(docker exec "$master" grep -Fxc 'local all all peer' /bitnami/postgresql/conf/pg_hba.conf)" == 1 ]]

‎checks/check-redis-cluster.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,37 @@ wait_for() {
6262
return 1
6363
}
6464

65+
check_logs() {
66+
local container="$1" role="$2"
67+
shift 2
68+
local started_at logs message password missing
69+
started_at="$(docker inspect --format '{{.State.StartedAt}}' "$container")"
70+
local deadline=$((SECONDS + 60))
71+
while (( SECONDS < deadline )); do
72+
logs="$(docker logs --since "$started_at" "$container" 2>&1)"
73+
for password in "$master_password" "$replica_password" config-password; do
74+
if grep -Fq -- "$password" <<< "$logs"; then
75+
printf 'Configured password found in container output: %s\n' "$container" >&2
76+
return 1
77+
fi
78+
done
79+
missing=''
80+
for message in 'Validating replication settings' 'Preparing data and configuration directories' 'Starting Redis server' "$@"; do
81+
if ! grep -Fq -- "[INFO] Redis cluster ($role): $message" <<< "$logs"; then
82+
missing="$message"
83+
break
84+
fi
85+
done
86+
if [[ -z "$missing" && "$logs" != *'Ready to accept connections'* ]]; then
87+
missing='Redis engine accepting connections'
88+
fi
89+
[[ -n "$missing" ]] || return 0
90+
sleep 1
91+
done
92+
printf 'Log check timed out after 60s: %s, waiting for %s\n' "$container" "$missing" >&2
93+
return 1
94+
}
95+
6596
containers+=("$prefix-invalid" "$prefix-init")
6697
if docker run --rm --name "$prefix-invalid" -e REDIS_PASSWORD=unused -e REDIS_REPLICATION_MODE=sentinel "$IMAGE" >/dev/null 2>&1; then
6798
printf '%s\n' 'Unsupported replication mode was accepted' >&2
@@ -95,6 +126,8 @@ containers+=("$replica")
95126
docker start "$master" "$replica" >/dev/null
96127
wait_for PONG master_cli PING
97128
wait_for PONG replica_cli PING
129+
check_logs "$master" master 'Using mounted Redis configuration' 'Applying additional Redis arguments'
130+
check_logs "$replica" slave 'Generating default Redis configuration with AOF enabled' 'Applying replica connection settings' 'Applying additional Redis arguments'
98131

99132
[[ "$(master_cli INFO server | tr -d '\r' | sed -n 's/^redis_version://p')" == "$redis_version" ]]
100133
[[ "$(docker exec "$master" id -u)" == 1001 ]]
@@ -125,6 +158,7 @@ docker exec "$replica" test -s "$aof_file"
125158
docker kill --signal KILL "$master" >/dev/null
126159
docker start "$master" >/dev/null
127160
wait_for PONG master_cli PING
161+
check_logs "$master" master 'Using mounted Redis configuration'
128162
[[ "$(master_cli GET counter)" == 40 ]]
129163
[[ "$(master_cli GET entry:40)" == 'value 40' ]]
130164

@@ -143,11 +177,15 @@ docker stop --time 10 "$master" >/dev/null
143177
docker kill --signal KILL "$replica" >/dev/null
144178
docker start "$replica" >/dev/null
145179
wait_for PONG replica_cli PING
180+
check_logs "$replica" slave 'Generating default Redis configuration with AOF enabled' 'Applying replica connection settings'
146181
[[ "$(replica_cli GET counter)" == 80 ]]
147182
[[ "$(replica_cli GET entry:80)" == 'value 80' ]]
148183

149184
docker start "$master" >/dev/null
150185
wait_for PONG master_cli PING
186+
check_logs "$master" master 'Using mounted Redis configuration'
151187
[[ "$(master_cli INCR counter)" == 81 ]]
152188
wait_for 81 replica_cli GET counter
189+
check_logs "$master" master 'Using mounted Redis configuration'
190+
check_logs "$replica" slave 'Applying replica connection settings'
153191
printf 'Redis %s replication, configuration, persistence and reconnect checks passed.\n' "$redis_version"

‎database-ha/mysql/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ COPY healthcheck.sh /opt/bitnami/scripts/mysql/healthcheck.sh
2222

2323
RUN chmod 0755 /usr/local/bin/1panel-mysql-entrypoint /opt/bitnami/scripts/mysql/healthcheck.sh && \
2424
chown mysql:mysql /opt/bitnami/mysql/conf/my.cnf && \
25-
ln -sf /opt/bitnami/mysql/conf/my.cnf /etc/my.cnf
25+
ln -sf /opt/bitnami/mysql/conf/my.cnf /etc/my.cnf && \
26+
ln -sf /proc/1/fd/1 /opt/bitnami/mysql/logs/mysqld.log
2627

2728
USER 1001:1001
2829

‎database-ha/mysql/README.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,20 @@ bash checks/check-mysql-cluster.sh 1panel/mysql-cluster:8.4.13
4343
- 配置:`/opt/bitnami/mysql/conf/my.cnf`,支持只读挂载
4444
- 健康检查:`/opt/bitnami/scripts/mysql/healthcheck.sh`
4545
- socket、PID:`/opt/bitnami/mysql/tmp/`
46-
- 错误日志:`/opt/bitnami/mysql/logs/mysqld.log`
46+
- 错误日志:`/opt/bitnami/mysql/logs/mysqld.log`,镜像内链接到 `/proc/1/fd/1`
4747

4848
官方客户端默认配置路径 `/etc/my.cnf` 关联到上述兼容配置文件,因此面板直接调用 `mysql`、`mysqladmin`、`mysqldump` 时也会使用正确的 socket;兼容配置仍支持只读挂载。
4949

5050
入口覆盖配置中的数据库程序目录、插件目录、数据目录、socket、PID、错误日志、复制日志与 server ID,以兼容现有 Bitnami 布局的配置文件。从库以 `read_only`、`super_read_only` 启动。选择 `mysql_native_password` 时会显式启用 MySQL 8.4 的该认证插件。
5151

52+
## 日志
53+
54+
MySQL 引擎错误日志通过上述链接写入 PID 1 的标准输出,初始化和正常运行期间都可以通过 `docker logs <容器名>` 或面板容器日志查看。日志文件路径保留用于兼容配置,不再是镜像内独立累积的普通文件。
55+
56+
入口同时输出新库初始化、已有数据启动、等待主库、全量备份与导入、复制配置等阶段的进度。等待主库期间按间隔报告重试进度;阶段与重试消息不输出密码或完整认证命令。
57+
58+
初始化配置完成表示入口已完成当前节点的初始化和配置步骤,不代表复制 IO、SQL 线程均已运行或数据已经追平。健康检查仅验证本地 SQL 连接;复制状态仍需通过面板主从状态或 `SHOW REPLICA STATUS\G` 检查,包括 `Replica_IO_Running`、`Replica_SQL_Running` 及对应错误字段。
59+
5260
## 初始化与重启
5361

5462
首次启动主库时创建 root 与复制账户。从库全量初始化通过 `mysqldump --single-transaction --source-data=2` 获取数据与对应 binlog 位点,导入后配置复制,从该位点继续接收增量。主库已有数据时也使用该流程。

0 commit comments

Comments
 (0)