Skip to content

Commit d712b03

Browse files
pdiniz13pdiniz13
committed
azure_patroni
0 parents  commit d712b03

File tree

5 files changed

+975
-0
lines changed

5 files changed

+975
-0
lines changed

‎autopart.sh

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
#!/bin/bash
2+
3+
# An set of disks to ignore from partitioning and formatting
4+
BLACKLIST="/dev/sda|/dev/sdb"
5+
# Base directory to hold the data* files
6+
DATA_BASE="/media"
7+
8+
usage() {
9+
echo "Usage: $(basename $0) <new disk>"
10+
}
11+
12+
scan_for_new_disks() {
13+
# Looks for unpartitioned disks
14+
declare -a RET
15+
DEVS=($(ls -1 /dev/sd*|egrep -v "${BLACKLIST}"|egrep -v "[0-9]$"))
16+
for DEV in "${DEVS[@]}";
17+
do
18+
# Check each device if there is a "1" partition. If not,
19+
# "assume" it is not partitioned.
20+
if [ ! -b ${DEV}1 ];
21+
then
22+
RET+="${DEV} "
23+
fi
24+
done
25+
echo "${RET}"
26+
}
27+
28+
get_next_mountpoint() {
29+
DIRS=($(ls -1d ${DATA_BASE}/data* 2>&1| sort --version-sort))
30+
if [ -z "${DIRS[0]}" ];
31+
then
32+
echo "${DATA_BASE}/data1"
33+
return
34+
else
35+
IDX=$(echo "${DIRS[${#DIRS[@]}-1]}"|tr -d "[a-zA-Z/]" )
36+
IDX=$(( ${IDX} + 1 ))
37+
echo "${DATA_BASE}/data${IDX}"
38+
fi
39+
}
40+
41+
add_to_fstab() {
42+
UUID=${1}
43+
MOUNTPOINT=${2}
44+
grep "${UUID}" /etc/fstab >/dev/null 2>&1
45+
if [ ${?} -eq 0 ];
46+
then
47+
echo "Not adding ${UUID} to fstab again (it's already there!)"
48+
else
49+
LINE="UUID=\"${UUID}\"\t${MOUNTPOINT}\text4\tnoatime,nodiratime,nodev,noexec,nosuid\t1 2"
50+
echo -e "${LINE}" >> /etc/fstab
51+
fi
52+
}
53+
54+
is_partitioned() {
55+
# Checks if there is a valid partition table on the
56+
# specified disk
57+
OUTPUT=$(sfdisk -l ${1} 2>&1)
58+
grep "No partitions found" "${OUTPUT}" >/dev/null 2>&1
59+
return "${?}"
60+
}
61+
62+
has_filesystem() {
63+
DEVICE=${1}
64+
OUTPUT=$(file -L -s ${DEVICE})
65+
grep filesystem <<< "${OUTPUT}" > /dev/null 2>&1
66+
return ${?}
67+
}
68+
69+
do_partition() {
70+
# This function creates one (1) primary partition on the
71+
# disk, using all available space
72+
DISK=${1}
73+
echo "n
74+
p
75+
1
76+
77+
78+
w"| fdisk "${DISK}" > /dev/null 2>&1
79+
80+
#
81+
# Use the bash-specific $PIPESTATUS to ensure we get the correct exit code
82+
# from fdisk and not from echo
83+
if [ ${PIPESTATUS[1]} -ne 0 ];
84+
then
85+
echo "An error occurred partitioning ${DISK}" >&2
86+
echo "I cannot continue" >&2
87+
exit 2
88+
fi
89+
}
90+
91+
if [ -z "${1}" ];
92+
then
93+
DISKS=($(scan_for_new_disks))
94+
else
95+
DISKS=("${@}")
96+
fi
97+
echo "Disks are ${DISKS[@]}"
98+
for DISK in "${DISKS[@]}";
99+
do
100+
echo "Working on ${DISK}"
101+
is_partitioned ${DISK}
102+
if [ ${?} -ne 0 ];
103+
then
104+
echo "${DISK} is not partitioned, partitioning"
105+
do_partition ${DISK}
106+
fi
107+
PARTITION=$(fdisk -l ${DISK}|grep -A 1 Device|tail -n 1|awk '{print $1}')
108+
has_filesystem ${PARTITION}
109+
if [ ${?} -ne 0 ];
110+
then
111+
echo "Creating filesystem on ${PARTITION}."
112+
#echo "Press Ctrl-C if you don't want to destroy all data on ${PARTITION}"
113+
#sleep 5
114+
mkfs -j -t ext4 ${PARTITION}
115+
fi
116+
MOUNTPOINT=$(get_next_mountpoint)
117+
echo "Next mount point appears to be ${MOUNTPOINT}"
118+
[ -d "${MOUNTPOINT}" ] || mkdir "${MOUNTPOINT}"
119+
read UUID FS_TYPE < <(blkid -u filesystem ${PARTITION}|awk -F "[= ]" '{print $3" "$5}'|tr -d "\"")
120+
add_to_fstab "${UUID}" "${MOUNTPOINT}"
121+
echo "Mounting disk ${PARTITION} on ${MOUNTPOINT}"
122+
mount "${MOUNTPOINT}"
123+
done
124+
125+
126+

‎patroni-master.zip

127 KB
Binary file not shown.

‎postgres_startup.sh

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
#!/bin/bash
2+
3+
clusterName=$1
4+
amountZooKeepers=$2
5+
amountPostgres=$3
6+
myIndex=$4
7+
adminUsername=$5
8+
adminPassword=$6
9+
hacfgFile=postgresha.cfg
10+
patroniCfg=postgres.yml
11+
12+
sudo touch /usr/local/startup.log
13+
sudo chmod 666 /usr/local/startup.log
14+
echo "Cluster name: $clusterName" >> /usr/local/startup.log
15+
echo "Zookeepers: $amountZooKeepers" >> /usr/local/startup.log
16+
echo "Postgres: $amountPostgres" >> /usr/local/startup.log
17+
echo "Admin user: $adminUsername" >> /usr/local/startup.log
18+
echo "Admin password: $adminPassword" >> /usr/local/startup.log
19+
echo "MyIndex: $myIndex" >> /usr/local/startup.log
20+
echo "" >> /usr/local/startup.log
21+
22+
# format and partition data disk(s)
23+
# will be mounted at "/media/data(n)"
24+
#sudo chmod +x ./autopart.sh >> /usr/local/startup.log
25+
#sudo ./autopart.sh >> /usr/local/startup.log
26+
#sudo mkdir /media/data1/data
27+
#sudo chmod 777 /media/data1/data
28+
29+
# create RAID
30+
sudo apt-get --assume-yes -qq install mdadm
31+
sudo apt-get --assume-yes -qq install xfsprogs
32+
sudo chmod +x ./ebs_raid0.sh >> /usr/local/startup.log
33+
sudo ./ebs_raid0.sh /mnt/database >> /usr/local/startup.log
34+
sudo mkdir /mnt/database/data
35+
sudo chmod 777 /mnt/database/data
36+
37+
# update package lists for PostgreSQL 9.6
38+
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" >> /etc/apt/sources.list.d/pgdg.list'
39+
wget -q https://www.postgresql.org/media/keys/ACCC4CF8.asc -O - | sudo apt-key add -
40+
sudo apt-get -qq update
41+
echo "apt-get update exited with: $?"
42+
43+
44+
# install python
45+
sudo apt-get --assume-yes -qq install libpq-dev
46+
echo "apt-get libpq-dev exited with: $?"
47+
sudo apt-get --assume-yes -qq install python-dev
48+
echo "apt-get python-dev exited with: $?"
49+
sudo apt-get --assume-yes install python-pip
50+
echo "apt-get python-pip exited with: $?"
51+
sudo pip -q install boto
52+
echo "pip boto exited with: $?"
53+
sudo pip -q install psycopg2
54+
echo "pip psycopg2 exited with: $?"
55+
sudo pip -q install PyYAML --upgrade
56+
echo "pip PyYAML exited with: $?"
57+
sudo pip -q install requests --upgrade
58+
echo "pip requests exited with: $?"
59+
sudo pip -q install six --upgrade
60+
echo "pip six exited with: $?"
61+
sudo pip -q install kazoo
62+
echo "pip kazoo exited with: $?"
63+
sudo pip -q install python-etcd
64+
echo "pip python-etcd exited with: $?"
65+
sudo pip -q install python-consul
66+
echo "pip python-consul exited with: $?"
67+
sudo pip -q install click
68+
echo "pip click exited with: $?"
69+
sudo pip -q install prettytable --upgrade
70+
echo "pip prettytable exited with: $?"
71+
sudo pip -q install tzlocal
72+
echo "pip tzlocal exited with: $?"
73+
sudo pip -q install python-dateutil
74+
echo "pip python-dateutil exited with: $?"
75+
76+
77+
# install postgres
78+
# prevent postgres from autostart
79+
#sudo touch /usr/sbin/policy-rc.d
80+
#sudo chmod 777 /usr/sbin/policy-rc.d
81+
#echo exit 101 > /usr/sbin/policy-rc.d
82+
sudo apt-get --assume-yes --force-yes -qq install postgresql postgresql-contrib
83+
echo "apt-get postgresql postgresql-contrib exited with: $?"
84+
export PATH=/usr/lib/postgresql/9.6/bin:$PATH
85+
86+
# download patroni
87+
sudo apt-get --assume-yes -qq install unzip
88+
echo "apt-get unzip exited with: $0"
89+
cd /usr/local
90+
sudo wget -O /usr/local/patroni-master.zip https://github.com/zalando/patroni/archive/master.zip
91+
sudo unzip patroni-master.zip
92+
cd patroni-master
93+
94+
# write configuration
95+
sudo touch $patroniCfg
96+
sudo chmod 666 $patroniCfg
97+
echo "scope: &scope $clusterName" >> $patroniCfg
98+
echo "ttl: &ttl 30" >> $patroniCfg
99+
echo "loop_wait: &loop_wait 10" >> $patroniCfg
100+
if [ $myIndex -eq 0 ]
101+
then
102+
echo "name: postgres$myIndex" >> $patroniCfg
103+
fi
104+
echo "restapi:" >> $patroniCfg
105+
echo " listen: 10.0.101.$(($myIndex + 10)):8008" >> $patroniCfg
106+
echo " connect_address: 10.0.101.$(($myIndex + 10)):8008" >> $patroniCfg
107+
echo "" >> $patroniCfg
108+
echo "zookeeper:" >> $patroniCfg
109+
echo " scope: *scope" >> $patroniCfg
110+
echo " session_timeout: *ttl" >> $patroniCfg
111+
echo " reconnect_timeout: *loop_wait" >> $patroniCfg
112+
echo " hosts:" >> $patroniCfg
113+
i=0
114+
while [ $i -lt $amountZooKeepers ]
115+
do
116+
echo " - 10.0.100.$(($i + 10)):2181" >> $patroniCfg
117+
i=$(($i+1))
118+
done
119+
echo "" >> $patroniCfg
120+
if [ $myIndex -eq 0 ]
121+
then
122+
echo "bootstrap:" >> $patroniCfg
123+
echo " dcs:" >> $patroniCfg
124+
echo " ttl: *ttl" >> $patroniCfg
125+
echo " loop_wait: *loop_wait" >> $patroniCfg
126+
echo " retry_timeout: *loop_wait" >> $patroniCfg
127+
echo " maximum_lag_on_failover: 1048576" >> $patroniCfg
128+
echo " postgresql:" >> $patroniCfg
129+
echo " use_pg_rewind: true" >> $patroniCfg
130+
echo " use_slots: true" >> $patroniCfg
131+
echo " parameters:" >> $patroniCfg
132+
echo " archive_mode: \"on\"" >> $patroniCfg
133+
echo " archive_timeout: 1800s" >> $patroniCfg
134+
echo " archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f" >> $patroniCfg
135+
echo " recovery_conf:" >> $patroniCfg
136+
echo " restore_command: cp ../wal_archive/%f %p" >> $patroniCfg
137+
echo " initdb:" >> $patroniCfg
138+
echo " - encoding: UTF8" >> $patroniCfg
139+
echo " - data-checksums" >> $patroniCfg
140+
echo " pg_hba:" >> $patroniCfg
141+
echo " - host replication all 0.0.0.0/0 md5" >> $patroniCfg
142+
echo " - host all all 0.0.0.0/0 md5" >> $patroniCfg
143+
echo " users:" >> $patroniCfg
144+
echo " admin:" >> $patroniCfg
145+
echo " password: \"$adminPassword\"" >> $patroniCfg
146+
echo " options:" >> $patroniCfg
147+
echo " - createrole" >> $patroniCfg
148+
echo " - createdb" >> $patroniCfg
149+
fi
150+
echo "" >> $patroniCfg
151+
echo "tags:" >> $patroniCfg
152+
echo " nofailover: false" >> $patroniCfg
153+
echo " noloadbalance: false" >> $patroniCfg
154+
echo " clonefrom: false" >> $patroniCfg
155+
echo "" >> $patroniCfg
156+
echo "postgresql:" >> $patroniCfg
157+
if [ $myIndex -ne 0 ]
158+
then
159+
echo " name: postgres$myIndex" >> $patroniCfg
160+
fi
161+
echo " listen: '*:5433'" >> $patroniCfg
162+
echo " connect_address: 10.0.101.$(($myIndex + 10)):5433" >> $patroniCfg
163+
echo " data_dir: /mnt/database/data/postgresql" >> $patroniCfg
164+
echo " pgpass: /tmp/pgpass" >> $patroniCfg
165+
if [ $myIndex -ne 0 ]
166+
then
167+
echo " maximum_lag_on_failover: 1048576" >> $patroniCfg
168+
echo " use_slots: true" >> $patroniCfg
169+
echo " initdb:" >> $patroniCfg
170+
echo " - encoding: UTF8" >> $patroniCfg
171+
echo " - data-checksums" >> $patroniCfg
172+
echo " pg_rewind:" >> $patroniCfg
173+
echo " username: postgres" >> $patroniCfg
174+
echo " password: \"$adminPassword\"" >> $patroniCfg
175+
echo " pg_hba:" >> $patroniCfg
176+
echo " - host replication all 0.0.0.0/0 md5" >> $patroniCfg
177+
echo " - host all all 0.0.0.0/0 md5" >> $patroniCfg
178+
echo " replication:" >> $patroniCfg
179+
echo " username: replicator" >> $patroniCfg
180+
echo " password: \"$adminPassword\"" >> $patroniCfg
181+
echo " superuser:" >> $patroniCfg
182+
echo " username: postgres" >> $patroniCfg
183+
echo " password: \"$adminPassword\"" >> $patroniCfg
184+
echo " admin:" >> $patroniCfg
185+
echo " username: admin" >> $patroniCfg
186+
echo " password: \"$adminPassword\"" >> $patroniCfg
187+
echo " create_replica_method:" >> $patroniCfg
188+
echo " - basebackup" >> $patroniCfg
189+
echo " recovery_conf:" >> $patroniCfg
190+
echo " restore_command: cp ../wal_archive/%f %p" >> $patroniCfg
191+
echo " parameters:" >> $patroniCfg
192+
echo " archive_mode: \"on\"" >> $patroniCfg
193+
echo " wal_level: hot_standby" >> $patroniCfg
194+
echo " archive_command: mkdir -r ../wal_archive && test ! -f ../wal_archive/%f && cp %cp ../wal_archive/%f" >> $patroniCfg
195+
echo " max_wal_senders: 10" >> $patroniCfg
196+
echo " wal_keep_segments: 8" >> $patroniCfg
197+
echo " archive_timeout: 1800s" >> $patroniCfg
198+
echo " max_replication_slots: 10" >> $patroniCfg
199+
echo " hot_standby: \"on\"" >> $patroniCfg
200+
echo " wal_log_hints: \"on\"" >> $patroniCfg
201+
echo " unix_socket_directories: '.'" >> $patroniCfg
202+
else
203+
echo " authentication:" >> $patroniCfg
204+
echo " replication:" >> $patroniCfg
205+
echo " username: replicator" >> $patroniCfg
206+
echo " password: \"$adminPassword\"" >> $patroniCfg
207+
echo " superuser:" >> $patroniCfg
208+
echo " username: postgres" >> $patroniCfg
209+
echo " password: \"$adminPassword\"" >> $patroniCfg
210+
echo " parameters:" >> $patroniCfg
211+
echo " unix_socket_directories: '.'" >> $patroniCfg
212+
fi
213+
214+
215+
# install HA PROXY
216+
sudo apt-get --assume-yes install haproxy
217+
# write configuration
218+
sudo touch $hacfgFile
219+
sudo chmod 666 $hacfgFile
220+
echo "global" >> $hacfgFile
221+
echo " maxconn 100" >> $hacfgFile
222+
echo "" >> $hacfgFile
223+
echo "defaults" >> $hacfgFile
224+
echo " log global" >> $hacfgFile
225+
echo " mode tcp" >> $hacfgFile
226+
echo " retries 2" >> $hacfgFile
227+
echo " timeout client 30m" >> $hacfgFile
228+
echo " timeout connect 4s" >> $hacfgFile
229+
echo " timeout server 30m" >> $hacfgFile
230+
echo " timeout check 5s" >> $hacfgFile
231+
echo "" >> $hacfgFile
232+
echo "frontend ft_postgresql" >> $hacfgFile
233+
echo " bind *:5000" >> $hacfgFile
234+
echo " default_backend bk_db" >> $hacfgFile
235+
echo "" >> $hacfgFile
236+
echo "backend bk_db" >> $hacfgFile
237+
echo " option httpchk" >> $hacfgFile
238+
echo "" >> $hacfgFile
239+
i=0
240+
while [ $i -lt $amountPostgres ]
241+
do
242+
echo " server Postgres$i 10.0.101.$(($i + 10)):5433 maxconn 100 check port 8008" >> $hacfgFile
243+
i=$(($i+1))
244+
done
245+
246+
# start ha as deamon
247+
sudo haproxy -D -f $hacfgFile
248+
249+
# prepare patroni to restart after reboot
250+
sudo chmod 777 /etc/rc.local
251+
echo "sudo haproxy -D -f /usr/local/patroni-master/$hacfgFile" > /etc/rc.local
252+
echo "su -s /bin/bash -c 'export PATH=/usr/lib/postgresql/9.6/bin:\$PATH; /usr/local/patroni-master/patroni.py /usr/local/patroni-master/$patroniCfg >> /usr/local/startup.log' $adminUsername" >> /etc/rc.local
253+
echo "exit 0" >> /etc/rc.local
254+
255+
# start patroni
256+
su -c "export PATH=/usr/lib/postgresql/9.6/bin:$PATH; ./patroni.py $patroniCfg >> /usr/local/startup.log" $adminUsername & echo "spawned"

0 commit comments

Comments
 (0)