diff --git a/pmm_psmdb_diffauth_setup/test-auth.sh b/pmm_psmdb_diffauth_setup/test-auth.sh index ade6c3fa..7cca03a4 100755 --- a/pmm_psmdb_diffauth_setup/test-auth.sh +++ b/pmm_psmdb_diffauth_setup/test-auth.sh @@ -55,7 +55,7 @@ docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server systemctl re set +e i=1 while [ $i -le 3 ]; do - output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1) + output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address=pmm-server:8443 --metrics-mode=auto --server-username=admin --server-password=admin --server-insecure-tls) exit_code=$? if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then diff --git a/pmm_qa/data/load_pgsql.sql b/pmm_qa/data/load_pgsql.sql new file mode 100644 index 00000000..227e904c --- /dev/null +++ b/pmm_qa/data/load_pgsql.sql @@ -0,0 +1,32 @@ +-- Step 1: Show initial buffers_alloc value +SELECT 'Initial buffers_alloc' AS info, buffers_alloc FROM pg_stat_bgwriter; + +-- Step 2: Drop and create a large test table (~500MB+) +DROP TABLE IF EXISTS buffer_test; + +CREATE TABLE buffer_test AS +SELECT + generate_series(1, 1_000_000) AS id, + md5(random()::text) AS filler; + +ANALYZE buffer_test; + +-- Step 3: Perform repeated full-table scans to stress buffer allocation +DO $$ +BEGIN + FOR i IN 1..10 LOOP + RAISE NOTICE 'Running scan iteration %', i; + PERFORM COUNT(*) FROM buffer_test; + END LOOP; +END $$; + +---- Step 4: Show final buffers_alloc value +--SELECT 'Final buffers_alloc' AS info, buffers_alloc FROM pg_stat_bgwriter; +VACUUM; +SELECT pg_switch_wal(); + +-- Step 5: Query the data +SELECT * FROM buffer_test; + +-- Step 6: Delete the data +DELETE FROM buffer_test; diff --git a/pmm_qa/percona-distribution-postgresql/data/background_load.sql b/pmm_qa/percona-distribution-postgresql/data/background_load.sql new file mode 100644 index 00000000..d0dbb9a6 --- /dev/null +++ b/pmm_qa/percona-distribution-postgresql/data/background_load.sql @@ -0,0 +1,25 @@ +-- Step 1: Create a test table +CREATE TABLE IF NOT EXISTS test_users ( + id SERIAL PRIMARY KEY, + name VARCHAR(100), + email VARCHAR(150), + created_at TIMESTAMP DEFAULT NOW(), + bio TEXT +); + +-- Step 2: Insert 10,000 rows +INSERT INTO test_users (name, email, bio) +SELECT + 'User_' || gs::TEXT AS name, + 'user_' || gs::TEXT || '@example.com' AS email, + 'This is a generated bio for user #' || gs::TEXT +FROM generate_series(1, 10000) AS gs; + +-- Step 3: Query the data +SELECT * FROM test_users; + +-- Step 4: Delete the data +DELETE FROM test_users; + +-- Step 5: Drop the table +DROP TABLE test_users; diff --git a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf index 1bff8edd..eb09610b 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf @@ -1,4 +1,4 @@ -wal_level = replica +wal_level = logical max_wal_senders = 10 wal_keep_size = 64MB hot_standby = on diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml index 4cab04b1..f828d78e 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml @@ -2,6 +2,9 @@ set_fact: nodes_count: 3 when: nodes_count | int < 3 +- name: Set external facing port in patroni setup + set_fact: + pdpgsql_port: 6432 - name: Remove old data folders shell: 'rm -fr {{ data_dir }}' @@ -173,14 +176,19 @@ - name: Add service to pmm server community.docker.docker_container_exec: container: "pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=postgres --password=postgres --query-source=pgstatmonitor pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + command: pmm-admin add postgresql --username=postgres --cluster=pdpgsql_patroni_cluster --environment=pdpgsql_patroni_environment --password=postgres --query-source=pgstatmonitor pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" +- name: Add patroni service to pmm server + community.docker.docker_container_exec: + container: "pdpgsql_pmm_patroni_{{ pg_version }}_1" + command: pmm-admin add external --listen-port=8008 --service-name=patroni_service_1{{ random_service_name_value }} + - name: Add patroni service to pmm server community.docker.docker_container_exec: container: "pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}" - command: pmm-admin add external --listen-port=8008 --service-name=patroni_service_{{ item }}{{ random_service_name_value }} - loop: "{{ range(1, nodes_count | int + 1) | list }}" + command: pmm-admin add external --listen-port=8008 --cluster=pdpgsql_patroni_service_cluster --environment=pdpgsql_patroni_service_environment --service-name=patroni_service_{{ item }}{{ random_service_name_value }} + loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Log Patroni cluster community.docker.docker_container_exec: diff --git a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 242a984b..98efcc50 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -36,7 +36,7 @@ - name: Remove old PostgreSQL primary container community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_1" + name: "pdpgsql_pmm_replication_{{ pg_version }}_1" image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: always state: absent @@ -44,7 +44,7 @@ - name: Start PostgreSQL primary container community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_1" + name: "pdpgsql_pmm_replication_{{ pg_version }}_1" image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: always state: started @@ -70,7 +70,7 @@ - name: Create replication user community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: > psql -c " @@ -79,14 +79,14 @@ - name: Stop and remove replica if exists community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes - name: Start Percona Distribution PostgreSQL replica container community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: "no" state: started @@ -104,14 +104,14 @@ - name: Wipe replica data directory before basebackup community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" user: root command: rm -rf /data/db/* loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Create PostgreSQL user 'pmm' with password community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: > bash -c ' @@ -121,16 +121,16 @@ " ' -- name: Create custom database for pgbench +- name: Create custom database community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: > bash -c " echo \" - CREATE DATABASE pgbench; - \\c pgbench - GRANT CONNECT ON DATABASE pgbench TO pmm; + CREATE DATABASE test_database; + \\c test_database + GRANT CONNECT ON DATABASE test_database TO pmm; GRANT USAGE ON SCHEMA public TO pmm; GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; ALTER DEFAULT PRIVILEGES IN SCHEMA public @@ -140,20 +140,20 @@ - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" user: root command: > bash -c " export PGPASSWORD='{{ replication_password }}' && \ timeout 120s \ pg_basebackup --pgdata=/data/db -R -v -Fp -Xs -P \ - --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} > /tmp/pg_basebackup.log 2>&1 + --host=pdpgsql_pmm_replication_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} > /tmp/pg_basebackup.log 2>&1 " loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Remove temporary backup container docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" @@ -168,7 +168,7 @@ - name: Restart Percona Distribution PostgreSQL container with custom command community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" image: "{{ docker_repo }}:{{ pg_version }}" restart: true state: started @@ -188,17 +188,17 @@ - name: Install pg stat monitor. include_tasks: ./tasks/install_pg_stat-monitor.yml vars: - container_name: "pdpgsql_pmm_{{ pg_version }}" + container_name: "pdpgsql_pmm_replication_{{ pg_version }}" - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: - container_name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + container_name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Get already connected services to pmm server community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" command: > sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' register: pmm_server_services @@ -217,20 +217,57 @@ - name: Add service to pmm server community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}" + command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" -- name: Run load on primary node. - include_tasks: ../postgresql/tasks/run_load_pgsql.yml - vars: - node_name: "pdpgsql_pmm_{{ pg_version }}_1" - pgbench_clients: 10 - pgbench_time: 120 - pgbench_scale: 1000 +- name: Create pg_custom_publication view using psql + community.docker.docker_container_exec: + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + user: postgres + command: > + bash -c 'psql -U postgres -d postgres < + psql -U postgres -d test_database -c " + CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; + SELECT pg_stat_monitor_version(); + " + +- name: Copy sample load into container + community.docker.docker_container_copy_into: + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + path: ../data/load_pgsql.sql + container_path: /load_pgsql.sql + +- name: Start permanent SQL load in background + community.docker.docker_container_exec: + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + command: > + sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d test_database -f /load_pgsql.sql; sleep 30; done' > /tmp/sql_loop.log 2>&1 &" + +- name: Create logical replication slots + community.docker.docker_container_exec: + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" + user: postgres + command: psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('test_slot', 'test_decoding');" - name: Check replication status on primary community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_replication_{{ pg_version }}_1" user: postgres command: psql -c "SELECT * FROM pg_stat_replication;" diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index c5cba045..498d80b7 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -4,6 +4,8 @@ import sys import requests import re +import shutil +import yaml from scripts.get_env_value import get_value from scripts.database_options import database_options as database_configs from scripts.run_ansible_playbook import run_ansible_playbook @@ -591,6 +593,7 @@ def mongo_ssl_setup(script_filename, args): # Temporary docker compose filename compose_filename = f'docker-compose-psmdb.yml' compose_file_path = scripts_path + compose_filename + compose_file_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/pmm_psmdb_diffauth_setup/' # Create pmm-qa n/w used in workaround result = subprocess.run(['docker', 'network', 'inspect', 'pmm-qa'], capture_output=True) @@ -601,27 +604,72 @@ def mongo_ssl_setup(script_filename, args): # Add workaround (copy files) till sharding only support is ready. try: if no_server: - # Search & Replace content in the temporary compose files - subprocess.run( - ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) + shutil.copy(compose_file_folder + 'docker-compose-pmm-psmdb.yml', compose_file_folder + compose_filename) + print(f'File location is: {compose_file_folder + compose_filename}') admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' - subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) - subprocess.run(['sed', '-i', '/container_name/a\\ networks:\\\\n \\\\- pmm-qa', f'{compose_file_path}']) - subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', - f'{compose_file_path}']) - subprocess.run(['sed', '-i', - '/ depends_on:/{N;N;N;/ depends_on:\\\n pmm-server:\\\n condition: service_healthy/d;}', - f'{compose_file_path}']) - subprocess.run(['sed', '-i', '/^ pmm-server:/,/^$/{/^ ldap-server:/!d}', f'{compose_file_path}']) + with open(compose_file_folder + compose_filename, 'r') as f: + data = yaml.safe_load(f) + + if 'services' in data and 'pmm-server' in data['services']: + del data['services']['pmm-server'] + + if 'services' in data and 'kerberos' in data['services']: + del data['services']['kerberos'] + + if 'pmm-agent setup 2' in data: + data = data.replace('pmm-agent setup 2', 'pmm-agent setup --server-insecure-tls 2') + + for service in data.get('services', {}).values(): + networks = service.get('networks', []) + if isinstance(networks, list): + if 'pmm-qa' not in networks: + networks.append('pmm-qa') + service['networks'] = networks + elif isinstance(networks, dict): + networks['pmm-qa'] = {} + else: + service['networks'] = ['pmm-qa'] - # Search replace content in-line in shell file - subprocess.run(['sed', '-i', f's/pmm-agent setup 2/pmm-agent setup --server-insecure-tls 2/g', - f'{shellscript_file_path}']) - subprocess.run(['sed', '-i', f's/docker-compose-pmm-psmdb.yml/{compose_filename}/g', - f'{shellscript_file_path}']) - except subprocess.CalledProcessError as e: + # Ensure the network is declared globally + if 'networks' not in data: + data['networks'] = {} + + data['networks']['pmm-qa'] = {'external': True, 'name': 'pmm-qa'} + + psmdb_service = data.get('services', {}).get('psmdb-server') + if psmdb_service: + env = psmdb_service.get('environment', []) + + # If environment is a list (common in Docker Compose) + if isinstance(env, list): + for i, entry in enumerate(env): + if entry.startswith('PMM_AGENT_SERVER_PASSWORD='): + env[i] = f'PMM_AGENT_SERVER_PASSWORD={admin_password}' + break + else: + env.append(f'PMM_AGENT_SERVER_PASSWORD={admin_password}') + psmdb_service['environment'] = env + + # If environment is a dict (less common but valid) + elif isinstance(env, dict): + env['PMM_AGENT_SERVER_PASSWORD'] = admin_password + psmdb_service['environment'] = env + + depends_on = psmdb_service.get('depends_on') + print(f'Service depends on: {depends_on}') + if 'pmm-server' in depends_on or 'kerberos' in depends_on: + del psmdb_service['depends_on'] + + # Save it back + with open(compose_file_path, 'w') as f: + yaml.dump(data, f, sort_keys=False, default_flow_style=False) + except yaml.YAMLError as e: print(f"Error occurred: {e}") + try: + subprocess.run(['sed', '-i', f's/docker-compose-pmm-psmdb.yml/{compose_filename}/g', f'{shellscript_file_path}']) + except subprocess.CalledProcessError as e: + print(f"Error occurred: {e}") def setup_ssl_psmdb(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running