diff --git a/cli/bin/postgres-ai.ts b/cli/bin/postgres-ai.ts index 485e6d9..c75b8e1 100644 --- a/cli/bin/postgres-ai.ts +++ b/cli/bin/postgres-ai.ts @@ -17,6 +17,7 @@ import { startMcpServer } from "../lib/mcp-server"; import { fetchIssues, fetchIssueComments, createIssueComment, fetchIssue } from "../lib/issues"; import { resolveBaseUrls } from "../lib/util"; import { applyInitPlan, buildInitPlan, connectWithSslFallback, DEFAULT_MONITORING_USER, redactPasswordsInSql, resolveAdminConnection, resolveMonitoringPassword, verifyInitSetup } from "../lib/init"; +import { REPORT_GENERATORS, CHECK_INFO, generateAllReports } from "../lib/checkup"; const execPromise = promisify(exec); const execFilePromise = promisify(execFile); @@ -2251,5 +2252,164 @@ mcp } }); +// Express checkup - direct database analysis +program + .command("checkup [conn]") + .description("generate health check reports directly from PostgreSQL (express mode)") + .option("--db-url ", "PostgreSQL connection URL (deprecated; pass as positional arg)") + .option("-h, --host ", "PostgreSQL host") + .option("-p, --port ", "PostgreSQL port") + .option("-U, --username ", "PostgreSQL user") + .option("-d, --dbname ", "PostgreSQL database name") + .option("--password ", "PostgreSQL password (otherwise uses PGPASSWORD)") + .option("--check-id ", `specific check to run: ${Object.keys(CHECK_INFO).join(", ")}, or ALL`, "ALL") + .option("--node-name ", "node name for reports", "node-01") + .option("--output ", "output directory for JSON files (default: current directory)") + .option("--json", "output to stdout as JSON instead of files") + .addHelpText( + "after", + [ + "", + "Examples:", + " postgresai checkup postgresql://user:pass@host:5432/dbname", + " postgresai checkup -h localhost -p 5432 -U postgres -d mydb", + " postgresai checkup --db-url postgresql://... --check-id A003", + " postgresai checkup postgresql://... --json", + "", + "Available checks:", + ...Object.entries(CHECK_INFO).map(([id, title]) => ` ${id}: ${title}`), + "", + "Express mode runs SQL queries directly against PostgreSQL,", + "without requiring Prometheus. Useful for quick health checks.", + ].join("\n") + ) + .action(async (conn: string | undefined, opts: { + dbUrl?: string; + host?: string; + port?: string; + username?: string; + dbname?: string; + password?: string; + checkId: string; + nodeName: string; + output?: string; + json?: boolean; + }) => { + // Build connection config + let connectionString: string | undefined; + let connectionConfig: { + host?: string; + port?: number; + user?: string; + database?: string; + password?: string; + } | undefined; + + if (conn) { + connectionString = conn; + } else if (opts.dbUrl) { + connectionString = opts.dbUrl; + } else if (opts.host || opts.username || opts.dbname) { + connectionConfig = { + host: opts.host || process.env.PGHOST || "localhost", + port: parseInt(opts.port || process.env.PGPORT || "5432", 10), + user: opts.username || process.env.PGUSER || "postgres", + database: opts.dbname || process.env.PGDATABASE || "postgres", + password: opts.password || process.env.PGPASSWORD, + }; + } else { + // Try environment variables + if (process.env.PGHOST || process.env.DATABASE_URL) { + connectionString = process.env.DATABASE_URL; + if (!connectionString) { + connectionConfig = { + host: process.env.PGHOST, + port: parseInt(process.env.PGPORT || "5432", 10), + user: process.env.PGUSER || "postgres", + database: process.env.PGDATABASE || "postgres", + password: process.env.PGPASSWORD, + }; + } + } else { + console.error("Error: Connection details required"); + console.error(""); + console.error("Provide connection via:"); + console.error(" positional argument: postgresai checkup postgresql://..."); + console.error(" flags: postgresai checkup -h host -p port -U user -d database"); + console.error(" environment: PGHOST, PGPORT, PGUSER, PGDATABASE, PGPASSWORD"); + process.exitCode = 1; + return; + } + } + + // Validate check ID + const checkId = opts.checkId.toUpperCase(); + if (checkId !== "ALL" && !REPORT_GENERATORS[checkId]) { + console.error(`Error: Unknown check ID: ${opts.checkId}`); + console.error(`Available checks: ${Object.keys(CHECK_INFO).join(", ")}, ALL`); + process.exitCode = 1; + return; + } + + // Connect to database + const client = new Client(connectionString ? { connectionString } : connectionConfig); + + try { + console.error("Connecting to PostgreSQL..."); + await client.connect(); + + const dbResult = await client.query("SELECT current_database() as db, version() as ver"); + const dbName = dbResult.rows[0]?.db; + const dbVersion = dbResult.rows[0]?.ver; + console.error(`Connected to: ${dbName}`); + console.error(`Version: ${dbVersion}`); + console.error(""); + + if (checkId === "ALL") { + // Generate all reports + console.error("Generating all reports..."); + const reports = await generateAllReports(client, opts.nodeName); + + if (opts.json) { + // Output all reports as JSON to stdout + console.log(JSON.stringify(reports, null, 2)); + } else { + // Write each report to a file + const outputDir = opts.output || process.cwd(); + for (const [id, report] of Object.entries(reports)) { + const filePath = path.join(outputDir, `${id}.json`); + fs.writeFileSync(filePath, JSON.stringify(report, null, 2), "utf8"); + console.error(`Generated: ${filePath}`); + } + console.error(""); + console.error(`All reports written to: ${outputDir}`); + } + } else { + // Generate specific report + console.error(`Generating ${checkId} report...`); + const generator = REPORT_GENERATORS[checkId]; + const report = await generator(client, opts.nodeName); + + if (opts.json) { + // Output to stdout + console.log(JSON.stringify(report, null, 2)); + } else { + // Write to file + const outputDir = opts.output || process.cwd(); + const filePath = path.join(outputDir, `${checkId}.json`); + fs.writeFileSync(filePath, JSON.stringify(report, null, 2), "utf8"); + console.error(`Generated: ${filePath}`); + } + } + + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`Error: ${message}`); + process.exitCode = 1; + } finally { + await client.end(); + } + }); + program.parseAsync(process.argv); diff --git a/cli/lib/checkup.ts b/cli/lib/checkup.ts new file mode 100644 index 0000000..8df24d0 --- /dev/null +++ b/cli/lib/checkup.ts @@ -0,0 +1,266 @@ +/** + * Express checkup module - generates JSON reports directly from PostgreSQL + * without going through Prometheus. + * + * This module reuses the same SQL queries from metrics.yml but runs them + * directly against the target database. + */ + +import { Client } from "pg"; + +/** + * PostgreSQL version information + */ +export interface PostgresVersion { + version: string; + server_version_num: string; + server_major_ver: string; + server_minor_ver: string; +} + +/** + * Setting information from pg_settings + */ +export interface SettingInfo { + setting: string; + unit: string; + category: string; + context: string; + vartype: string; + pretty_value: string; +} + +/** + * Node result for reports + */ +export interface NodeResult { + data: Record; + postgres_version?: PostgresVersion; +} + +/** + * Report structure matching JSON schemas + */ +export interface Report { + version: string | null; + build_ts: string | null; + checkId: string; + checkTitle: string; + timestamptz: string; + nodes: { + primary: string; + standbys: string[]; + }; + results: Record; +} + +/** + * SQL queries derived from metrics.yml + * These are the same queries used by pgwatch to export metrics to Prometheus + */ +export const METRICS_SQL = { + // From metrics.yml: settings metric + // Queries pg_settings for all configuration parameters + settings: ` + SELECT + name, + setting, + COALESCE(unit, '') as unit, + category, + context, + vartype, + CASE + WHEN unit = '8kB' THEN pg_size_pretty(setting::bigint * 8192) + WHEN unit = 'kB' THEN pg_size_pretty(setting::bigint * 1024) + WHEN unit = 'MB' THEN pg_size_pretty(setting::bigint * 1024 * 1024) + WHEN unit = 'B' THEN pg_size_pretty(setting::bigint) + WHEN unit = 'ms' THEN setting || ' ms' + WHEN unit = 's' THEN setting || ' s' + WHEN unit = 'min' THEN setting || ' min' + ELSE setting + END as pretty_value, + source, + CASE WHEN source <> 'default' THEN 0 ELSE 1 END as is_default + FROM pg_settings + ORDER BY name + `, + + // Version info - extracts server_version and server_version_num + version: ` + SELECT + name, + setting + FROM pg_settings + WHERE name IN ('server_version', 'server_version_num') + `, +}; + +/** + * Parse PostgreSQL version number into major and minor components + */ +export function parseVersionNum(versionNum: string): { major: string; minor: string } { + if (!versionNum || versionNum.length < 6) { + return { major: "", minor: "" }; + } + try { + const num = parseInt(versionNum, 10); + return { + major: Math.floor(num / 10000).toString(), + minor: (num % 10000).toString(), + }; + } catch { + return { major: "", minor: "" }; + } +} + +/** + * Get PostgreSQL version information + */ +export async function getPostgresVersion(client: Client): Promise { + const result = await client.query(METRICS_SQL.version); + + let version = ""; + let serverVersionNum = ""; + + for (const row of result.rows) { + if (row.name === "server_version") { + version = row.setting; + } else if (row.name === "server_version_num") { + serverVersionNum = row.setting; + } + } + + const { major, minor } = parseVersionNum(serverVersionNum); + + return { + version, + server_version_num: serverVersionNum, + server_major_ver: major, + server_minor_ver: minor, + }; +} + +/** + * Get all PostgreSQL settings + */ +export async function getSettings(client: Client): Promise> { + const result = await client.query(METRICS_SQL.settings); + const settings: Record = {}; + + for (const row of result.rows) { + settings[row.name] = { + setting: row.setting, + unit: row.unit, + category: row.category, + context: row.context, + vartype: row.vartype, + pretty_value: row.pretty_value, + }; + } + + return settings; +} + +/** + * Create base report structure + */ +export function createBaseReport( + checkId: string, + checkTitle: string, + nodeName: string +): Report { + return { + version: null, + build_ts: null, + checkId, + checkTitle, + timestamptz: new Date().toISOString(), + nodes: { + primary: nodeName, + standbys: [], + }, + results: {}, + }; +} + +/** + * Generate A002 report - Postgres major version + */ +export async function generateA002(client: Client, nodeName: string = "node-01"): Promise { + const report = createBaseReport("A002", "Postgres major version", nodeName); + const postgresVersion = await getPostgresVersion(client); + + report.results[nodeName] = { + data: { + version: postgresVersion, + }, + }; + + return report; +} + +/** + * Generate A003 report - Postgres settings + */ +export async function generateA003(client: Client, nodeName: string = "node-01"): Promise { + const report = createBaseReport("A003", "Postgres settings", nodeName); + const settings = await getSettings(client); + const postgresVersion = await getPostgresVersion(client); + + report.results[nodeName] = { + data: settings, + postgres_version: postgresVersion, + }; + + return report; +} + +/** + * Generate A013 report - Postgres minor version + */ +export async function generateA013(client: Client, nodeName: string = "node-01"): Promise { + const report = createBaseReport("A013", "Postgres minor version", nodeName); + const postgresVersion = await getPostgresVersion(client); + + report.results[nodeName] = { + data: { + version: postgresVersion, + }, + }; + + return report; +} + +/** + * Available report generators + */ +export const REPORT_GENERATORS: Record Promise> = { + A002: generateA002, + A003: generateA003, + A013: generateA013, +}; + +/** + * Check IDs and titles + */ +export const CHECK_INFO: Record = { + A002: "Postgres major version", + A003: "Postgres settings", + A013: "Postgres minor version", +}; + +/** + * Generate all available reports + */ +export async function generateAllReports( + client: Client, + nodeName: string = "node-01" +): Promise> { + const reports: Record = {}; + + for (const [checkId, generator] of Object.entries(REPORT_GENERATORS)) { + reports[checkId] = await generator(client, nodeName); + } + + return reports; +} diff --git a/reporter/postgres_reports.py b/reporter/postgres_reports.py index e480307..3aacedb 100644 --- a/reporter/postgres_reports.py +++ b/reporter/postgres_reports.py @@ -25,7 +25,73 @@ class PostgresReportGenerator: # Default databases to always exclude DEFAULT_EXCLUDED_DATABASES = {'template0', 'template1', 'rdsadmin', 'azure_maintenance', 'cloudsqladmin'} - + + # Settings filter lists for reports based on A003 + D004_SETTINGS = [ + 'pg_stat_statements.max', + 'pg_stat_statements.track', + 'pg_stat_statements.track_utility', + 'pg_stat_statements.save', + 'pg_stat_statements.track_planning', + 'shared_preload_libraries', + 'track_activities', + 'track_counts', + 'track_functions', + 'track_io_timing', + 'track_wal_io_timing' + ] + + F001_SETTINGS = [ + 'autovacuum', + 'autovacuum_analyze_scale_factor', + 'autovacuum_analyze_threshold', + 'autovacuum_freeze_max_age', + 'autovacuum_max_workers', + 'autovacuum_multixact_freeze_max_age', + 'autovacuum_naptime', + 'autovacuum_vacuum_cost_delay', + 'autovacuum_vacuum_cost_limit', + 'autovacuum_vacuum_insert_scale_factor', + 'autovacuum_vacuum_scale_factor', + 'autovacuum_vacuum_threshold', + 'autovacuum_work_mem', + 'vacuum_cost_delay', + 'vacuum_cost_limit', + 'vacuum_cost_page_dirty', + 'vacuum_cost_page_hit', + 'vacuum_cost_page_miss', + 'vacuum_freeze_min_age', + 'vacuum_freeze_table_age', + 'vacuum_multixact_freeze_min_age', + 'vacuum_multixact_freeze_table_age' + ] + + G001_SETTINGS = [ + 'shared_buffers', + 'work_mem', + 'maintenance_work_mem', + 'effective_cache_size', + 'autovacuum_work_mem', + 'max_wal_size', + 'min_wal_size', + 'wal_buffers', + 'checkpoint_completion_target', + 'max_connections', + 'max_prepared_transactions', + 'max_locks_per_transaction', + 'max_pred_locks_per_transaction', + 'max_pred_locks_per_relation', + 'max_pred_locks_per_page', + 'logical_decoding_work_mem', + 'hash_mem_multiplier', + 'temp_buffers', + 'shared_preload_libraries', + 'dynamic_shared_memory_type', + 'huge_pages', + 'max_files_per_process', + 'max_stack_depth' + ] + def __init__(self, prometheus_url: str = "http://sink-prometheus:9090", postgres_sink_url: str = "postgresql://pgwatch@sink-postgres:5432/measurements", excluded_databases: Optional[List[str]] = None): @@ -1954,13 +2020,364 @@ def format_report_data(self, check_id: str, data: Dict[str, Any], host: str = "t return template_data + def filter_a003_settings(self, a003_report: Dict[str, Any], setting_names: List[str]) -> Dict[str, Any]: + """ + Filter A003 settings data to include only specified settings. + + Args: + a003_report: Full A003 report containing all settings + setting_names: List of setting names to include + + Returns: + Filtered settings dictionary + """ + filtered = {} + # Handle both single-node and multi-node A003 report structures + results = a003_report.get('results', {}) + for node_name, node_data in results.items(): + data = node_data.get('data', {}) + for setting_name, setting_info in data.items(): + if setting_name in setting_names: + filtered[setting_name] = setting_info + return filtered + + def extract_postgres_version_from_a003(self, a003_report: Dict[str, Any], node_name: str = None) -> Dict[str, str]: + """ + Extract PostgreSQL version info from A003 report settings data. + + Derives version from server_version and server_version_num settings + which are part of the A003 settings data. + + Args: + a003_report: Full A003 report + node_name: Optional specific node name. If None, uses first available node. + + Returns: + Dictionary with postgres version info (version, server_version_num, server_major_ver, server_minor_ver) + """ + results = a003_report.get('results', {}) + if not results: + return {} + + # Get the node data + if node_name and node_name in results: + node_data = results[node_name] + else: + node_data = next(iter(results.values()), {}) + + # First check if postgres_version is already in the node result + if node_data.get('postgres_version'): + return node_data['postgres_version'] + + # Otherwise, extract from settings data (server_version, server_version_num) + data = node_data.get('data', {}) + version_str = None + version_num = None + + # Look for server_version and server_version_num in settings + if 'server_version' in data: + version_str = data['server_version'].get('setting', '') + if 'server_version_num' in data: + version_num = data['server_version_num'].get('setting', '') + + if not version_str and not version_num: + return {} + + # Parse version numbers + major_ver = "" + minor_ver = "" + if version_num and len(version_num) >= 6: + try: + num = int(version_num) + major_ver = str(num // 10000) + minor_ver = str(num % 10000) + except ValueError: + pass + + return { + "version": version_str or "", + "server_version_num": version_num or "", + "server_major_ver": major_ver, + "server_minor_ver": minor_ver + } + + def generate_d004_from_a003(self, a003_report: Dict[str, Any], cluster: str = "local", + node_name: str = "node-01") -> Dict[str, Any]: + """ + Generate D004 report by filtering A003 data for pg_stat_statements settings. + + Args: + a003_report: Full A003 report containing all settings + cluster: Cluster name (for status checks) + node_name: Node name + + Returns: + D004 report dictionary + """ + print("Generating D004 from A003 data...") + + # Filter A003 settings for D004-relevant settings + pgstat_data = self.filter_a003_settings(a003_report, self.D004_SETTINGS) + + # Check extension status (still needs direct queries) + kcache_status = self._check_pg_stat_kcache_status(cluster, node_name) + pgss_status = self._check_pg_stat_statements_status(cluster, node_name) + + # Extract postgres version from A003 + postgres_version = self.extract_postgres_version_from_a003(a003_report, node_name) + + return self.format_report_data( + "D004", + { + "settings": pgstat_data, + "pg_stat_statements_status": pgss_status, + "pg_stat_kcache_status": kcache_status, + }, + node_name, + postgres_version=postgres_version, + ) + + def generate_f001_from_a003(self, a003_report: Dict[str, Any], node_name: str = "node-01") -> Dict[str, Any]: + """ + Generate F001 report by filtering A003 data for autovacuum settings. + + Args: + a003_report: Full A003 report containing all settings + node_name: Node name + + Returns: + F001 report dictionary + """ + print("Generating F001 from A003 data...") + + # Filter A003 settings for F001-relevant settings + autovacuum_data = self.filter_a003_settings(a003_report, self.F001_SETTINGS) + + # Extract postgres version from A003 + postgres_version = self.extract_postgres_version_from_a003(a003_report, node_name) + + return self.format_report_data("F001", autovacuum_data, node_name, postgres_version=postgres_version) + + def generate_g001_from_a003(self, a003_report: Dict[str, Any], node_name: str = "node-01") -> Dict[str, Any]: + """ + Generate G001 report by filtering A003 data for memory settings. + + Args: + a003_report: Full A003 report containing all settings + node_name: Node name + + Returns: + G001 report dictionary with memory analysis + """ + print("Generating G001 from A003 data...") + + # Filter A003 settings for G001-relevant settings + memory_data = self.filter_a003_settings(a003_report, self.G001_SETTINGS) + + # Calculate memory analysis + memory_analysis = self._analyze_memory_settings(memory_data) + + # Extract postgres version from A003 + postgres_version = self.extract_postgres_version_from_a003(a003_report, node_name) + + return self.format_report_data( + "G001", + { + "settings": memory_data, + "analysis": memory_analysis, + }, + node_name, + postgres_version=postgres_version, + ) + + def report_to_markdown(self, report: Dict[str, Any]) -> str: + """ + Convert a report dictionary to markdown format. + + Args: + report: Report dictionary (D004, F001, G001, etc.) + + Returns: + Markdown string representation of the report + """ + check_id = report.get('checkId', 'Unknown') + check_title = report.get('checkTitle', 'Unknown Report') + timestamp = report.get('timestamptz', '') + nodes = report.get('nodes', {}) + results = report.get('results', {}) + + lines = [ + f"# {check_id}: {check_title}", + "", + f"**Generated:** {timestamp}", + "", + ] + + # Add postgres version if available + for node_name, node_data in results.items(): + pg_version = node_data.get('postgres_version', {}) + if pg_version: + version_str = pg_version.get('version', 'Unknown') + lines.append(f"**PostgreSQL Version:** {version_str}") + lines.append("") + break + + # Add node info + primary = nodes.get('primary', '') + standbys = nodes.get('standbys', []) + if primary: + lines.append(f"**Primary Node:** {primary}") + if standbys: + lines.append(f"**Standby Nodes:** {', '.join(standbys)}") + lines.append("") + + # Generate content based on check type + if check_id == 'D004': + lines.extend(self._d004_to_markdown_content(results)) + elif check_id == 'F001': + lines.extend(self._f001_to_markdown_content(results)) + elif check_id == 'G001': + lines.extend(self._g001_to_markdown_content(results)) + else: + lines.extend(self._generic_settings_to_markdown_content(results)) + + return '\n'.join(lines) + + def _settings_table_markdown(self, settings: Dict[str, Any]) -> List[str]: + """Generate markdown table for settings.""" + if not settings: + return ["*No settings data available.*", ""] + + lines = [ + "| Setting | Value | Category | Context |", + "|---------|-------|----------|---------|", + ] + for name, info in sorted(settings.items()): + value = info.get('pretty_value', info.get('setting', '')) + category = info.get('category', '') + context = info.get('context', '') + lines.append(f"| `{name}` | {value} | {category} | {context} |") + lines.append("") + return lines + + def _d004_to_markdown_content(self, results: Dict[str, Any]) -> List[str]: + """Generate D004-specific markdown content.""" + lines = [] + + for node_name, node_data in results.items(): + data = node_data.get('data', {}) + + lines.append(f"## Node: {node_name}") + lines.append("") + + # Settings table + lines.append("### pg_stat_statements and Related Settings") + lines.append("") + settings = data.get('settings', data) + lines.extend(self._settings_table_markdown(settings)) + + # pg_stat_statements status + pgss_status = data.get('pg_stat_statements_status', {}) + if pgss_status: + lines.append("### pg_stat_statements Status") + lines.append("") + available = "✅ Available" if pgss_status.get('extension_available') else "❌ Not Available" + lines.append(f"- **Extension:** {available}") + lines.append(f"- **Metrics Count:** {pgss_status.get('metrics_count', 0)}") + lines.append(f"- **Total Calls:** {pgss_status.get('total_calls', 0)}") + lines.append("") + + # pg_stat_kcache status + kcache_status = data.get('pg_stat_kcache_status', {}) + if kcache_status: + lines.append("### pg_stat_kcache Status") + lines.append("") + available = "✅ Available" if kcache_status.get('extension_available') else "❌ Not Available" + lines.append(f"- **Extension:** {available}") + lines.append(f"- **Metrics Count:** {kcache_status.get('metrics_count', 0)}") + lines.append("") + + return lines + + def _f001_to_markdown_content(self, results: Dict[str, Any]) -> List[str]: + """Generate F001-specific markdown content.""" + lines = [] + + for node_name, node_data in results.items(): + data = node_data.get('data', {}) + + lines.append(f"## Node: {node_name}") + lines.append("") + lines.append("### Autovacuum Settings") + lines.append("") + lines.extend(self._settings_table_markdown(data)) + + return lines + + def _g001_to_markdown_content(self, results: Dict[str, Any]) -> List[str]: + """Generate G001-specific markdown content.""" + lines = [] + + for node_name, node_data in results.items(): + data = node_data.get('data', {}) + + lines.append(f"## Node: {node_name}") + lines.append("") + + # Settings table + lines.append("### Memory-Related Settings") + lines.append("") + settings = data.get('settings', {}) + lines.extend(self._settings_table_markdown(settings)) + + # Memory analysis + analysis = data.get('analysis', {}) + mem_usage = analysis.get('estimated_total_memory_usage', {}) + if mem_usage: + lines.append("### Memory Usage Analysis") + lines.append("") + lines.append("| Metric | Value |") + lines.append("|--------|-------|") + + if mem_usage.get('shared_buffers_pretty'): + lines.append(f"| Shared Buffers | {mem_usage.get('shared_buffers_pretty')} |") + if mem_usage.get('wal_buffers_pretty'): + lines.append(f"| WAL Buffers | {mem_usage.get('wal_buffers_pretty')} |") + if mem_usage.get('shared_memory_total_pretty'): + lines.append(f"| Shared Memory Total | {mem_usage.get('shared_memory_total_pretty')} |") + if mem_usage.get('work_mem_per_connection_pretty'): + lines.append(f"| Work Mem (per connection) | {mem_usage.get('work_mem_per_connection_pretty')} |") + if mem_usage.get('max_work_mem_usage_pretty'): + lines.append(f"| Max Work Mem (all connections) | {mem_usage.get('max_work_mem_usage_pretty')} |") + if mem_usage.get('maintenance_work_mem_pretty'): + lines.append(f"| Maintenance Work Mem | {mem_usage.get('maintenance_work_mem_pretty')} |") + if mem_usage.get('effective_cache_size_pretty'): + lines.append(f"| Effective Cache Size | {mem_usage.get('effective_cache_size_pretty')} |") + + lines.append("") + + return lines + + def _generic_settings_to_markdown_content(self, results: Dict[str, Any]) -> List[str]: + """Generate generic markdown content for settings-based reports.""" + lines = [] + + for node_name, node_data in results.items(): + data = node_data.get('data', {}) + + lines.append(f"## Node: {node_name}") + lines.append("") + lines.extend(self._settings_table_markdown(data)) + + return lines + def get_check_title(self, check_id: str) -> str: """ Get the human-readable title for a check ID. - + Args: check_id: The check identifier (e.g., "H004") - + Returns: Human-readable title for the check """ @@ -2229,17 +2646,14 @@ def generate_all_reports(self, cluster: str = "local", node_name: str = None, co nodes_to_process = [node_name] all_nodes = {"primary": node_name, "standbys": []} - # Generate each report type - report_types = [ + # Reports that don't depend on A003 (generate first) + independent_report_types = [ ('A002', self.generate_a002_version_report), ('A003', self.generate_a003_settings_report), ('A004', self.generate_a004_cluster_report), ('A007', self.generate_a007_altered_settings_report), - ('D004', self.generate_d004_pgstat_settings_report), - ('F001', self.generate_f001_autovacuum_settings_report), ('F004', self.generate_f004_heap_bloat_report), ('F005', self.generate_f005_btree_bloat_report), - ('G001', self.generate_g001_memory_settings_report), ('H001', self.generate_h001_invalid_indexes_report), ('H002', self.generate_h002_unused_indexes_report), ('H004', self.generate_h004_redundant_indexes_report), @@ -2247,7 +2661,7 @@ def generate_all_reports(self, cluster: str = "local", node_name: str = None, co ('K003', self.generate_k003_top_queries_report), ] - for check_id, report_func in report_types: + for check_id, report_func in independent_report_types: if len(nodes_to_process) == 1: # Single node - generate report normally reports[check_id] = report_func(cluster, nodes_to_process[0]) @@ -2260,15 +2674,70 @@ def generate_all_reports(self, cluster: str = "local", node_name: str = None, co # Extract the data from the node report if 'results' in node_report and node in node_report['results']: combined_results[node] = node_report['results'][node] - + # Create combined report with all nodes reports[check_id] = self.format_report_data( - check_id, - combined_results, + check_id, + combined_results, all_nodes["primary"] if all_nodes["primary"] else nodes_to_process[0], all_nodes ) + # Generate D004, F001, G001 from A003 data (if A003 was generated successfully) + a003_report = reports.get('A003') + if a003_report: + # Reports derived from A003 + a003_derived_reports = [ + ('D004', lambda c, n: self.generate_d004_from_a003(a003_report, c, n)), + ('F001', lambda c, n: self.generate_f001_from_a003(a003_report, n)), + ('G001', lambda c, n: self.generate_g001_from_a003(a003_report, n)), + ] + + for check_id, report_func in a003_derived_reports: + if len(nodes_to_process) == 1: + reports[check_id] = report_func(cluster, nodes_to_process[0]) + else: + # For multi-node, use the first node as reference + # (A003 data already contains all nodes) + combined_results = {} + for node in nodes_to_process: + print(f"Generating {check_id} report for node {node} from A003...") + node_report = report_func(cluster, node) + if 'results' in node_report and node in node_report['results']: + combined_results[node] = node_report['results'][node] + + reports[check_id] = self.format_report_data( + check_id, + combined_results, + all_nodes["primary"] if all_nodes["primary"] else nodes_to_process[0], + all_nodes + ) + else: + # Fallback to direct generation if A003 failed + print("Warning: A003 report not available, generating D004/F001/G001 directly") + fallback_report_types = [ + ('D004', self.generate_d004_pgstat_settings_report), + ('F001', self.generate_f001_autovacuum_settings_report), + ('G001', self.generate_g001_memory_settings_report), + ] + for check_id, report_func in fallback_report_types: + if len(nodes_to_process) == 1: + reports[check_id] = report_func(cluster, nodes_to_process[0]) + else: + combined_results = {} + for node in nodes_to_process: + print(f"Generating {check_id} report for node {node}...") + node_report = report_func(cluster, node) + if 'results' in node_report and node in node_report['results']: + combined_results[node] = node_report['results'][node] + + reports[check_id] = self.format_report_data( + check_id, + combined_results, + all_nodes["primary"] if all_nodes["primary"] else nodes_to_process[0], + all_nodes + ) + return reports def get_all_clusters(self) -> List[str]: @@ -2566,6 +3035,8 @@ def main(): help='Specific check ID to generate (default: ALL)') parser.add_argument('--output', default='-', help='Output file (default: stdout)') + parser.add_argument('--format', choices=['json', 'markdown', 'both'], default='json', + help='Output format: json, markdown, or both (default: json)') parser.add_argument('--api-url', default='https://postgres.ai/api/general') parser.add_argument('--token', default='') parser.add_argument('--project-name', default='project-name', @@ -2623,16 +3094,28 @@ def main(): report_id = generator.create_report(args.api_url, args.token, project_name, args.epoch) reports = generator.generate_all_reports(cluster, args.node_name, combine_nodes) - + # Save reports with cluster name prefix - for report in reports: - output_filename = f"{cluster}_{report}.json" if len(clusters_to_process) > 1 else f"{report}.json" - with open(output_filename, "w") as f: - json.dump(reports[report], f, indent=2) - print(f"Generated report: {output_filename}") - if not args.no_upload: - generator.upload_report_file(args.api_url, args.token, report_id, output_filename) - + for check_id in reports: + report_data = reports[check_id] + base_name = f"{cluster}_{check_id}" if len(clusters_to_process) > 1 else check_id + + # Write JSON if format is json or both + if args.format in ('json', 'both'): + json_filename = f"{base_name}.json" + with open(json_filename, "w") as f: + json.dump(report_data, f, indent=2) + print(f"Generated JSON report: {json_filename}") + if not args.no_upload: + generator.upload_report_file(args.api_url, args.token, report_id, json_filename) + + # Write Markdown if format is markdown or both + if args.format in ('markdown', 'both'): + md_filename = f"{base_name}.md" + with open(md_filename, 'w') as f: + f.write(generator.report_to_markdown(report_data)) + print(f"Generated Markdown report: {md_filename}") + if args.output == '-': pass elif len(clusters_to_process) == 1: @@ -2650,7 +3133,13 @@ def main(): # Generate specific report - use node_name or default if args.node_name is None: args.node_name = "node-01" - + + # For D004, F001, G001 - generate A003 first and derive from it + a003_report = None + if args.check_id in ('D004', 'F001', 'G001'): + print(f"Generating A003 first for {args.check_id}...") + a003_report = generator.generate_a003_settings_report(cluster, args.node_name) + if args.check_id == 'A002': report = generator.generate_a002_version_report(cluster, args.node_name) elif args.check_id == 'A003': @@ -2660,15 +3149,24 @@ def main(): elif args.check_id == 'A007': report = generator.generate_a007_altered_settings_report(cluster, args.node_name) elif args.check_id == 'D004': - report = generator.generate_d004_pgstat_settings_report(cluster, args.node_name) + if a003_report: + report = generator.generate_d004_from_a003(a003_report, cluster, args.node_name) + else: + report = generator.generate_d004_pgstat_settings_report(cluster, args.node_name) elif args.check_id == 'F001': - report = generator.generate_f001_autovacuum_settings_report(cluster, args.node_name) + if a003_report: + report = generator.generate_f001_from_a003(a003_report, args.node_name) + else: + report = generator.generate_f001_autovacuum_settings_report(cluster, args.node_name) elif args.check_id == 'F004': report = generator.generate_f004_heap_bloat_report(cluster, args.node_name) elif args.check_id == 'F005': report = generator.generate_f005_btree_bloat_report(cluster, args.node_name) elif args.check_id == 'G001': - report = generator.generate_g001_memory_settings_report(cluster, args.node_name) + if a003_report: + report = generator.generate_g001_from_a003(a003_report, args.node_name) + else: + report = generator.generate_g001_memory_settings_report(cluster, args.node_name) elif args.check_id == 'H001': report = generator.generate_h001_invalid_indexes_report(cluster, args.node_name) elif args.check_id == 'H002': @@ -2680,18 +3178,37 @@ def main(): elif args.check_id == 'K003': report = generator.generate_k003_top_queries_report(cluster, args.node_name) - output_filename = f"{cluster}_{args.check_id}.json" if len(clusters_to_process) > 1 else args.output - + # Determine output filenames + base_name = f"{cluster}_{args.check_id}" if len(clusters_to_process) > 1 else args.check_id + json_filename = f"{base_name}.json" if args.output == '-' else args.output + md_filename = f"{base_name}.md" + + # Output based on format if args.output == '-' and len(clusters_to_process) == 1: - print(json.dumps(report, indent=2)) + if args.format == 'markdown': + print(generator.report_to_markdown(report)) + elif args.format == 'both': + print(json.dumps(report, indent=2)) + print("\n--- Markdown ---\n") + print(generator.report_to_markdown(report)) + else: + print(json.dumps(report, indent=2)) else: - with open(output_filename, 'w') as f: - json.dump(report, f, indent=2) - print(f"Report written to {output_filename}") - if not args.no_upload: - project_name = args.project_name if args.project_name != 'project-name' else cluster - report_id = generator.create_report(args.api_url, args.token, project_name, args.epoch) - generator.upload_report_file(args.api_url, args.token, report_id, output_filename) + # Write JSON if format is json or both + if args.format in ('json', 'both'): + with open(json_filename, 'w') as f: + json.dump(report, f, indent=2) + print(f"JSON report written to {json_filename}") + if not args.no_upload: + project_name = args.project_name if args.project_name != 'project-name' else cluster + report_id = generator.create_report(args.api_url, args.token, project_name, args.epoch) + generator.upload_report_file(args.api_url, args.token, report_id, json_filename) + + # Write Markdown if format is markdown or both + if args.format in ('markdown', 'both'): + with open(md_filename, 'w') as f: + f.write(generator.report_to_markdown(report)) + print(f"Markdown report written to {md_filename}") except Exception as e: print(f"Error generating reports: {e}") raise e diff --git a/reporter/schemas/A013.schema.json b/reporter/schemas/A013.schema.json new file mode 100644 index 0000000..e53d28c --- /dev/null +++ b/reporter/schemas/A013.schema.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "A013 report schema", + "type": "object", + "additionalProperties": false, + "required": ["checkId", "checkTitle", "timestamptz", "nodes", "results"], + "properties": { + "version": { "type": ["string", "null"] }, + "build_ts": { "type": ["string", "null"] }, + "checkId": { "const": "A013" }, + "checkTitle": { "const": "Postgres minor version" }, + "timestamptz": { "type": "string" }, + "nodes": { "$ref": "#/$defs/nodes" }, + "results": { + "type": "object", + "minProperties": 1, + "additionalProperties": { "$ref": "#/$defs/nodeResult" } + } + }, + "$defs": { + "nodes": { + "type": "object", + "additionalProperties": false, + "required": ["primary", "standbys"], + "properties": { + "primary": { "type": "string" }, + "standbys": { "type": "array", "items": { "type": "string" } } + } + }, + "postgresVersion": { + "type": "object", + "additionalProperties": false, + "required": ["version", "server_version_num", "server_major_ver", "server_minor_ver"], + "properties": { + "version": { "type": "string" }, + "server_version_num": { "type": "string" }, + "server_major_ver": { "type": "string" }, + "server_minor_ver": { "type": "string" } + } + }, + "data": { + "type": "object", + "additionalProperties": false, + "required": ["version"], + "properties": { + "version": { "$ref": "#/$defs/postgresVersion" } + } + }, + "nodeResult": { + "type": "object", + "additionalProperties": false, + "required": ["data"], + "properties": { + "data": { "$ref": "#/$defs/data" } + } + } + } +} diff --git a/tests/reporter/test_generators_unit.py b/tests/reporter/test_generators_unit.py index 37463ca..7e6bdff 100644 --- a/tests/reporter/test_generators_unit.py +++ b/tests/reporter/test_generators_unit.py @@ -864,16 +864,14 @@ def _(*args, **kwargs): return _ - builders = [ + # Independent builders (not derived from A003) + independent_builders = [ "generate_a002_version_report", "generate_a003_settings_report", "generate_a004_cluster_report", "generate_a007_altered_settings_report", - "generate_d004_pgstat_settings_report", - "generate_f001_autovacuum_settings_report", "generate_f004_heap_bloat_report", "generate_f005_btree_bloat_report", - "generate_g001_memory_settings_report", "generate_h001_invalid_indexes_report", "generate_h002_unused_indexes_report", "generate_h004_redundant_indexes_report", @@ -881,13 +879,28 @@ def _(*args, **kwargs): "generate_k003_top_queries_report", ] - for name in builders: + # Builders derived from A003 + a003_derived_builders = [ + "generate_d004_from_a003", + "generate_f001_from_a003", + "generate_g001_from_a003", + ] + + for name in independent_builders: + monkeypatch.setattr(generator, name, stub(name)) + + for name in a003_derived_builders: monkeypatch.setattr(generator, name, stub(name)) reports = generator.generate_all_reports("local", "node-1") - assert set(reports.keys()) == {code.split("_")[1].upper() for code in builders} - assert set(called) == set(builders) + # All report types should be generated + expected_report_codes = {'A002', 'A003', 'A004', 'A007', 'D004', 'F001', 'F004', 'F005', 'G001', 'H001', 'H002', 'H004', 'K001', 'K003'} + assert set(reports.keys()) == expected_report_codes + + # All builders should be called + all_builders = independent_builders + a003_derived_builders + assert set(called) == set(all_builders) @pytest.mark.unit