diff --git a/cli/bin/postgres-ai.ts b/cli/bin/postgres-ai.ts index 485e6d9..a9f4d5a 100644 --- a/cli/bin/postgres-ai.ts +++ b/cli/bin/postgres-ai.ts @@ -17,6 +17,7 @@ import { startMcpServer } from "../lib/mcp-server"; import { fetchIssues, fetchIssueComments, createIssueComment, fetchIssue } from "../lib/issues"; import { resolveBaseUrls } from "../lib/util"; import { applyInitPlan, buildInitPlan, connectWithSslFallback, DEFAULT_MONITORING_USER, redactPasswordsInSql, resolveAdminConnection, resolveMonitoringPassword, verifyInitSetup } from "../lib/init"; +import { REPORT_GENERATORS, CHECK_INFO, generateAllReports } from "../lib/checkup"; const execPromise = promisify(exec); const execFilePromise = promisify(execFile); @@ -529,6 +530,116 @@ program } }); +program + .command("checkup [conn]") + .description("generate health check reports directly from PostgreSQL (express mode)") + .option("--check-id ", `specific check to run: ${Object.keys(CHECK_INFO).join(", ")}, or ALL`, "ALL") + .option("--node-name ", "node name for reports", "node-01") + .option("--output ", "output directory for JSON files") + .option("--json", "output to stdout as JSON instead of files") + .addHelpText( + "after", + [ + "", + "Available checks:", + ...Object.entries(CHECK_INFO).map(([id, title]) => ` ${id}: ${title}`), + "", + "Examples:", + " postgresai checkup postgresql://user:pass@host:5432/db", + " postgresai checkup postgresql://user:pass@host:5432/db --check-id A003", + " postgresai checkup postgresql://user:pass@host:5432/db --json", + " postgresai checkup postgresql://user:pass@host:5432/db --output ./reports", + ].join("\n") + ) + .action(async (conn: string | undefined, opts: { + checkId: string; + nodeName: string; + output?: string; + json?: boolean; + }) => { + if (!conn) { + console.error("Error: PostgreSQL connection string is required"); + console.error(""); + console.error("Usage: postgresai checkup [options]"); + console.error(""); + console.error("Example:"); + console.error(" postgresai checkup postgresql://user:pass@host:5432/db"); + process.exitCode = 1; + return; + } + + const client = new Client({ connectionString: conn }); + + try { + await client.connect(); + + let reports: Record; + + if (opts.checkId === "ALL") { + reports = await generateAllReports(client, opts.nodeName); + } else { + const checkId = opts.checkId.toUpperCase(); + const generator = REPORT_GENERATORS[checkId]; + if (!generator) { + console.error(`Unknown check ID: ${opts.checkId}`); + console.error(`Available: ${Object.keys(CHECK_INFO).join(", ")}, ALL`); + process.exitCode = 1; + return; + } + reports = { [checkId]: await generator(client, opts.nodeName) }; + } + + // Output results + if (opts.json) { + console.log(JSON.stringify(reports, null, 2)); + } else if (opts.output) { + // Write to files + if (!fs.existsSync(opts.output)) { + fs.mkdirSync(opts.output, { recursive: true }); + } + for (const [checkId, report] of Object.entries(reports)) { + const filePath = path.join(opts.output, `${checkId}.json`); + fs.writeFileSync(filePath, JSON.stringify(report, null, 2), "utf8"); + console.log(`✓ ${checkId}: ${filePath}`); + } + } else { + // Default: print summary + console.log("\nHealth Check Reports Generated:"); + console.log("================================\n"); + for (const [checkId, report] of Object.entries(reports)) { + const r = report as any; + console.log(`${checkId}: ${r.checkTitle}`); + if (r.results && r.results[opts.nodeName]) { + const nodeData = r.results[opts.nodeName]; + if (nodeData.postgres_version) { + console.log(` PostgreSQL: ${nodeData.postgres_version.version}`); + } + if (checkId === "A007" && nodeData.data) { + const count = Object.keys(nodeData.data).length; + console.log(` Altered settings: ${count}`); + } + if (checkId === "A004" && nodeData.data) { + if (nodeData.data.database_sizes) { + const dbCount = Object.keys(nodeData.data.database_sizes).length; + console.log(` Databases: ${dbCount}`); + } + if (nodeData.data.general_info?.cache_hit_ratio) { + console.log(` Cache hit ratio: ${nodeData.data.general_info.cache_hit_ratio.value}%`); + } + } + } + } + console.log("\nUse --json for full output or --output to save files"); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`Error: ${message}`); + process.exitCode = 1; + } finally { + await client.end(); + } + }); + /** * Stub function for not implemented commands */ diff --git a/cli/lib/checkup.ts b/cli/lib/checkup.ts new file mode 100644 index 0000000..7ea3554 --- /dev/null +++ b/cli/lib/checkup.ts @@ -0,0 +1,568 @@ +/** + * Express checkup module - generates JSON reports directly from PostgreSQL + * without going through Prometheus. + * + * This module reuses the same SQL queries from metrics.yml but runs them + * directly against the target database. + */ + +import { Client } from "pg"; + +/** + * PostgreSQL version information + */ +export interface PostgresVersion { + version: string; + server_version_num: string; + server_major_ver: string; + server_minor_ver: string; +} + +/** + * Setting information from pg_settings + */ +export interface SettingInfo { + setting: string; + unit: string; + category: string; + context: string; + vartype: string; + pretty_value: string; +} + +/** + * Altered setting (A007) - subset of SettingInfo + */ +export interface AlteredSetting { + value: string; + unit: string; + category: string; + pretty_value: string; +} + +/** + * Cluster metric (A004) + */ +export interface ClusterMetric { + value: string; + unit: string; + description: string; +} + +/** + * Node result for reports + */ +export interface NodeResult { + data: Record; + postgres_version?: PostgresVersion; +} + +/** + * Report structure matching JSON schemas + */ +export interface Report { + version: string | null; + build_ts: string | null; + checkId: string; + checkTitle: string; + timestamptz: string; + nodes: { + primary: string; + standbys: string[]; + }; + results: Record; +} + +/** + * SQL queries derived from metrics.yml + * These are the same queries used by pgwatch to export metrics to Prometheus + */ +export const METRICS_SQL = { + // From metrics.yml: settings metric + // Queries pg_settings for all configuration parameters + settings: ` + SELECT + name, + setting, + COALESCE(unit, '') as unit, + category, + context, + vartype, + CASE + WHEN unit = '8kB' THEN pg_size_pretty(setting::bigint * 8192) + WHEN unit = 'kB' THEN pg_size_pretty(setting::bigint * 1024) + WHEN unit = 'MB' THEN pg_size_pretty(setting::bigint * 1024 * 1024) + WHEN unit = 'B' THEN pg_size_pretty(setting::bigint) + WHEN unit = 'ms' THEN setting || ' ms' + WHEN unit = 's' THEN setting || ' s' + WHEN unit = 'min' THEN setting || ' min' + ELSE setting + END as pretty_value, + source, + CASE WHEN source <> 'default' THEN 0 ELSE 1 END as is_default + FROM pg_settings + ORDER BY name + `, + + // Altered settings - non-default values only (A007) + alteredSettings: ` + SELECT + name, + setting, + COALESCE(unit, '') as unit, + category, + CASE + WHEN unit = '8kB' THEN pg_size_pretty(setting::bigint * 8192) + WHEN unit = 'kB' THEN pg_size_pretty(setting::bigint * 1024) + WHEN unit = 'MB' THEN pg_size_pretty(setting::bigint * 1024 * 1024) + WHEN unit = 'B' THEN pg_size_pretty(setting::bigint) + WHEN unit = 'ms' THEN setting || ' ms' + WHEN unit = 's' THEN setting || ' s' + WHEN unit = 'min' THEN setting || ' min' + ELSE setting + END as pretty_value + FROM pg_settings + WHERE source <> 'default' + ORDER BY name + `, + + // Version info - extracts server_version and server_version_num + version: ` + SELECT + name, + setting + FROM pg_settings + WHERE name IN ('server_version', 'server_version_num') + `, + + // Database sizes (A004) + databaseSizes: ` + SELECT + datname, + pg_database_size(datname) as size_bytes + FROM pg_database + WHERE datistemplate = false + ORDER BY size_bytes DESC + `, + + // Cluster statistics (A004) + clusterStats: ` + SELECT + sum(numbackends) as total_connections, + sum(xact_commit) as total_commits, + sum(xact_rollback) as total_rollbacks, + sum(blks_read) as blocks_read, + sum(blks_hit) as blocks_hit, + sum(tup_returned) as tuples_returned, + sum(tup_fetched) as tuples_fetched, + sum(tup_inserted) as tuples_inserted, + sum(tup_updated) as tuples_updated, + sum(tup_deleted) as tuples_deleted, + sum(deadlocks) as total_deadlocks, + sum(temp_files) as temp_files_created, + sum(temp_bytes) as temp_bytes_written + FROM pg_stat_database + WHERE datname IS NOT NULL + `, + + // Connection states (A004) + connectionStates: ` + SELECT + COALESCE(state, 'null') as state, + count(*) as count + FROM pg_stat_activity + GROUP BY state + `, + + // Uptime info (A004) + uptimeInfo: ` + SELECT + pg_postmaster_start_time() as start_time, + current_timestamp - pg_postmaster_start_time() as uptime + `, +}; + +/** + * Parse PostgreSQL version number into major and minor components + */ +export function parseVersionNum(versionNum: string): { major: string; minor: string } { + if (!versionNum || versionNum.length < 6) { + return { major: "", minor: "" }; + } + try { + const num = parseInt(versionNum, 10); + return { + major: Math.floor(num / 10000).toString(), + minor: (num % 10000).toString(), + }; + } catch { + return { major: "", minor: "" }; + } +} + +/** + * Format bytes to human readable string + */ +export function formatBytes(bytes: number): string { + if (bytes === 0) return "0 B"; + const units = ["B", "kB", "MB", "GB", "TB", "PB"]; + const i = Math.floor(Math.log(bytes) / Math.log(1024)); + return `${(bytes / Math.pow(1024, i)).toFixed(2)} ${units[i]}`; +} + +/** + * Get PostgreSQL version information + */ +export async function getPostgresVersion(client: Client): Promise { + const result = await client.query(METRICS_SQL.version); + + let version = ""; + let serverVersionNum = ""; + + for (const row of result.rows) { + if (row.name === "server_version") { + version = row.setting; + } else if (row.name === "server_version_num") { + serverVersionNum = row.setting; + } + } + + const { major, minor } = parseVersionNum(serverVersionNum); + + return { + version, + server_version_num: serverVersionNum, + server_major_ver: major, + server_minor_ver: minor, + }; +} + +/** + * Get all PostgreSQL settings + */ +export async function getSettings(client: Client): Promise> { + const result = await client.query(METRICS_SQL.settings); + const settings: Record = {}; + + for (const row of result.rows) { + settings[row.name] = { + setting: row.setting, + unit: row.unit, + category: row.category, + context: row.context, + vartype: row.vartype, + pretty_value: row.pretty_value, + }; + } + + return settings; +} + +/** + * Get altered (non-default) PostgreSQL settings + */ +export async function getAlteredSettings(client: Client): Promise> { + const result = await client.query(METRICS_SQL.alteredSettings); + const settings: Record = {}; + + for (const row of result.rows) { + settings[row.name] = { + value: row.setting, + unit: row.unit, + category: row.category, + pretty_value: row.pretty_value, + }; + } + + return settings; +} + +/** + * Get database sizes + */ +export async function getDatabaseSizes(client: Client): Promise> { + const result = await client.query(METRICS_SQL.databaseSizes); + const sizes: Record = {}; + + for (const row of result.rows) { + sizes[row.datname] = parseInt(row.size_bytes, 10); + } + + return sizes; +} + +/** + * Get cluster general info metrics + */ +export async function getClusterInfo(client: Client): Promise> { + const info: Record = {}; + + // Get cluster statistics + const statsResult = await client.query(METRICS_SQL.clusterStats); + if (statsResult.rows.length > 0) { + const stats = statsResult.rows[0]; + + info.total_connections = { + value: String(stats.total_connections || 0), + unit: "connections", + description: "Total active database connections", + }; + + info.total_commits = { + value: String(stats.total_commits || 0), + unit: "transactions", + description: "Total committed transactions", + }; + + info.total_rollbacks = { + value: String(stats.total_rollbacks || 0), + unit: "transactions", + description: "Total rolled back transactions", + }; + + const blocksHit = parseInt(stats.blocks_hit || "0", 10); + const blocksRead = parseInt(stats.blocks_read || "0", 10); + const totalBlocks = blocksHit + blocksRead; + const cacheHitRatio = totalBlocks > 0 ? ((blocksHit / totalBlocks) * 100).toFixed(2) : "0.00"; + + info.cache_hit_ratio = { + value: cacheHitRatio, + unit: "%", + description: "Buffer cache hit ratio", + }; + + info.blocks_read = { + value: String(blocksRead), + unit: "blocks", + description: "Total disk blocks read", + }; + + info.blocks_hit = { + value: String(blocksHit), + unit: "blocks", + description: "Total buffer cache hits", + }; + + info.tuples_returned = { + value: String(stats.tuples_returned || 0), + unit: "rows", + description: "Total rows returned by queries", + }; + + info.tuples_fetched = { + value: String(stats.tuples_fetched || 0), + unit: "rows", + description: "Total rows fetched by queries", + }; + + info.tuples_inserted = { + value: String(stats.tuples_inserted || 0), + unit: "rows", + description: "Total rows inserted", + }; + + info.tuples_updated = { + value: String(stats.tuples_updated || 0), + unit: "rows", + description: "Total rows updated", + }; + + info.tuples_deleted = { + value: String(stats.tuples_deleted || 0), + unit: "rows", + description: "Total rows deleted", + }; + + info.total_deadlocks = { + value: String(stats.total_deadlocks || 0), + unit: "deadlocks", + description: "Total deadlocks detected", + }; + + info.temp_files_created = { + value: String(stats.temp_files_created || 0), + unit: "files", + description: "Total temporary files created", + }; + + const tempBytes = parseInt(stats.temp_bytes_written || "0", 10); + info.temp_bytes_written = { + value: formatBytes(tempBytes), + unit: "bytes", + description: "Total temporary file bytes written", + }; + } + + // Get connection states + const connResult = await client.query(METRICS_SQL.connectionStates); + for (const row of connResult.rows) { + const stateKey = `connections_${row.state.replace(/\s+/g, "_")}`; + info[stateKey] = { + value: String(row.count), + unit: "connections", + description: `Connections in '${row.state}' state`, + }; + } + + // Get uptime info + const uptimeResult = await client.query(METRICS_SQL.uptimeInfo); + if (uptimeResult.rows.length > 0) { + const uptime = uptimeResult.rows[0]; + info.start_time = { + value: uptime.start_time.toISOString(), + unit: "timestamp", + description: "PostgreSQL server start time", + }; + info.uptime = { + value: uptime.uptime, + unit: "interval", + description: "Server uptime", + }; + } + + return info; +} + +/** + * Create base report structure + */ +export function createBaseReport( + checkId: string, + checkTitle: string, + nodeName: string +): Report { + return { + version: null, + build_ts: null, + checkId, + checkTitle, + timestamptz: new Date().toISOString(), + nodes: { + primary: nodeName, + standbys: [], + }, + results: {}, + }; +} + +/** + * Generate A002 report - Postgres major version + */ +export async function generateA002(client: Client, nodeName: string = "node-01"): Promise { + const report = createBaseReport("A002", "Postgres major version", nodeName); + const postgresVersion = await getPostgresVersion(client); + + report.results[nodeName] = { + data: { + version: postgresVersion, + }, + }; + + return report; +} + +/** + * Generate A003 report - Postgres settings + */ +export async function generateA003(client: Client, nodeName: string = "node-01"): Promise { + const report = createBaseReport("A003", "Postgres settings", nodeName); + const settings = await getSettings(client); + const postgresVersion = await getPostgresVersion(client); + + report.results[nodeName] = { + data: settings, + postgres_version: postgresVersion, + }; + + return report; +} + +/** + * Generate A004 report - Cluster information + */ +export async function generateA004(client: Client, nodeName: string = "node-01"): Promise { + const report = createBaseReport("A004", "Cluster information", nodeName); + const generalInfo = await getClusterInfo(client); + const databaseSizes = await getDatabaseSizes(client); + const postgresVersion = await getPostgresVersion(client); + + report.results[nodeName] = { + data: { + general_info: generalInfo, + database_sizes: databaseSizes, + }, + postgres_version: postgresVersion, + }; + + return report; +} + +/** + * Generate A007 report - Altered settings + */ +export async function generateA007(client: Client, nodeName: string = "node-01"): Promise { + const report = createBaseReport("A007", "Altered settings", nodeName); + const alteredSettings = await getAlteredSettings(client); + const postgresVersion = await getPostgresVersion(client); + + report.results[nodeName] = { + data: alteredSettings, + postgres_version: postgresVersion, + }; + + return report; +} + +/** + * Generate A013 report - Postgres minor version + */ +export async function generateA013(client: Client, nodeName: string = "node-01"): Promise { + const report = createBaseReport("A013", "Postgres minor version", nodeName); + const postgresVersion = await getPostgresVersion(client); + + report.results[nodeName] = { + data: { + version: postgresVersion, + }, + }; + + return report; +} + +/** + * Available report generators + */ +export const REPORT_GENERATORS: Record Promise> = { + A002: generateA002, + A003: generateA003, + A004: generateA004, + A007: generateA007, + A013: generateA013, +}; + +/** + * Check IDs and titles + */ +export const CHECK_INFO: Record = { + A002: "Postgres major version", + A003: "Postgres settings", + A004: "Cluster information", + A007: "Altered settings", + A013: "Postgres minor version", +}; + +/** + * Generate all available reports + */ +export async function generateAllReports( + client: Client, + nodeName: string = "node-01" +): Promise> { + const reports: Record = {}; + + for (const [checkId, generator] of Object.entries(REPORT_GENERATORS)) { + reports[checkId] = await generator(client, nodeName); + } + + return reports; +} diff --git a/cli/test/checkup.test.cjs b/cli/test/checkup.test.cjs new file mode 100644 index 0000000..da29851 --- /dev/null +++ b/cli/test/checkup.test.cjs @@ -0,0 +1,545 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); + +// These tests intentionally import the compiled JS output. +// Run via: npm --prefix cli test +const checkup = require("../dist/lib/checkup.js"); + +function runCli(args, env = {}) { + const { spawnSync } = require("node:child_process"); + const path = require("node:path"); + const node = process.execPath; + const cliPath = path.resolve(__dirname, "..", "dist", "bin", "postgres-ai.js"); + return spawnSync(node, [cliPath, ...args], { + encoding: "utf8", + env: { ...process.env, ...env }, + }); +} + +// Unit tests for parseVersionNum +test("parseVersionNum parses PG 16.3 version number", () => { + const result = checkup.parseVersionNum("160003"); + assert.equal(result.major, "16"); + assert.equal(result.minor, "3"); +}); + +test("parseVersionNum parses PG 15.7 version number", () => { + const result = checkup.parseVersionNum("150007"); + assert.equal(result.major, "15"); + assert.equal(result.minor, "7"); +}); + +test("parseVersionNum parses PG 14.12 version number", () => { + const result = checkup.parseVersionNum("140012"); + assert.equal(result.major, "14"); + assert.equal(result.minor, "12"); +}); + +test("parseVersionNum handles empty string", () => { + const result = checkup.parseVersionNum(""); + assert.equal(result.major, ""); + assert.equal(result.minor, ""); +}); + +test("parseVersionNum handles null/undefined", () => { + const result = checkup.parseVersionNum(null); + assert.equal(result.major, ""); + assert.equal(result.minor, ""); +}); + +test("parseVersionNum handles short string", () => { + const result = checkup.parseVersionNum("123"); + assert.equal(result.major, ""); + assert.equal(result.minor, ""); +}); + +// Unit tests for createBaseReport +test("createBaseReport creates correct structure", () => { + const report = checkup.createBaseReport("A002", "Postgres major version", "test-node"); + + assert.equal(report.checkId, "A002"); + assert.equal(report.checkTitle, "Postgres major version"); + assert.equal(report.version, null); + assert.equal(report.build_ts, null); + assert.equal(report.nodes.primary, "test-node"); + assert.deepEqual(report.nodes.standbys, []); + assert.deepEqual(report.results, {}); + assert.ok(typeof report.timestamptz === "string"); + // Verify timestamp is ISO format + assert.ok(new Date(report.timestamptz).toISOString() === report.timestamptz); +}); + +test("createBaseReport uses provided node name", () => { + const report = checkup.createBaseReport("A003", "Postgres settings", "my-custom-node"); + assert.equal(report.nodes.primary, "my-custom-node"); +}); + +// Tests for CHECK_INFO +test("CHECK_INFO contains A002", () => { + assert.ok("A002" in checkup.CHECK_INFO); + assert.equal(checkup.CHECK_INFO.A002, "Postgres major version"); +}); + +test("CHECK_INFO contains A003", () => { + assert.ok("A003" in checkup.CHECK_INFO); + assert.equal(checkup.CHECK_INFO.A003, "Postgres settings"); +}); + +test("CHECK_INFO contains A013", () => { + assert.ok("A013" in checkup.CHECK_INFO); + assert.equal(checkup.CHECK_INFO.A013, "Postgres minor version"); +}); + +test("CHECK_INFO contains A004", () => { + assert.ok("A004" in checkup.CHECK_INFO); + assert.equal(checkup.CHECK_INFO.A004, "Cluster information"); +}); + +test("CHECK_INFO contains A007", () => { + assert.ok("A007" in checkup.CHECK_INFO); + assert.equal(checkup.CHECK_INFO.A007, "Altered settings"); +}); + +// Tests for REPORT_GENERATORS +test("REPORT_GENERATORS has generator for A002", () => { + assert.ok("A002" in checkup.REPORT_GENERATORS); + assert.equal(typeof checkup.REPORT_GENERATORS.A002, "function"); +}); + +test("REPORT_GENERATORS has generator for A003", () => { + assert.ok("A003" in checkup.REPORT_GENERATORS); + assert.equal(typeof checkup.REPORT_GENERATORS.A003, "function"); +}); + +test("REPORT_GENERATORS has generator for A013", () => { + assert.ok("A013" in checkup.REPORT_GENERATORS); + assert.equal(typeof checkup.REPORT_GENERATORS.A013, "function"); +}); + +test("REPORT_GENERATORS has generator for A004", () => { + assert.ok("A004" in checkup.REPORT_GENERATORS); + assert.equal(typeof checkup.REPORT_GENERATORS.A004, "function"); +}); + +test("REPORT_GENERATORS has generator for A007", () => { + assert.ok("A007" in checkup.REPORT_GENERATORS); + assert.equal(typeof checkup.REPORT_GENERATORS.A007, "function"); +}); + +test("REPORT_GENERATORS and CHECK_INFO have same keys", () => { + const generatorKeys = Object.keys(checkup.REPORT_GENERATORS).sort(); + const infoKeys = Object.keys(checkup.CHECK_INFO).sort(); + assert.deepEqual(generatorKeys, infoKeys); +}); + +// Tests for METRICS_SQL +test("METRICS_SQL.settings queries pg_settings", () => { + assert.ok(checkup.METRICS_SQL.settings.includes("pg_settings")); + assert.ok(checkup.METRICS_SQL.settings.includes("name")); + assert.ok(checkup.METRICS_SQL.settings.includes("setting")); +}); + +test("METRICS_SQL.version queries server_version fields", () => { + assert.ok(checkup.METRICS_SQL.version.includes("server_version")); + assert.ok(checkup.METRICS_SQL.version.includes("server_version_num")); +}); + +test("METRICS_SQL.alteredSettings filters non-default settings", () => { + assert.ok(checkup.METRICS_SQL.alteredSettings.includes("pg_settings")); + assert.ok(checkup.METRICS_SQL.alteredSettings.includes("source <> 'default'")); +}); + +test("METRICS_SQL.databaseSizes queries pg_database", () => { + assert.ok(checkup.METRICS_SQL.databaseSizes.includes("pg_database")); + assert.ok(checkup.METRICS_SQL.databaseSizes.includes("pg_database_size")); +}); + +test("METRICS_SQL.clusterStats queries pg_stat_database", () => { + assert.ok(checkup.METRICS_SQL.clusterStats.includes("pg_stat_database")); + assert.ok(checkup.METRICS_SQL.clusterStats.includes("xact_commit")); + assert.ok(checkup.METRICS_SQL.clusterStats.includes("deadlocks")); +}); + +test("METRICS_SQL.connectionStates queries pg_stat_activity", () => { + assert.ok(checkup.METRICS_SQL.connectionStates.includes("pg_stat_activity")); + assert.ok(checkup.METRICS_SQL.connectionStates.includes("state")); +}); + +// Tests for formatBytes +test("formatBytes formats zero bytes", () => { + assert.equal(checkup.formatBytes(0), "0 B"); +}); + +test("formatBytes formats bytes", () => { + assert.equal(checkup.formatBytes(500), "500.00 B"); +}); + +test("formatBytes formats kilobytes", () => { + assert.equal(checkup.formatBytes(1024), "1.00 kB"); + assert.equal(checkup.formatBytes(1536), "1.50 kB"); +}); + +test("formatBytes formats megabytes", () => { + assert.equal(checkup.formatBytes(1048576), "1.00 MB"); +}); + +test("formatBytes formats gigabytes", () => { + assert.equal(checkup.formatBytes(1073741824), "1.00 GB"); +}); + +// Mock client tests for report generators +function createMockClient(versionRows, settingsRows, options = {}) { + const { + alteredSettingsRows = [], + databaseSizesRows = [], + clusterStatsRows = [], + connectionStatesRows = [], + uptimeRows = [], + } = options; + + return { + query: async (sql) => { + // Version query (used by many reports) + if (sql.includes("server_version") && sql.includes("server_version_num") && !sql.includes("ORDER BY")) { + return { rows: versionRows }; + } + // Full settings query (A003) - check this BEFORE altered settings + // because full settings has "ORDER BY" and "CASE WHEN source <> 'default'" + // while altered settings has "WHERE source <> 'default'" + if (sql.includes("pg_settings") && sql.includes("ORDER BY") && sql.includes("is_default")) { + return { rows: settingsRows }; + } + // Altered settings query (A007) - has "WHERE source <> 'default'" (not in a CASE) + if (sql.includes("pg_settings") && sql.includes("WHERE source <> 'default'")) { + return { rows: alteredSettingsRows }; + } + // Database sizes (A004) + if (sql.includes("pg_database") && sql.includes("pg_database_size")) { + return { rows: databaseSizesRows }; + } + // Cluster stats (A004) + if (sql.includes("pg_stat_database") && sql.includes("xact_commit")) { + return { rows: clusterStatsRows }; + } + // Connection states (A004) + if (sql.includes("pg_stat_activity") && sql.includes("state")) { + return { rows: connectionStatesRows }; + } + // Uptime info (A004) + if (sql.includes("pg_postmaster_start_time")) { + return { rows: uptimeRows }; + } + throw new Error(`Unexpected query: ${sql}`); + }, + }; +} + +test("getPostgresVersion extracts version info from mock client", async () => { + const mockClient = createMockClient([ + { name: "server_version", setting: "16.3" }, + { name: "server_version_num", setting: "160003" }, + ], []); + + const version = await checkup.getPostgresVersion(mockClient); + assert.equal(version.version, "16.3"); + assert.equal(version.server_version_num, "160003"); + assert.equal(version.server_major_ver, "16"); + assert.equal(version.server_minor_ver, "3"); +}); + +test("getSettings transforms rows to keyed object", async () => { + const mockClient = createMockClient([], [ + { + name: "shared_buffers", + setting: "16384", + unit: "8kB", + category: "Resource Usage / Memory", + context: "postmaster", + vartype: "integer", + pretty_value: "128 MB", + }, + { + name: "work_mem", + setting: "4096", + unit: "kB", + category: "Resource Usage / Memory", + context: "user", + vartype: "integer", + pretty_value: "4 MB", + }, + ]); + + const settings = await checkup.getSettings(mockClient); + assert.ok("shared_buffers" in settings); + assert.ok("work_mem" in settings); + assert.equal(settings.shared_buffers.setting, "16384"); + assert.equal(settings.shared_buffers.unit, "8kB"); + assert.equal(settings.work_mem.pretty_value, "4 MB"); +}); + +test("generateA002 creates report with version data", async () => { + const mockClient = createMockClient([ + { name: "server_version", setting: "16.3" }, + { name: "server_version_num", setting: "160003" }, + ], []); + + const report = await checkup.generateA002(mockClient, "test-node"); + assert.equal(report.checkId, "A002"); + assert.equal(report.checkTitle, "Postgres major version"); + assert.equal(report.nodes.primary, "test-node"); + assert.ok("test-node" in report.results); + assert.ok("version" in report.results["test-node"].data); + assert.equal(report.results["test-node"].data.version.version, "16.3"); +}); + +test("generateA003 creates report with settings and version", async () => { + const mockClient = createMockClient( + [ + { name: "server_version", setting: "16.3" }, + { name: "server_version_num", setting: "160003" }, + ], + [ + { + name: "shared_buffers", + setting: "16384", + unit: "8kB", + category: "Resource Usage / Memory", + context: "postmaster", + vartype: "integer", + pretty_value: "128 MB", + }, + ] + ); + + const report = await checkup.generateA003(mockClient, "test-node"); + assert.equal(report.checkId, "A003"); + assert.equal(report.checkTitle, "Postgres settings"); + assert.ok("test-node" in report.results); + assert.ok("shared_buffers" in report.results["test-node"].data); + assert.ok(report.results["test-node"].postgres_version); + assert.equal(report.results["test-node"].postgres_version.version, "16.3"); +}); + +test("generateA013 creates report with minor version data", async () => { + const mockClient = createMockClient([ + { name: "server_version", setting: "16.3" }, + { name: "server_version_num", setting: "160003" }, + ], []); + + const report = await checkup.generateA013(mockClient, "test-node"); + assert.equal(report.checkId, "A013"); + assert.equal(report.checkTitle, "Postgres minor version"); + assert.equal(report.nodes.primary, "test-node"); + assert.ok("test-node" in report.results); + assert.ok("version" in report.results["test-node"].data); + assert.equal(report.results["test-node"].data.version.server_minor_ver, "3"); +}); + +test("generateAllReports returns reports for all checks", async () => { + const mockClient = createMockClient( + [ + { name: "server_version", setting: "16.3" }, + { name: "server_version_num", setting: "160003" }, + ], + [ + { + name: "shared_buffers", + setting: "16384", + unit: "8kB", + category: "Resource Usage / Memory", + context: "postmaster", + vartype: "integer", + pretty_value: "128 MB", + }, + ], + { + alteredSettingsRows: [ + { name: "shared_buffers", setting: "16384", unit: "8kB", category: "Resource Usage / Memory", pretty_value: "128 MB" }, + ], + databaseSizesRows: [{ datname: "postgres", size_bytes: "1073741824" }], + clusterStatsRows: [{ total_connections: 5, total_commits: 100, total_rollbacks: 1, blocks_read: 1000, blocks_hit: 9000, tuples_returned: 500, tuples_fetched: 400, tuples_inserted: 50, tuples_updated: 30, tuples_deleted: 10, total_deadlocks: 0, temp_files_created: 0, temp_bytes_written: 0 }], + connectionStatesRows: [{ state: "active", count: 2 }, { state: "idle", count: 3 }], + uptimeRows: [{ start_time: new Date("2024-01-01T00:00:00Z"), uptime: "10 days" }], + } + ); + + const reports = await checkup.generateAllReports(mockClient, "test-node"); + assert.ok("A002" in reports); + assert.ok("A003" in reports); + assert.ok("A004" in reports); + assert.ok("A007" in reports); + assert.ok("A013" in reports); + assert.equal(reports.A002.checkId, "A002"); + assert.equal(reports.A003.checkId, "A003"); + assert.equal(reports.A004.checkId, "A004"); + assert.equal(reports.A007.checkId, "A007"); + assert.equal(reports.A013.checkId, "A013"); +}); + +// Tests for A007 (Altered settings) +test("getAlteredSettings returns non-default settings", async () => { + const mockClient = createMockClient([], [], { + alteredSettingsRows: [ + { name: "shared_buffers", setting: "256MB", unit: "", category: "Resource Usage / Memory", pretty_value: "256 MB" }, + { name: "work_mem", setting: "64MB", unit: "", category: "Resource Usage / Memory", pretty_value: "64 MB" }, + ], + }); + + const settings = await checkup.getAlteredSettings(mockClient); + assert.ok("shared_buffers" in settings); + assert.ok("work_mem" in settings); + assert.equal(settings.shared_buffers.value, "256MB"); + assert.equal(settings.work_mem.pretty_value, "64 MB"); +}); + +test("generateA007 creates report with altered settings", async () => { + const mockClient = createMockClient( + [ + { name: "server_version", setting: "16.3" }, + { name: "server_version_num", setting: "160003" }, + ], + [], + { + alteredSettingsRows: [ + { name: "max_connections", setting: "200", unit: "", category: "Connections and Authentication", pretty_value: "200" }, + ], + } + ); + + const report = await checkup.generateA007(mockClient, "test-node"); + assert.equal(report.checkId, "A007"); + assert.equal(report.checkTitle, "Altered settings"); + assert.equal(report.nodes.primary, "test-node"); + assert.ok("test-node" in report.results); + assert.ok("max_connections" in report.results["test-node"].data); + assert.equal(report.results["test-node"].data.max_connections.value, "200"); + assert.ok(report.results["test-node"].postgres_version); +}); + +// Tests for A004 (Cluster information) +test("getDatabaseSizes returns database sizes", async () => { + const mockClient = createMockClient([], [], { + databaseSizesRows: [ + { datname: "postgres", size_bytes: "1073741824" }, + { datname: "mydb", size_bytes: "536870912" }, + ], + }); + + const sizes = await checkup.getDatabaseSizes(mockClient); + assert.ok("postgres" in sizes); + assert.ok("mydb" in sizes); + assert.equal(sizes.postgres, 1073741824); + assert.equal(sizes.mydb, 536870912); +}); + +test("getClusterInfo returns cluster metrics", async () => { + const mockClient = createMockClient([], [], { + clusterStatsRows: [{ + total_connections: 10, + total_commits: 1000, + total_rollbacks: 5, + blocks_read: 500, + blocks_hit: 9500, + tuples_returned: 5000, + tuples_fetched: 4000, + tuples_inserted: 100, + tuples_updated: 50, + tuples_deleted: 25, + total_deadlocks: 0, + temp_files_created: 2, + temp_bytes_written: 1048576, + }], + connectionStatesRows: [ + { state: "active", count: 3 }, + { state: "idle", count: 7 }, + ], + uptimeRows: [{ + start_time: new Date("2024-01-01T00:00:00Z"), + uptime: "30 days", + }], + }); + + const info = await checkup.getClusterInfo(mockClient); + assert.ok("total_connections" in info); + assert.ok("cache_hit_ratio" in info); + assert.ok("connections_active" in info); + assert.ok("connections_idle" in info); + assert.ok("start_time" in info); + assert.equal(info.total_connections.value, "10"); + assert.equal(info.cache_hit_ratio.value, "95.00"); + assert.equal(info.connections_active.value, "3"); +}); + +test("generateA004 creates report with cluster info and database sizes", async () => { + const mockClient = createMockClient( + [ + { name: "server_version", setting: "16.3" }, + { name: "server_version_num", setting: "160003" }, + ], + [], + { + databaseSizesRows: [ + { datname: "postgres", size_bytes: "1073741824" }, + ], + clusterStatsRows: [{ + total_connections: 5, + total_commits: 100, + total_rollbacks: 1, + blocks_read: 100, + blocks_hit: 900, + tuples_returned: 500, + tuples_fetched: 400, + tuples_inserted: 50, + tuples_updated: 30, + tuples_deleted: 10, + total_deadlocks: 0, + temp_files_created: 0, + temp_bytes_written: 0, + }], + connectionStatesRows: [{ state: "active", count: 2 }], + uptimeRows: [{ start_time: new Date("2024-01-01T00:00:00Z"), uptime: "10 days" }], + } + ); + + const report = await checkup.generateA004(mockClient, "test-node"); + assert.equal(report.checkId, "A004"); + assert.equal(report.checkTitle, "Cluster information"); + assert.equal(report.nodes.primary, "test-node"); + assert.ok("test-node" in report.results); + + const data = report.results["test-node"].data; + assert.ok("general_info" in data); + assert.ok("database_sizes" in data); + assert.ok("total_connections" in data.general_info); + assert.ok("postgres" in data.database_sizes); + assert.equal(data.database_sizes.postgres, 1073741824); + assert.ok(report.results["test-node"].postgres_version); +}); + +// CLI tests +test("cli: checkup command exists and shows help", () => { + const r = runCli(["checkup", "--help"]); + assert.equal(r.status, 0, r.stderr || r.stdout); + assert.match(r.stdout, /express mode/i); + assert.match(r.stdout, /--check-id/); + assert.match(r.stdout, /--node-name/); + assert.match(r.stdout, /--output/); + assert.match(r.stdout, /--json/); +}); + +test("cli: checkup --help shows available check IDs", () => { + const r = runCli(["checkup", "--help"]); + assert.equal(r.status, 0, r.stderr || r.stdout); + assert.match(r.stdout, /A002/); + assert.match(r.stdout, /A003/); + assert.match(r.stdout, /A004/); + assert.match(r.stdout, /A007/); + assert.match(r.stdout, /A013/); +}); + +test("cli: checkup without connection shows error", () => { + const r = runCli(["checkup"]); + assert.notEqual(r.status, 0); + // Should show connection required error + assert.match(r.stderr, /connection|required|PostgreSQL/i); +}); diff --git a/reporter/postgres_reports.py b/reporter/postgres_reports.py index e480307..3aacedb 100644 --- a/reporter/postgres_reports.py +++ b/reporter/postgres_reports.py @@ -25,7 +25,73 @@ class PostgresReportGenerator: # Default databases to always exclude DEFAULT_EXCLUDED_DATABASES = {'template0', 'template1', 'rdsadmin', 'azure_maintenance', 'cloudsqladmin'} - + + # Settings filter lists for reports based on A003 + D004_SETTINGS = [ + 'pg_stat_statements.max', + 'pg_stat_statements.track', + 'pg_stat_statements.track_utility', + 'pg_stat_statements.save', + 'pg_stat_statements.track_planning', + 'shared_preload_libraries', + 'track_activities', + 'track_counts', + 'track_functions', + 'track_io_timing', + 'track_wal_io_timing' + ] + + F001_SETTINGS = [ + 'autovacuum', + 'autovacuum_analyze_scale_factor', + 'autovacuum_analyze_threshold', + 'autovacuum_freeze_max_age', + 'autovacuum_max_workers', + 'autovacuum_multixact_freeze_max_age', + 'autovacuum_naptime', + 'autovacuum_vacuum_cost_delay', + 'autovacuum_vacuum_cost_limit', + 'autovacuum_vacuum_insert_scale_factor', + 'autovacuum_vacuum_scale_factor', + 'autovacuum_vacuum_threshold', + 'autovacuum_work_mem', + 'vacuum_cost_delay', + 'vacuum_cost_limit', + 'vacuum_cost_page_dirty', + 'vacuum_cost_page_hit', + 'vacuum_cost_page_miss', + 'vacuum_freeze_min_age', + 'vacuum_freeze_table_age', + 'vacuum_multixact_freeze_min_age', + 'vacuum_multixact_freeze_table_age' + ] + + G001_SETTINGS = [ + 'shared_buffers', + 'work_mem', + 'maintenance_work_mem', + 'effective_cache_size', + 'autovacuum_work_mem', + 'max_wal_size', + 'min_wal_size', + 'wal_buffers', + 'checkpoint_completion_target', + 'max_connections', + 'max_prepared_transactions', + 'max_locks_per_transaction', + 'max_pred_locks_per_transaction', + 'max_pred_locks_per_relation', + 'max_pred_locks_per_page', + 'logical_decoding_work_mem', + 'hash_mem_multiplier', + 'temp_buffers', + 'shared_preload_libraries', + 'dynamic_shared_memory_type', + 'huge_pages', + 'max_files_per_process', + 'max_stack_depth' + ] + def __init__(self, prometheus_url: str = "http://sink-prometheus:9090", postgres_sink_url: str = "postgresql://pgwatch@sink-postgres:5432/measurements", excluded_databases: Optional[List[str]] = None): @@ -1954,13 +2020,364 @@ def format_report_data(self, check_id: str, data: Dict[str, Any], host: str = "t return template_data + def filter_a003_settings(self, a003_report: Dict[str, Any], setting_names: List[str]) -> Dict[str, Any]: + """ + Filter A003 settings data to include only specified settings. + + Args: + a003_report: Full A003 report containing all settings + setting_names: List of setting names to include + + Returns: + Filtered settings dictionary + """ + filtered = {} + # Handle both single-node and multi-node A003 report structures + results = a003_report.get('results', {}) + for node_name, node_data in results.items(): + data = node_data.get('data', {}) + for setting_name, setting_info in data.items(): + if setting_name in setting_names: + filtered[setting_name] = setting_info + return filtered + + def extract_postgres_version_from_a003(self, a003_report: Dict[str, Any], node_name: str = None) -> Dict[str, str]: + """ + Extract PostgreSQL version info from A003 report settings data. + + Derives version from server_version and server_version_num settings + which are part of the A003 settings data. + + Args: + a003_report: Full A003 report + node_name: Optional specific node name. If None, uses first available node. + + Returns: + Dictionary with postgres version info (version, server_version_num, server_major_ver, server_minor_ver) + """ + results = a003_report.get('results', {}) + if not results: + return {} + + # Get the node data + if node_name and node_name in results: + node_data = results[node_name] + else: + node_data = next(iter(results.values()), {}) + + # First check if postgres_version is already in the node result + if node_data.get('postgres_version'): + return node_data['postgres_version'] + + # Otherwise, extract from settings data (server_version, server_version_num) + data = node_data.get('data', {}) + version_str = None + version_num = None + + # Look for server_version and server_version_num in settings + if 'server_version' in data: + version_str = data['server_version'].get('setting', '') + if 'server_version_num' in data: + version_num = data['server_version_num'].get('setting', '') + + if not version_str and not version_num: + return {} + + # Parse version numbers + major_ver = "" + minor_ver = "" + if version_num and len(version_num) >= 6: + try: + num = int(version_num) + major_ver = str(num // 10000) + minor_ver = str(num % 10000) + except ValueError: + pass + + return { + "version": version_str or "", + "server_version_num": version_num or "", + "server_major_ver": major_ver, + "server_minor_ver": minor_ver + } + + def generate_d004_from_a003(self, a003_report: Dict[str, Any], cluster: str = "local", + node_name: str = "node-01") -> Dict[str, Any]: + """ + Generate D004 report by filtering A003 data for pg_stat_statements settings. + + Args: + a003_report: Full A003 report containing all settings + cluster: Cluster name (for status checks) + node_name: Node name + + Returns: + D004 report dictionary + """ + print("Generating D004 from A003 data...") + + # Filter A003 settings for D004-relevant settings + pgstat_data = self.filter_a003_settings(a003_report, self.D004_SETTINGS) + + # Check extension status (still needs direct queries) + kcache_status = self._check_pg_stat_kcache_status(cluster, node_name) + pgss_status = self._check_pg_stat_statements_status(cluster, node_name) + + # Extract postgres version from A003 + postgres_version = self.extract_postgres_version_from_a003(a003_report, node_name) + + return self.format_report_data( + "D004", + { + "settings": pgstat_data, + "pg_stat_statements_status": pgss_status, + "pg_stat_kcache_status": kcache_status, + }, + node_name, + postgres_version=postgres_version, + ) + + def generate_f001_from_a003(self, a003_report: Dict[str, Any], node_name: str = "node-01") -> Dict[str, Any]: + """ + Generate F001 report by filtering A003 data for autovacuum settings. + + Args: + a003_report: Full A003 report containing all settings + node_name: Node name + + Returns: + F001 report dictionary + """ + print("Generating F001 from A003 data...") + + # Filter A003 settings for F001-relevant settings + autovacuum_data = self.filter_a003_settings(a003_report, self.F001_SETTINGS) + + # Extract postgres version from A003 + postgres_version = self.extract_postgres_version_from_a003(a003_report, node_name) + + return self.format_report_data("F001", autovacuum_data, node_name, postgres_version=postgres_version) + + def generate_g001_from_a003(self, a003_report: Dict[str, Any], node_name: str = "node-01") -> Dict[str, Any]: + """ + Generate G001 report by filtering A003 data for memory settings. + + Args: + a003_report: Full A003 report containing all settings + node_name: Node name + + Returns: + G001 report dictionary with memory analysis + """ + print("Generating G001 from A003 data...") + + # Filter A003 settings for G001-relevant settings + memory_data = self.filter_a003_settings(a003_report, self.G001_SETTINGS) + + # Calculate memory analysis + memory_analysis = self._analyze_memory_settings(memory_data) + + # Extract postgres version from A003 + postgres_version = self.extract_postgres_version_from_a003(a003_report, node_name) + + return self.format_report_data( + "G001", + { + "settings": memory_data, + "analysis": memory_analysis, + }, + node_name, + postgres_version=postgres_version, + ) + + def report_to_markdown(self, report: Dict[str, Any]) -> str: + """ + Convert a report dictionary to markdown format. + + Args: + report: Report dictionary (D004, F001, G001, etc.) + + Returns: + Markdown string representation of the report + """ + check_id = report.get('checkId', 'Unknown') + check_title = report.get('checkTitle', 'Unknown Report') + timestamp = report.get('timestamptz', '') + nodes = report.get('nodes', {}) + results = report.get('results', {}) + + lines = [ + f"# {check_id}: {check_title}", + "", + f"**Generated:** {timestamp}", + "", + ] + + # Add postgres version if available + for node_name, node_data in results.items(): + pg_version = node_data.get('postgres_version', {}) + if pg_version: + version_str = pg_version.get('version', 'Unknown') + lines.append(f"**PostgreSQL Version:** {version_str}") + lines.append("") + break + + # Add node info + primary = nodes.get('primary', '') + standbys = nodes.get('standbys', []) + if primary: + lines.append(f"**Primary Node:** {primary}") + if standbys: + lines.append(f"**Standby Nodes:** {', '.join(standbys)}") + lines.append("") + + # Generate content based on check type + if check_id == 'D004': + lines.extend(self._d004_to_markdown_content(results)) + elif check_id == 'F001': + lines.extend(self._f001_to_markdown_content(results)) + elif check_id == 'G001': + lines.extend(self._g001_to_markdown_content(results)) + else: + lines.extend(self._generic_settings_to_markdown_content(results)) + + return '\n'.join(lines) + + def _settings_table_markdown(self, settings: Dict[str, Any]) -> List[str]: + """Generate markdown table for settings.""" + if not settings: + return ["*No settings data available.*", ""] + + lines = [ + "| Setting | Value | Category | Context |", + "|---------|-------|----------|---------|", + ] + for name, info in sorted(settings.items()): + value = info.get('pretty_value', info.get('setting', '')) + category = info.get('category', '') + context = info.get('context', '') + lines.append(f"| `{name}` | {value} | {category} | {context} |") + lines.append("") + return lines + + def _d004_to_markdown_content(self, results: Dict[str, Any]) -> List[str]: + """Generate D004-specific markdown content.""" + lines = [] + + for node_name, node_data in results.items(): + data = node_data.get('data', {}) + + lines.append(f"## Node: {node_name}") + lines.append("") + + # Settings table + lines.append("### pg_stat_statements and Related Settings") + lines.append("") + settings = data.get('settings', data) + lines.extend(self._settings_table_markdown(settings)) + + # pg_stat_statements status + pgss_status = data.get('pg_stat_statements_status', {}) + if pgss_status: + lines.append("### pg_stat_statements Status") + lines.append("") + available = "✅ Available" if pgss_status.get('extension_available') else "❌ Not Available" + lines.append(f"- **Extension:** {available}") + lines.append(f"- **Metrics Count:** {pgss_status.get('metrics_count', 0)}") + lines.append(f"- **Total Calls:** {pgss_status.get('total_calls', 0)}") + lines.append("") + + # pg_stat_kcache status + kcache_status = data.get('pg_stat_kcache_status', {}) + if kcache_status: + lines.append("### pg_stat_kcache Status") + lines.append("") + available = "✅ Available" if kcache_status.get('extension_available') else "❌ Not Available" + lines.append(f"- **Extension:** {available}") + lines.append(f"- **Metrics Count:** {kcache_status.get('metrics_count', 0)}") + lines.append("") + + return lines + + def _f001_to_markdown_content(self, results: Dict[str, Any]) -> List[str]: + """Generate F001-specific markdown content.""" + lines = [] + + for node_name, node_data in results.items(): + data = node_data.get('data', {}) + + lines.append(f"## Node: {node_name}") + lines.append("") + lines.append("### Autovacuum Settings") + lines.append("") + lines.extend(self._settings_table_markdown(data)) + + return lines + + def _g001_to_markdown_content(self, results: Dict[str, Any]) -> List[str]: + """Generate G001-specific markdown content.""" + lines = [] + + for node_name, node_data in results.items(): + data = node_data.get('data', {}) + + lines.append(f"## Node: {node_name}") + lines.append("") + + # Settings table + lines.append("### Memory-Related Settings") + lines.append("") + settings = data.get('settings', {}) + lines.extend(self._settings_table_markdown(settings)) + + # Memory analysis + analysis = data.get('analysis', {}) + mem_usage = analysis.get('estimated_total_memory_usage', {}) + if mem_usage: + lines.append("### Memory Usage Analysis") + lines.append("") + lines.append("| Metric | Value |") + lines.append("|--------|-------|") + + if mem_usage.get('shared_buffers_pretty'): + lines.append(f"| Shared Buffers | {mem_usage.get('shared_buffers_pretty')} |") + if mem_usage.get('wal_buffers_pretty'): + lines.append(f"| WAL Buffers | {mem_usage.get('wal_buffers_pretty')} |") + if mem_usage.get('shared_memory_total_pretty'): + lines.append(f"| Shared Memory Total | {mem_usage.get('shared_memory_total_pretty')} |") + if mem_usage.get('work_mem_per_connection_pretty'): + lines.append(f"| Work Mem (per connection) | {mem_usage.get('work_mem_per_connection_pretty')} |") + if mem_usage.get('max_work_mem_usage_pretty'): + lines.append(f"| Max Work Mem (all connections) | {mem_usage.get('max_work_mem_usage_pretty')} |") + if mem_usage.get('maintenance_work_mem_pretty'): + lines.append(f"| Maintenance Work Mem | {mem_usage.get('maintenance_work_mem_pretty')} |") + if mem_usage.get('effective_cache_size_pretty'): + lines.append(f"| Effective Cache Size | {mem_usage.get('effective_cache_size_pretty')} |") + + lines.append("") + + return lines + + def _generic_settings_to_markdown_content(self, results: Dict[str, Any]) -> List[str]: + """Generate generic markdown content for settings-based reports.""" + lines = [] + + for node_name, node_data in results.items(): + data = node_data.get('data', {}) + + lines.append(f"## Node: {node_name}") + lines.append("") + lines.extend(self._settings_table_markdown(data)) + + return lines + def get_check_title(self, check_id: str) -> str: """ Get the human-readable title for a check ID. - + Args: check_id: The check identifier (e.g., "H004") - + Returns: Human-readable title for the check """ @@ -2229,17 +2646,14 @@ def generate_all_reports(self, cluster: str = "local", node_name: str = None, co nodes_to_process = [node_name] all_nodes = {"primary": node_name, "standbys": []} - # Generate each report type - report_types = [ + # Reports that don't depend on A003 (generate first) + independent_report_types = [ ('A002', self.generate_a002_version_report), ('A003', self.generate_a003_settings_report), ('A004', self.generate_a004_cluster_report), ('A007', self.generate_a007_altered_settings_report), - ('D004', self.generate_d004_pgstat_settings_report), - ('F001', self.generate_f001_autovacuum_settings_report), ('F004', self.generate_f004_heap_bloat_report), ('F005', self.generate_f005_btree_bloat_report), - ('G001', self.generate_g001_memory_settings_report), ('H001', self.generate_h001_invalid_indexes_report), ('H002', self.generate_h002_unused_indexes_report), ('H004', self.generate_h004_redundant_indexes_report), @@ -2247,7 +2661,7 @@ def generate_all_reports(self, cluster: str = "local", node_name: str = None, co ('K003', self.generate_k003_top_queries_report), ] - for check_id, report_func in report_types: + for check_id, report_func in independent_report_types: if len(nodes_to_process) == 1: # Single node - generate report normally reports[check_id] = report_func(cluster, nodes_to_process[0]) @@ -2260,15 +2674,70 @@ def generate_all_reports(self, cluster: str = "local", node_name: str = None, co # Extract the data from the node report if 'results' in node_report and node in node_report['results']: combined_results[node] = node_report['results'][node] - + # Create combined report with all nodes reports[check_id] = self.format_report_data( - check_id, - combined_results, + check_id, + combined_results, all_nodes["primary"] if all_nodes["primary"] else nodes_to_process[0], all_nodes ) + # Generate D004, F001, G001 from A003 data (if A003 was generated successfully) + a003_report = reports.get('A003') + if a003_report: + # Reports derived from A003 + a003_derived_reports = [ + ('D004', lambda c, n: self.generate_d004_from_a003(a003_report, c, n)), + ('F001', lambda c, n: self.generate_f001_from_a003(a003_report, n)), + ('G001', lambda c, n: self.generate_g001_from_a003(a003_report, n)), + ] + + for check_id, report_func in a003_derived_reports: + if len(nodes_to_process) == 1: + reports[check_id] = report_func(cluster, nodes_to_process[0]) + else: + # For multi-node, use the first node as reference + # (A003 data already contains all nodes) + combined_results = {} + for node in nodes_to_process: + print(f"Generating {check_id} report for node {node} from A003...") + node_report = report_func(cluster, node) + if 'results' in node_report and node in node_report['results']: + combined_results[node] = node_report['results'][node] + + reports[check_id] = self.format_report_data( + check_id, + combined_results, + all_nodes["primary"] if all_nodes["primary"] else nodes_to_process[0], + all_nodes + ) + else: + # Fallback to direct generation if A003 failed + print("Warning: A003 report not available, generating D004/F001/G001 directly") + fallback_report_types = [ + ('D004', self.generate_d004_pgstat_settings_report), + ('F001', self.generate_f001_autovacuum_settings_report), + ('G001', self.generate_g001_memory_settings_report), + ] + for check_id, report_func in fallback_report_types: + if len(nodes_to_process) == 1: + reports[check_id] = report_func(cluster, nodes_to_process[0]) + else: + combined_results = {} + for node in nodes_to_process: + print(f"Generating {check_id} report for node {node}...") + node_report = report_func(cluster, node) + if 'results' in node_report and node in node_report['results']: + combined_results[node] = node_report['results'][node] + + reports[check_id] = self.format_report_data( + check_id, + combined_results, + all_nodes["primary"] if all_nodes["primary"] else nodes_to_process[0], + all_nodes + ) + return reports def get_all_clusters(self) -> List[str]: @@ -2566,6 +3035,8 @@ def main(): help='Specific check ID to generate (default: ALL)') parser.add_argument('--output', default='-', help='Output file (default: stdout)') + parser.add_argument('--format', choices=['json', 'markdown', 'both'], default='json', + help='Output format: json, markdown, or both (default: json)') parser.add_argument('--api-url', default='https://postgres.ai/api/general') parser.add_argument('--token', default='') parser.add_argument('--project-name', default='project-name', @@ -2623,16 +3094,28 @@ def main(): report_id = generator.create_report(args.api_url, args.token, project_name, args.epoch) reports = generator.generate_all_reports(cluster, args.node_name, combine_nodes) - + # Save reports with cluster name prefix - for report in reports: - output_filename = f"{cluster}_{report}.json" if len(clusters_to_process) > 1 else f"{report}.json" - with open(output_filename, "w") as f: - json.dump(reports[report], f, indent=2) - print(f"Generated report: {output_filename}") - if not args.no_upload: - generator.upload_report_file(args.api_url, args.token, report_id, output_filename) - + for check_id in reports: + report_data = reports[check_id] + base_name = f"{cluster}_{check_id}" if len(clusters_to_process) > 1 else check_id + + # Write JSON if format is json or both + if args.format in ('json', 'both'): + json_filename = f"{base_name}.json" + with open(json_filename, "w") as f: + json.dump(report_data, f, indent=2) + print(f"Generated JSON report: {json_filename}") + if not args.no_upload: + generator.upload_report_file(args.api_url, args.token, report_id, json_filename) + + # Write Markdown if format is markdown or both + if args.format in ('markdown', 'both'): + md_filename = f"{base_name}.md" + with open(md_filename, 'w') as f: + f.write(generator.report_to_markdown(report_data)) + print(f"Generated Markdown report: {md_filename}") + if args.output == '-': pass elif len(clusters_to_process) == 1: @@ -2650,7 +3133,13 @@ def main(): # Generate specific report - use node_name or default if args.node_name is None: args.node_name = "node-01" - + + # For D004, F001, G001 - generate A003 first and derive from it + a003_report = None + if args.check_id in ('D004', 'F001', 'G001'): + print(f"Generating A003 first for {args.check_id}...") + a003_report = generator.generate_a003_settings_report(cluster, args.node_name) + if args.check_id == 'A002': report = generator.generate_a002_version_report(cluster, args.node_name) elif args.check_id == 'A003': @@ -2660,15 +3149,24 @@ def main(): elif args.check_id == 'A007': report = generator.generate_a007_altered_settings_report(cluster, args.node_name) elif args.check_id == 'D004': - report = generator.generate_d004_pgstat_settings_report(cluster, args.node_name) + if a003_report: + report = generator.generate_d004_from_a003(a003_report, cluster, args.node_name) + else: + report = generator.generate_d004_pgstat_settings_report(cluster, args.node_name) elif args.check_id == 'F001': - report = generator.generate_f001_autovacuum_settings_report(cluster, args.node_name) + if a003_report: + report = generator.generate_f001_from_a003(a003_report, args.node_name) + else: + report = generator.generate_f001_autovacuum_settings_report(cluster, args.node_name) elif args.check_id == 'F004': report = generator.generate_f004_heap_bloat_report(cluster, args.node_name) elif args.check_id == 'F005': report = generator.generate_f005_btree_bloat_report(cluster, args.node_name) elif args.check_id == 'G001': - report = generator.generate_g001_memory_settings_report(cluster, args.node_name) + if a003_report: + report = generator.generate_g001_from_a003(a003_report, args.node_name) + else: + report = generator.generate_g001_memory_settings_report(cluster, args.node_name) elif args.check_id == 'H001': report = generator.generate_h001_invalid_indexes_report(cluster, args.node_name) elif args.check_id == 'H002': @@ -2680,18 +3178,37 @@ def main(): elif args.check_id == 'K003': report = generator.generate_k003_top_queries_report(cluster, args.node_name) - output_filename = f"{cluster}_{args.check_id}.json" if len(clusters_to_process) > 1 else args.output - + # Determine output filenames + base_name = f"{cluster}_{args.check_id}" if len(clusters_to_process) > 1 else args.check_id + json_filename = f"{base_name}.json" if args.output == '-' else args.output + md_filename = f"{base_name}.md" + + # Output based on format if args.output == '-' and len(clusters_to_process) == 1: - print(json.dumps(report, indent=2)) + if args.format == 'markdown': + print(generator.report_to_markdown(report)) + elif args.format == 'both': + print(json.dumps(report, indent=2)) + print("\n--- Markdown ---\n") + print(generator.report_to_markdown(report)) + else: + print(json.dumps(report, indent=2)) else: - with open(output_filename, 'w') as f: - json.dump(report, f, indent=2) - print(f"Report written to {output_filename}") - if not args.no_upload: - project_name = args.project_name if args.project_name != 'project-name' else cluster - report_id = generator.create_report(args.api_url, args.token, project_name, args.epoch) - generator.upload_report_file(args.api_url, args.token, report_id, output_filename) + # Write JSON if format is json or both + if args.format in ('json', 'both'): + with open(json_filename, 'w') as f: + json.dump(report, f, indent=2) + print(f"JSON report written to {json_filename}") + if not args.no_upload: + project_name = args.project_name if args.project_name != 'project-name' else cluster + report_id = generator.create_report(args.api_url, args.token, project_name, args.epoch) + generator.upload_report_file(args.api_url, args.token, report_id, json_filename) + + # Write Markdown if format is markdown or both + if args.format in ('markdown', 'both'): + with open(md_filename, 'w') as f: + f.write(generator.report_to_markdown(report)) + print(f"Markdown report written to {md_filename}") except Exception as e: print(f"Error generating reports: {e}") raise e diff --git a/tests/reporter/test_generators_unit.py b/tests/reporter/test_generators_unit.py index 37463ca..7e6bdff 100644 --- a/tests/reporter/test_generators_unit.py +++ b/tests/reporter/test_generators_unit.py @@ -864,16 +864,14 @@ def _(*args, **kwargs): return _ - builders = [ + # Independent builders (not derived from A003) + independent_builders = [ "generate_a002_version_report", "generate_a003_settings_report", "generate_a004_cluster_report", "generate_a007_altered_settings_report", - "generate_d004_pgstat_settings_report", - "generate_f001_autovacuum_settings_report", "generate_f004_heap_bloat_report", "generate_f005_btree_bloat_report", - "generate_g001_memory_settings_report", "generate_h001_invalid_indexes_report", "generate_h002_unused_indexes_report", "generate_h004_redundant_indexes_report", @@ -881,13 +879,28 @@ def _(*args, **kwargs): "generate_k003_top_queries_report", ] - for name in builders: + # Builders derived from A003 + a003_derived_builders = [ + "generate_d004_from_a003", + "generate_f001_from_a003", + "generate_g001_from_a003", + ] + + for name in independent_builders: + monkeypatch.setattr(generator, name, stub(name)) + + for name in a003_derived_builders: monkeypatch.setattr(generator, name, stub(name)) reports = generator.generate_all_reports("local", "node-1") - assert set(reports.keys()) == {code.split("_")[1].upper() for code in builders} - assert set(called) == set(builders) + # All report types should be generated + expected_report_codes = {'A002', 'A003', 'A004', 'A007', 'D004', 'F001', 'F004', 'F005', 'G001', 'H001', 'H002', 'H004', 'K001', 'K003'} + assert set(reports.keys()) == expected_report_codes + + # All builders should be called + all_builders = independent_builders + a003_derived_builders + assert set(called) == set(all_builders) @pytest.mark.unit