diff --git a/api/routes.ts b/api/routes.ts index e7b20a2..99133ff 100644 --- a/api/routes.ts +++ b/api/routes.ts @@ -24,6 +24,7 @@ import { } from './click-house-client.ts' import { decryptMessage, encryptMessage } from './user.ts' import { log } from './lib/log.ts' +import { type ColumnInfo, fetchTablesData } from './sql.ts' const withUserSession = ({ user }: RequestContext) => { if (!user) throw Error('Missing user session') @@ -417,6 +418,57 @@ const defs = { output: ARR(LogSchema, 'List of logs'), description: 'Get logs from ClickHouse', }), + 'POST/api/deployment/table/data': route({ + fn: (_, { deployment, table, ...input }) => { + const dep = DeploymentsCollection.get(deployment) + if (!dep) { + throw respond.NotFound({ message: 'Deployment not found' }) + } + + if (!dep?.databaseEnabled) { + throw respond.BadRequest({ + message: 'Database not enabled for deployment', + }) + } + + const schema = DatabaseSchemasCollection.get(deployment) + if (!schema) throw respond.NotFound({ message: 'Schema not cached yet' }) + const tableDef = schema.tables.find((t) => t.table === table) + if (!tableDef) { + throw respond.NotFound({ message: 'Table not found in schema' }) + } + + return fetchTablesData( + { ...input, deployment: dep, table }, + tableDef.columnsMap as unknown as Map, + ) + }, + input: OBJ({ + deployment: STR("The deployment's URL"), + table: STR('The table name'), + filter: ARR( + OBJ({ + key: STR('The column to filter by'), + comparator: LIST( + ['=', '!=', '<', '<=', '>', '>=', 'LIKE', 'ILIKE'], + 'The comparison operator', + ), + value: STR('The value to filter by'), + }), + 'The filtering criteria', + ), + sort: ARR( + OBJ({ + key: STR('The column to sort by'), + order: LIST(['ASC', 'DESC'], 'The sort order (ASC or DESC)'), + }), + 'The sorting criteria', + ), + limit: STR('The maximum number of rows to return'), + offset: STR('The number of rows to skip'), + search: STR('The search term to filter by'), + }), + }), } as const export type RouteDefinitions = typeof defs diff --git a/api/schema.ts b/api/schema.ts index 959ad69..106ed6f 100644 --- a/api/schema.ts +++ b/api/schema.ts @@ -52,6 +52,7 @@ export const DatabaseSchemaDef = OBJ({ type: STR(), ordinal: NUM(), })), + columnsMap: optional(OBJ({})), schema: optional(STR()), table: STR(), })), diff --git a/api/sql.ts b/api/sql.ts index f58724c..4b98253 100644 --- a/api/sql.ts +++ b/api/sql.ts @@ -1,4 +1,8 @@ -import { DatabaseSchemasCollection, DeploymentsCollection } from './schema.ts' +import { + DatabaseSchemasCollection, + Deployment, + DeploymentsCollection, +} from './schema.ts' import { DB_SCHEMA_REFRESH_MS } from './lib/env.ts' import { log } from './lib/log.ts' @@ -59,11 +63,12 @@ async function fetchSchema(endpoint: string, token: string, dialect: string) { return await runSQL(endpoint, token, sql) } -type ColumnInfo = { name: string; type: string; ordinal: number } +export type ColumnInfo = { name: string; type: string; ordinal: number } type TableInfo = { schema: string | undefined table: string columns: ColumnInfo[] + columnsMap: Map } export async function refreshOneSchema( @@ -80,7 +85,9 @@ export async function refreshOneSchema( const table = r.table_name as string if (!table) continue const key = (schema ? schema + '.' : '') + table - if (!tableMap.has(key)) tableMap.set(key, { schema, table, columns: [] }) + if (!tableMap.has(key)) { + tableMap.set(key, { schema, table, columns: [], columnsMap: new Map() }) + } tableMap.get(key)!.columns.push({ name: String(r.column_name), type: String(r.data_type || ''), @@ -90,6 +97,10 @@ export async function refreshOneSchema( const tables = [...tableMap.values()].map((t) => ({ ...t, columns: t.columns.sort((a, b) => a.ordinal - b.ordinal), + columnsMap: t.columns.reduce((map, col) => { + map.set(col.name, col) + return map + }, new Map()), })) const payload = { deploymentUrl: dep.url, @@ -129,3 +140,86 @@ export function startSchemaRefreshLoop() { }, DB_SCHEMA_REFRESH_MS) as unknown as number log.info('schema-refresh-loop-started', { everyMs: DB_SCHEMA_REFRESH_MS }) } + +type FetchTablesParams = { + deployment: Deployment + table: string + filter: { key: string; comparator: string; value: string }[] + sort: { key: string; order: 'ASC' | 'DESC' }[] + limit: string + offset: string + search: string +} + +const constructWhereClause = ( + params: FetchTablesParams, + columnsMap: Map, +) => { + const whereClauses: string[] = [] + if (params.filter.length) { + for (const filter of params.filter) { + const { key, comparator, value } = filter + const column = columnsMap.get(key) + if (!column) { + throw Error(`Invalid filter column: ${key}`) + } + const safeValue = value.replace(/'/g, "''") + whereClauses.push(`${key} ${comparator} '${safeValue}'`) + } + } + if (params.search) { + const searchClauses = columnsMap.values().map((col) => { + return `${col.name} LIKE '%${params.search.replace(/'/g, "''")}%'` + }).toArray() + if (searchClauses.length) { + whereClauses.push(`(${searchClauses.join(' OR ')})`) + } + } + return whereClauses.length ? 'WHERE ' + whereClauses.join(' AND ') : '' +} + +const constructOrderByClause = ( + params: FetchTablesParams, + columnsMap: Map, +) => { + if (!params.sort.length) return '' + const orderClauses: string[] = [] + for (const sort of params.sort) { + const { key, order } = sort + const column = columnsMap.get(key) + if (!column) { + throw Error(`Invalid sort column: ${key}`) + } + orderClauses.push(`${key} ${order}`) + } + return orderClauses.length ? 'ORDER BY ' + orderClauses.join(', ') : '' +} + +export const fetchTablesData = ( + params: FetchTablesParams, + columnsMap: Map, +) => { + const { sqlEndpoint, sqlToken } = params.deployment + if (!sqlToken || !sqlEndpoint) { + throw Error('Missing SQL endpoint or token') + } + const whereClause = constructWhereClause(params, columnsMap) + const orderByClause = constructOrderByClause(params, columnsMap) + + let limitOffsetClause = '' + const limit = Math.floor(Number(params.limit)) + + if (params.limit && limit > 0) { + limitOffsetClause += `LIMIT ${limit}` + + const offset = Math.floor(Number(params.offset)) + if (params.offset && offset >= 0) { + limitOffsetClause += ` OFFSET ${offset}` + } + } + + const query = + `SELECT * FROM ${params.table} ${whereClause} ${orderByClause} ${limitOffsetClause}` + + return runSQL(sqlEndpoint, sqlToken, query) +} diff --git a/web/components/Filtre.tsx b/web/components/Filtre.tsx index 2d5d254..85aeea8 100644 --- a/web/components/Filtre.tsx +++ b/web/components/Filtre.tsx @@ -5,7 +5,7 @@ type FilterRow = { idx: number; key: string; op: string; value: string } const filterOperators = ['eq', 'neq', 'gt', 'gte', 'lt', 'lte', 'like'] as const -function parseFilters(prefix: string): FilterRow[] { +export function parseFilters(prefix: string): FilterRow[] { const data = url.getAll(`f${prefix}`) const rows: FilterRow[] = [] for (let i = 0; i < data.length; i++) { @@ -61,7 +61,7 @@ function removeFilter(prefix: string, idx: number) { // --- Sort (URL) ------------------------------------------------------------- type SortRow = { idx: number; key: string; dir: 'asc' | 'desc' } -function parseSort(prefix: string): SortRow[] { +export function parseSort(prefix: string): SortRow[] { const data = url.getAll(`s${prefix}`) const rows: SortRow[] = [] for (let i = 0; i < data.length; i++) { diff --git a/web/pages/DeploymentPage.tsx b/web/pages/DeploymentPage.tsx index 651a2a1..06bc179 100644 --- a/web/pages/DeploymentPage.tsx +++ b/web/pages/DeploymentPage.tsx @@ -3,7 +3,10 @@ import { AlertCircle, AlertTriangle, Bug, + ChevronDown, + ChevronRight, Clock, + Columns, Download, Eye, FileText, @@ -18,10 +21,74 @@ import { XCircle, } from 'lucide-preact' import { deployments, sidebarItems } from './ProjectPage.tsx' -import { FilterMenu, SortMenu } from '../components/Filtre.tsx' +import { + FilterMenu, + parseFilters, + parseSort, + SortMenu, +} from '../components/Filtre.tsx' +import { effect } from '@preact/signals' +import { api } from '../lib/api.ts' type AnyRecord = Record +// API signals for schema and table data +const schema = api['GET/api/deployment/schema'].signal() + +const comparators = { + 'eq': '=', + 'neq': '!=', + 'lt': '<', + 'lte': '<=', + 'gt': '>', + 'gte': '>=', + 'like': 'LIKE', + 'ilike': 'ILIKE', +} as const + +const tableData = api['POST/api/deployment/table/data'].signal() +type Order = 'ASC' | 'DESC' + +// Effect to fetch schema when deployment URL changes +effect(() => { + const dep = url.params.dep + if (dep) { + schema.fetch({ url: dep }) + } +}) + +// Effect to fetch table data when filters, sort, or search change +effect(() => { + const { dep, tab, table, qt } = url.params + if (dep && tab === 'tables') { + const tableName = table || schema.data?.tables?.[0]?.table + if (tableName) { + const filterRows = parseFilters('t').filter((r) => + r.key !== 'key' && r.value + ).map((r) => ({ + key: r.key, + comparator: comparators[r.op as keyof typeof comparators], + value: r.value, + })) + const sortRows = parseSort('t').filter((r) => + r.key !== 'key' && r.key && r.dir + ).map((r) => ({ + key: r.key, + order: r.dir === 'asc' ? 'ASC' : 'DESC' as Order, + })) + tableData.fetch({ + deployment: dep, + table: tableName, + filter: filterRows, + sort: sortRows, + search: qt || '', + limit: '50', + offset: '0', + }) + } + } +}) + const onRun = async () => { // TODO: call backend here } @@ -76,7 +143,7 @@ export function QueryEditor() {
- +
) @@ -107,34 +174,40 @@ const logData = [ ] export function DataTable({ - data = logData as AnyRecord[], page = 1, pageSize = 50, totalRows, }: { - data?: AnyRecord[] page?: number pageSize?: number totalRows?: number }) { + const data = url.params.tab === 'tables' ? tableData.data || [] : [] const columns = Object.keys(data[0] || {}) - const rows = data ?? [] const count = totalRows ?? rows.length const totalPages = Math.max(1, Math.ceil(count / pageSize)) return ( -
+
- + - + {columns.length > 0 ? columns.map((key) => ( - )) : } @@ -144,34 +217,48 @@ export function DataTable({ {rows.length === 0 && (
# + + # + + - {key} + + {key} No columns
- No results to display +
+ + No results to display + )} {rows.map((row, index) => ( - - + {columns.map((key, i) => { const value = (row as AnyRecord)[key] const isObj = typeof value === 'object' && value !== null + const stringValue = isObj + ? JSON.stringify(value) + : String(value ?? '') + const isTooLong = stringValue.length > 100 + return ( ) @@ -185,14 +272,28 @@ export function DataTable({
- {count} rows + + {count > 0 + ? `${count.toLocaleString()} row${count !== 1 ? 's' : ''}` + : 'No rows'} +
- Page {page} of {totalPages} +
- -
@@ -205,7 +306,10 @@ export function DataTable({ export function Header() { const item = sidebarItems[url.params.sbi || Object.keys(sidebarItems)[0]] - const dep = url.params.dep || deployments.data?.[0]?.url + const dep = url.params.dep + if (!dep && deployments.data?.length) { + navigate({ params: { dep: deployments.data[0].url } }) + } const onChangeDeployment = (e: Event) => { const v = (e.target as HTMLSelectElement).value @@ -213,7 +317,7 @@ export function Header() { } return ( -
+
@@ -225,7 +329,7 @@ export function Header() {
+
{(page - 1) * pageSize + index + 1} {isObj ? ( - - {JSON.stringify(value)} + + {stringValue} ) : ( - String(value ?? '') + + {stringValue} + )}
- {table.table} + + {isExpanded + ? ( + + ) + : ( + + )} + +
+
+
+ {table.table} +
+
+ + + {table.columns.length} +
- - {table.columns.length} -
-
- {table.columns - .sort((a, b) => a.ordinal - b.ordinal) - .map((column) => { - return ( -
- - - {column.name} - - - {column.type} - - -
- ) - })} + + + {isExpanded && ( +
+
+ Columns +
+
+ {table.columns.map((col, idx) => ( +
+ + + {col.name} + + + {col.type} + +
+ ))} +
-
- ) - })} - + )} + + ) + })} ))} - - {(!schema.tables || schema.tables.length === 0) && ( -
-
-

No tables found

- - )} )} @@ -409,9 +528,16 @@ const TabButton = ({ tabName }: { tabName: 'tables' | 'queries' | 'logs' }) => ( ) export function TabNavigation({ - activeTab, -}: { activeTab: 'tables' | 'queries' | 'logs' }) { - const filterKeyOptions = [ + activeTab = 'tables', +}: { activeTab?: 'tables' | 'queries' | 'logs' }) { + // Get column names from the currently selected table for tables tab + const selectedTableName = url.params.table || schema.data?.tables?.[0]?.table + const selectedTable = schema.data?.tables?.find((t) => + t.table === selectedTableName + ) + const tableColumnNames = selectedTable?.columns.map((c) => c.name) || [] + + const filterKeyOptions = activeTab === 'tables' ? tableColumnNames : [ 'service_name', 'service_version', 'service_instance_id', @@ -420,7 +546,7 @@ export function TabNavigation({ ] as const return ( -
+
@@ -432,7 +558,16 @@ export function TabNavigation({ {(activeTab === 'tables' || activeTab === 'logs') && ( )} {activeTab !== 'logs' && ( @@ -506,17 +641,17 @@ export function LogsViewer() {
- + {logThreads.map((header) => ( ))} - @@ -534,43 +669,59 @@ export function LogsViewer() { key={log.id} class='hover:bg-base-200/50 border-b border-base-300/50' > - - - - - -
{header} +
-
+
+
- + {safeFormatTimestamp(log.timestamp)}
+
{log.severity_text}
-
- +
+
+ {log.event_name} {log.body && ( - + {log.body} )}
@@ -625,7 +776,7 @@ const TabViews = { } const Drawer = () => ( -
+