DT-66: Implementation of table data fetching and filtering

api/routes.ts

@@ -24,6 +24,7 @@ import {
 } from './click-house-client.ts'
 import { decryptMessage, encryptMessage } from './user.ts'
 import { log } from './lib/log.ts'
+import { type ColumnInfo, fetchTablesData } from './sql.ts'
 
 const withUserSession = ({ user }: RequestContext) => {
   if (!user) throw Error('Missing user session')
@@ -417,6 +418,57 @@ const defs = {
     output: ARR(LogSchema, 'List of logs'),
     description: 'Get logs from ClickHouse',
   }),
+  'POST/api/deployment/table/data': route({
+    fn: (_, { deployment, table, ...input }) => {
+      const dep = DeploymentsCollection.get(deployment)
+      if (!dep) {
+        throw respond.NotFound({ message: 'Deployment not found' })
+      }
+
+      if (!dep?.databaseEnabled) {
+        throw respond.BadRequest({
+          message: 'Database not enabled for deployment',
+        })
+      }
+
+      const schema = DatabaseSchemasCollection.get(deployment)
+      if (!schema) throw respond.NotFound({ message: 'Schema not cached yet' })
+      const tableDef = schema.tables.find((t) => t.table === table)
+      if (!tableDef) {
+        throw respond.NotFound({ message: 'Table not found in schema' })
+      }
+
+      return fetchTablesData(
+        { ...input, deployment: dep, table },
+        tableDef.columnsMap as unknown as Map<string, ColumnInfo>,
+      )
+    },
+    input: OBJ({
+      deployment: STR("The deployment's URL"),
+      table: STR('The table name'),
+      filter: ARR(
+        OBJ({
+          key: STR('The column to filter by'),
+          comparator: LIST(
+            ['=', '!=', '<', '<=', '>', '>=', 'LIKE', 'ILIKE'],
+            'The comparison operator',
+          ),
+          value: STR('The value to filter by'),
+        }),
+        'The filtering criteria',
+      ),
+      sort: ARR(
+        OBJ({
+          key: STR('The column to sort by'),
+          order: LIST(['ASC', 'DESC'], 'The sort order (ASC or DESC)'),
+        }),
+        'The sorting criteria',
+      ),
+      limit: STR('The maximum number of rows to return'),
+      offset: STR('The number of rows to skip'),
+      search: STR('The search term to filter by'),
+    }),
+  }),
 } as const
 
 export type RouteDefinitions = typeof defs

api/schema.ts

@@ -52,6 +52,7 @@ export const DatabaseSchemaDef = OBJ({
       type: STR(),
       ordinal: NUM(),
     })),
+    columnsMap: optional(OBJ({})),
     schema: optional(STR()),
     table: STR(),
   })),

api/sql.ts

@@ -1,4 +1,8 @@
-import { DatabaseSchemasCollection, DeploymentsCollection } from './schema.ts'
+import {
+  DatabaseSchemasCollection,
+  Deployment,
+  DeploymentsCollection,
+} from './schema.ts'
 import { DB_SCHEMA_REFRESH_MS } from './lib/env.ts'
 import { log } from './lib/log.ts'
 
@@ -59,11 +63,12 @@ async function fetchSchema(endpoint: string, token: string, dialect: string) {
   return await runSQL(endpoint, token, sql)
 }
 
-type ColumnInfo = { name: string; type: string; ordinal: number }
+export type ColumnInfo = { name: string; type: string; ordinal: number }
 type TableInfo = {
   schema: string | undefined
   table: string
   columns: ColumnInfo[]
+  columnsMap: Map<string, ColumnInfo>
 }
 
 export async function refreshOneSchema(
@@ -80,7 +85,9 @@ export async function refreshOneSchema(
       const table = r.table_name as string
       if (!table) continue
       const key = (schema ? schema + '.' : '') + table
-      if (!tableMap.has(key)) tableMap.set(key, { schema, table, columns: [] })
+      if (!tableMap.has(key)) {
+        tableMap.set(key, { schema, table, columns: [], columnsMap: new Map() })
+      }
       tableMap.get(key)!.columns.push({
         name: String(r.column_name),
         type: String(r.data_type || ''),
@@ -90,6 +97,10 @@ export async function refreshOneSchema(
     const tables = [...tableMap.values()].map((t) => ({
       ...t,
       columns: t.columns.sort((a, b) => a.ordinal - b.ordinal),
+      columnsMap: t.columns.reduce((map, col) => {
+        map.set(col.name, col)
+        return map
+      }, new Map<string, ColumnInfo>()),
     }))
     const payload = {
       deploymentUrl: dep.url,
@@ -129,3 +140,86 @@ export function startSchemaRefreshLoop() {
   }, DB_SCHEMA_REFRESH_MS) as unknown as number
   log.info('schema-refresh-loop-started', { everyMs: DB_SCHEMA_REFRESH_MS })
 }
+
+type FetchTablesParams = {
+  deployment: Deployment
+  table: string
+  filter: { key: string; comparator: string; value: string }[]
+  sort: { key: string; order: 'ASC' | 'DESC' }[]
+  limit: string
+  offset: string
+  search: string
+}
+
+const constructWhereClause = (
+  params: FetchTablesParams,
+  columnsMap: Map<string, ColumnInfo>,
+) => {
+  const whereClauses: string[] = []
+  if (params.filter.length) {
+    for (const filter of params.filter) {
+      const { key, comparator, value } = filter
+      const column = columnsMap.get(key)
+      if (!column) {
+        throw Error(`Invalid filter column: ${key}`)
+      }
+      const safeValue = value.replace(/'/g, "''")
+      whereClauses.push(`${key} ${comparator} '${safeValue}'`)
+    }
+  }
+  if (params.search) {
+    const searchClauses = columnsMap.values().map((col) => {
+      return `${col.name} LIKE '%${params.search.replace(/'/g, "''")}%'`
+    }).toArray()
+    if (searchClauses.length) {
+      whereClauses.push(`(${searchClauses.join(' OR ')})`)
+    }
+  }
+  return whereClauses.length ? 'WHERE ' + whereClauses.join(' AND ') : ''
+}
+
+const constructOrderByClause = (
+  params: FetchTablesParams,
+  columnsMap: Map<string, ColumnInfo>,
+) => {
+  if (!params.sort.length) return ''
+  const orderClauses: string[] = []
+  for (const sort of params.sort) {
+    const { key, order } = sort
+    const column = columnsMap.get(key)
+    if (!column) {
+      throw Error(`Invalid sort column: ${key}`)
+    }
+    orderClauses.push(`${key} ${order}`)
+  }
+  return orderClauses.length ? 'ORDER BY ' + orderClauses.join(', ') : ''
+}
+
+export const fetchTablesData = (
+  params: FetchTablesParams,
+  columnsMap: Map<string, ColumnInfo>,
+) => {
+  const { sqlEndpoint, sqlToken } = params.deployment
+  if (!sqlToken || !sqlEndpoint) {
+    throw Error('Missing SQL endpoint or token')
+  }
+  const whereClause = constructWhereClause(params, columnsMap)
+  const orderByClause = constructOrderByClause(params, columnsMap)
+
+  let limitOffsetClause = ''
+  const limit = Math.floor(Number(params.limit))
+
+  if (params.limit && limit > 0) {
+    limitOffsetClause += `LIMIT ${limit}`
+
+    const offset = Math.floor(Number(params.offset))
+    if (params.offset && offset >= 0) {
+      limitOffsetClause += ` OFFSET ${offset}`
+    }
+  }
+
+  const query =
+    `SELECT * FROM ${params.table} ${whereClause} ${orderByClause} ${limitOffsetClause}`
+
+  return runSQL(sqlEndpoint, sqlToken, query)
+}

web/components/Filtre.tsx

@@ -5,7 +5,7 @@ type FilterRow = { idx: number; key: string; op: string; value: string }
 
 const filterOperators = ['eq', 'neq', 'gt', 'gte', 'lt', 'lte', 'like'] as const
 
-function parseFilters(prefix: string): FilterRow[] {
+export function parseFilters(prefix: string): FilterRow[] {
   const data = url.getAll(`f${prefix}`)
   const rows: FilterRow[] = []
   for (let i = 0; i < data.length; i++) {
@@ -61,7 +61,7 @@ function removeFilter(prefix: string, idx: number) {
 // --- Sort (URL) -------------------------------------------------------------
 type SortRow = { idx: number; key: string; dir: 'asc' | 'desc' }
 
-function parseSort(prefix: string): SortRow[] {
+export function parseSort(prefix: string): SortRow[] {
   const data = url.getAll(`s${prefix}`)
   const rows: SortRow[] = []
   for (let i = 0; i < data.length; i++) {