DT-94: Add tooling system for custom data processing on DB operations #94 (#95)

* feat: implement SQL tool management page and refactor sidebar navigation into a dedicated module.

* feat: Implement server-side functions for SQL read transformation and remove the old Tools page.

* feat: Implement deployment function management with dedicated UI and API endpoints.

* feat: Introduce write transformers to the function API and add database functions for password hashing and tactic content compression.

* Remove the deployment functions feature, including its API routes, schema definitions, and associated UI components.

* feat: simplify function pipeline by removing dynamic configuration and move sidebar items to shared utility.

Author: abdotop

.gitignore

@@ -143,7 +143,10 @@ vite.config.ts.timestamp-*
 # Vite cache directory
 .vite
 
-db/
+db/*
+!db/functions/*
+!db/functions
+db/functions/test-*
 db_test/
 .vscode
 

api/lib/functions.ts

@@ -0,0 +1,191 @@
+import { batch } from '/api/lib/json_store.ts'
+import { join } from '@std/path'
+import { ensureDir } from '@std/fs'
+
+// Define the function signatures
+export type FunctionContext = {
+  deploymentUrl: string
+  projectId: string
+}
+
+export type ReadTransformer<T = unknown> = (
+  data: T,
+  ctx: FunctionContext,
+) => T | Promise<T>
+
+export type WriteTransformer<T = unknown> = (
+  table: string,
+  data: T,
+  query: string | undefined,
+  ctx: FunctionContext,
+) => T | Promise<T>
+
+export type ProjectFunctionModule = {
+  read?: ReadTransformer
+  write?: WriteTransformer
+  config?: {
+    targets?: string[]
+  }
+}
+
+export type LoadedFunction = {
+  name: string // filename
+  module: ProjectFunctionModule
+}
+
+// Map<projectSlug, List of loaded functions>
+const functionsMap = new Map<string, LoadedFunction[]>()
+let watcher: Deno.FsWatcher | null = null
+const functionsDir = './db/functions'
+
+export async function init() {
+  await ensureDir(functionsDir)
+  await loadAll()
+  startWatcher()
+}
+
+async function loadAll() {
+  console.info('Loading project functions...')
+  for await (const entry of Deno.readDir(functionsDir)) {
+    if (entry.isDirectory) {
+      await reloadProjectFunctions(entry.name)
+    }
+  }
+}
+
+async function reloadProjectFunctions(slug: string) {
+  const projectDir = join(functionsDir, slug)
+  const loaded: LoadedFunction[] = []
+
+  try {
+    await batch(5, Deno.readDir(projectDir), async (entry) => {
+      if (entry.isFile && entry.name.endsWith('.js')) {
+        const mainFile = join(projectDir, entry.name)
+        // Build a fresh import URL to bust cache
+        const importUrl = `file://${await Deno.realPath(
+          mainFile,
+        )}?t=${Date.now()}`
+        try {
+          const module = await import(importUrl)
+          // We expect a default export or specific named exports
+          const fns = module.default
+          if (fns && typeof fns === 'object') {
+            loaded.push({ name: entry.name, module: fns })
+          }
+        } catch (e) {
+          console.error(`Failed to import ${entry.name} for ${slug}:`, e)
+        }
+      }
+    })
+
+    // Sort by filename to ensure deterministic execution order
+    loaded.sort((a, b) => a.name.localeCompare(b.name))
+
+    if (loaded.length > 0) {
+      functionsMap.set(slug, loaded)
+      console.info(`Loaded ${loaded.length} functions for project: ${slug}`)
+    } else {
+      functionsMap.delete(slug)
+    }
+  } catch (err) {
+    if (!(err instanceof Deno.errors.NotFound)) {
+      console.error(`Failed to load functions for ${slug}:`, err)
+    }
+    functionsMap.delete(slug)
+  }
+}
+
+function startWatcher() {
+  if (watcher) return
+  console.info(`Starting function watcher on ${functionsDir}`)
+  watcher = Deno.watchFs(functionsDir, { recursive: true }) // Process events
+  ;(async () => {
+    for await (const event of watcher!) {
+      if (!['modify', 'create', 'remove', 'rename'].includes(event.kind)) {
+        continue
+      }
+      for (const path of event.paths) {
+        if (!path.endsWith('.js')) continue
+        const parts = path.split('/')
+        const fileName = parts.pop()
+        const slug = parts.pop()
+        if (!fileName || !slug) continue
+        await reloadProjectFunctions(slug)
+      }
+    }
+  })()
+}
+
+export function getProjectFunctions(
+  slug: string,
+): LoadedFunction[] | undefined {
+  return functionsMap.get(slug)
+}
+
+export function stopWatcher() {
+  if (watcher) {
+    watcher.close()
+    watcher = null
+  }
+}
+
+export async function applyReadTransformers<T>(
+  data: T,
+  projectId: string,
+  deploymentUrl: string,
+  tableName: string,
+  projectFunctions?: LoadedFunction[],
+): Promise<T> {
+  if (!projectFunctions || projectFunctions.length === 0) {
+    return data
+  }
+  let currentData = data
+  for (const { module } of projectFunctions) {
+    if (!module.read) continue
+    if (module.config?.targets && !module.config.targets.includes(tableName)) {
+      continue
+    }
+
+    const ctx: FunctionContext = {
+      deploymentUrl,
+      projectId,
+    }
+
+    currentData = await module.read(currentData, ctx) as T
+  }
+
+  return currentData
+}
+
+export async function applyWriteTransformers<T>(
+  data: T,
+  projectId: string,
+  deploymentUrl: string,
+  tableName: string,
+  projectFunctions?: LoadedFunction[],
+): Promise<T> {
+  if (!projectFunctions || projectFunctions.length === 0) {
+    return data
+  }
+  let currentData = data
+  for (const { module } of projectFunctions) {
+    if (!module.write) continue
+    if (module.config?.targets && !module.config.targets.includes(tableName)) {
+      continue
+    }
+
+    const ctx: FunctionContext = {
+      deploymentUrl,
+      projectId,
+    }
+
+    currentData = await module.write(
+      tableName,
+      currentData,
+      undefined,
+      ctx,
+    ) as T
+  }
+
+  return currentData
+}

api/lib/functions_test.ts

@@ -0,0 +1,109 @@
+import { assertEquals } from '@std/assert'
+import * as functions from './functions.ts'
+import { join } from '@std/path'
+import { ensureDir } from '@std/fs'
+
+Deno.test('Functions Module - Pipeline & Config', async () => {
+  const testSlug = 'test-project-' + Date.now()
+  const functionsDir = './db/functions'
+  const projectDir = join(functionsDir, testSlug)
+  const file1 = join(projectDir, '01-first.js')
+  const file2 = join(projectDir, '02-second.js')
+
+  try {
+    await Deno.remove('./db_test/deployment_functions', { recursive: true })
+    await ensureDir('./db_test/deployment_functions')
+  } catch {
+    // Skipped
+  }
+
+  await ensureDir(projectDir)
+
+  // Initialize module
+  await functions.init()
+
+  // Define test row type
+  type TestRow = {
+    id: number
+    step1?: boolean
+    step2?: boolean
+    var1?: string
+  }
+
+  // 1. Create function files
+  const code1 = `
+    export default {
+        read: (row, ctx) => {
+            return { ...row, step1: true, var1: 'secret-value' }
+        }
+    }
+    `
+  const code2 = `
+    export default {
+        read: (row) => {
+            return { ...row, step2: true }
+        }
+    }
+    `
+  await Deno.writeTextFile(file1, code1)
+  await Deno.writeTextFile(file2, code2)
+
+  // Give watcher time
+  await new Promise((r) => setTimeout(r, 1000))
+
+  // 2. Verify loading and sorting
+  const loaded = functions.getProjectFunctions(testSlug)
+  if (!loaded) throw new Error('Functions not loaded')
+  assertEquals(loaded.length, 2)
+  assertEquals(loaded[0].name, '01-first.js')
+  assertEquals(loaded[1].name, '02-second.js')
+
+  // 3. Mock Deployment Config
+  const deploymentUrl = 'test-pipeline-' + Date.now() + '.com'
+
+  // 4. Simulate Pipeline Execution (Manually, echoing sql.ts logic)
+  // We can't import sql.ts functions easily here without mocking runSQL,
+  // so we re-implement the pipeline logic to verify the components work.
+
+  let row: TestRow = { id: 1 }
+
+  for (const { module } of loaded) {
+    if (!module.read) continue
+
+    const ctx = {
+      deploymentUrl,
+      projectId: testSlug,
+    }
+    row = await module.read(row, ctx) as TestRow
+  }
+
+  const result = row
+  assertEquals(result.step1, true)
+  assertEquals(result.var1, 'secret-value')
+
+  // Rerun pipeline
+  row = { id: 1 }
+
+  for (const { module } of loaded) {
+    if (!module.read) continue
+    const ctx = {
+      deploymentUrl,
+      projectId: testSlug,
+    }
+    row = await module.read(row, ctx) as TestRow
+  }
+
+  const result2 = row
+  assertEquals(result2.step1, true)
+  assertEquals(result2.step2, true)
+
+  // Cleanup
+  await Deno.remove(projectDir, { recursive: true })
+  try {
+    await Deno.remove('./db_test/deployment_functions', { recursive: true })
+  } catch {
+    // Skipped
+  }
+  await new Promise((r) => setTimeout(r, 500))
+  functions.stopWatcher()
+})