From edf4dbb491efbdeb92c7d7fd153716af70eafb45 Mon Sep 17 00:00:00 2001 From: Abdou TOP Date: Fri, 29 Aug 2025 10:13:38 +0000 Subject: [PATCH 1/7] feat(clickhouse): Implement ClickHouse client and create necessary tables and views --- api/click-house-client.ts | 93 +++++++++++++++++++++++++++++++++++++++ api/lib/env.ts | 13 ++++++ deno.json | 1 + deno.lock | 11 +++++ 4 files changed, 118 insertions(+) create mode 100644 api/click-house-client.ts diff --git a/api/click-house-client.ts b/api/click-house-client.ts new file mode 100644 index 0000000..fc9d507 --- /dev/null +++ b/api/click-house-client.ts @@ -0,0 +1,93 @@ +import { createClient } from 'npm:@clickhouse/client' +import { + CLICKHOUSE_HOST, + CLICKHOUSE_PASSWORD, + CLICKHOUSE_USER, +} from './lib/env.ts' + +const client = createClient({ + url: CLICKHOUSE_HOST, + username: CLICKHOUSE_USER, + password: CLICKHOUSE_PASSWORD, + compression: {}, +}) + +try { + await client.ping() + + await client.command({ + query: ` + CREATE TABLE IF NOT EXISTS logs ( + deployment_id LowCardinality(String), + timestamp DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), + message String, + log JSON, + INDEX idx_message message TYPE tokenbf_v1(8192, 3, 0) GRANULARITY 4 + ) + ENGINE = MergeTree + PARTITION BY toYYYYMMDD(timestamp) + ORDER BY (deployment_id, timestamp) + SETTINGS index_granularity = 8192, min_bytes_for_wide_part = 0; + `, + }) + + await client.command({ + query: ` + CREATE TABLE IF NOT EXISTS log_attribute_keys + ( + deployment_id LowCardinality(String), + key_path String, + data_type LowCardinality(String), + last_seen DateTime DEFAULT now() + ) + ENGINE = ReplacingMergeTree(last_seen) + ORDER BY (deployment_id, key_path); + `, + }) + + await client.command({ + query: ` + CREATE MATERIALIZED VIEW IF NOT EXISTS log_key_extractor_mv + TO log_attribute_keys + AS + SELECT + deployment_id, + -- On utilise l'alias du tuple .1 pour accéder au chemin + arrayStringConcat(json_pair.1, '.') AS key_path, + multiIf( + -- On utilise l'alias du tuple .2 pour accéder à la valeur + JSONType(json_pair.2) = 'Object', 'Object', + JSONType(json_pair.2) = 'Array', 'Array', + JSONType(json_pair.2) = 'String', 'String', + JSONType(json_pair.2) = 'Number', 'Number', + JSONType(json_pair.2) = 'Bool', 'Boolean', + 'Unknown' + ) AS data_type, + now() AS last_seen + FROM logs + -- LA CORRECTION EST ICI : on donne un seul alias au tuple + ARRAY JOIN JSONAllPaths(log) AS json_pair + `, + }) + console.log('deployment_logs table is ready') +} catch (error) { + console.error('Error creating ClickHouse table:', error) +} + +async function getAvailableLogKeys( + deploymentId: string, +): Promise<{ key: string; type: string }[]> { + const resultSet = await client.query({ + query: ` + SELECT key_path AS key, data_type AS type + FROM log_attribute_keys + WHERE deployment_id = {deploymentId:String} + ORDER BY key_path + `, + query_params: { deploymentId }, + format: 'JSONEachRow', + }) + return await resultSet.json() +} + +export { client } diff --git a/api/lib/env.ts b/api/lib/env.ts index e56d5e5..b52700a 100644 --- a/api/lib/env.ts +++ b/api/lib/env.ts @@ -26,3 +26,16 @@ export const ORIGIN = new URL(REDIRECT_URI).origin export const SECRET = env.SECRET || 'iUokBru8WPSMAuMspijlt7F-Cnpqyg84F36b1G681h0' + +export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST +if (!CLICKHOUSE_HOST) { + throw Error('CLICKHOUSE_HOST: field required in the env') +} +export const CLICKHOUSE_USER = env.CLICKHOUSE_USER +if (!CLICKHOUSE_USER) { + throw Error('CLICKHOUSE_USER: field required in the env') +} +export const CLICKHOUSE_PASSWORD = env.CLICKHOUSE_PASSWORD +if (!CLICKHOUSE_PASSWORD) { + throw Error('CLICKHOUSE_PASSWORD: field required in the env') +} diff --git a/deno.json b/deno.json index a2b08fb..7a956ad 100644 --- a/deno.json +++ b/deno.json @@ -34,6 +34,7 @@ "preact": "npm:preact@^10.26.9", "@preact/preset-vite": "npm:@preact/preset-vite@^2.10.2", "@preact/signals": "npm:@preact/signals", + "@@clickhouse/client": "npm:@clickhouse/client", "@tailwindcss/vite": "npm:@tailwindcss/vite@^4.1.11", "tailwindcss": "npm:tailwindcss@^4.1.11", "daisyui": "npm:daisyui@^5.0.46", diff --git a/deno.lock b/deno.lock index 14d9f10..a074fbb 100644 --- a/deno.lock +++ b/deno.lock @@ -24,6 +24,7 @@ "jsr:@std/path@^1.1.1": "1.1.1", "jsr:@std/streams@^1.0.8": "1.0.8", "jsr:@std/testing@*": "1.0.15", + "npm:@clickhouse/client@*": "1.12.1", "npm:@preact/preset-vite@^2.10.2": "2.10.2_@babel+core@7.28.0_vite@7.0.4__picomatch@4.0.2__@types+node@22.15.15_preact@10.26.9_@types+node@22.15.15", "npm:@preact/signals@*": "2.2.1_preact@10.26.9", "npm:@tailwindcss/vite@^4.1.11": "4.1.11_vite@7.0.4__picomatch@4.0.2__@types+node@22.15.15_@types+node@22.15.15", @@ -278,6 +279,15 @@ "@babel/helper-validator-identifier" ] }, + "@clickhouse/client-common@1.12.1": { + "integrity": "sha512-ccw1N6hB4+MyaAHIaWBwGZ6O2GgMlO99FlMj0B0UEGfjxM9v5dYVYql6FpP19rMwrVAroYs/IgX2vyZEBvzQLg==" + }, + "@clickhouse/client@1.12.1": { + "integrity": "sha512-7ORY85rphRazqHzImNXMrh4vsaPrpetFoTWpZYueCO2bbO6PXYDXp/GQ4DgxnGIqbWB/Di1Ai+Xuwq2o7DJ36A==", + "dependencies": [ + "@clickhouse/client-common" + ] + }, "@emnapi/core@1.4.4": { "integrity": "sha512-A9CnAbC6ARNMKcIcrQwq6HeHCjpcBZ5wSx4U01WXCqEKlrzB9F9315WDNHkrs2xbx7YjjSxbUYxuN6EQzpcY2g==", "dependencies": [ @@ -1228,6 +1238,7 @@ "workspace": { "dependencies": [ "jsr:@std/assert@1", + "npm:@clickhouse/client@*", "npm:@preact/preset-vite@^2.10.2", "npm:@preact/signals@*", "npm:@tailwindcss/vite@^4.1.11", From 27d72328804cb465e8fa0d806b29ab91c3a9cead Mon Sep 17 00:00:00 2001 From: Abdou TOP Date: Sat, 30 Aug 2025 14:13:55 +0000 Subject: [PATCH 2/7] feat(logging): Enhance ClickHouse logging functionality and add log insertion endpoint --- api/click-house-client.ts | 122 +++++++++++++++++++------------------- api/lib/context.ts | 2 + api/lib/validator.ts | 36 +++++++++++ api/routes.ts | 19 ++++++ api/server.ts | 1 + api/user.ts | 2 +- 6 files changed, 120 insertions(+), 62 deletions(-) diff --git a/api/click-house-client.ts b/api/click-house-client.ts index fc9d507..30edffa 100644 --- a/api/click-house-client.ts +++ b/api/click-house-client.ts @@ -4,12 +4,34 @@ import { CLICKHOUSE_PASSWORD, CLICKHOUSE_USER, } from './lib/env.ts' +import { respond } from './lib/response.ts' +import { log } from './lib/log.ts' +import { ARR, NUM, OBJ, STR, UNION } from './lib/validator.ts' +import { Asserted } from './lib/router.ts' + +const LogSchema = OBJ({ + timestamp: STR(), + trace_id: STR(), + span_id: STR(), + severity_text: STR(), + severity_number: NUM(), + attributes: OBJ({}), + event_name: STR(), +}) + +const LogsInputSchema = UNION(LogSchema, ARR(LogSchema)) + +type Log = Asserted +type LogsInput = Asserted const client = createClient({ url: CLICKHOUSE_HOST, username: CLICKHOUSE_USER, password: CLICKHOUSE_PASSWORD, - compression: {}, + compression: { + request: true, + response: true, + }, }) try { @@ -18,76 +40,54 @@ try { await client.command({ query: ` CREATE TABLE IF NOT EXISTS logs ( - deployment_id LowCardinality(String), + resource String, timestamp DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), - message String, - log JSON, - INDEX idx_message message TYPE tokenbf_v1(8192, 3, 0) GRANULARITY 4 + observed_timestamp DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), + trace_id UInt64, + span_id UInt64, + severity_text String, + severity_number UInt8, + attributes JSON, + event_name String ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(timestamp) - ORDER BY (deployment_id, timestamp) + ORDER BY (resource, timestamp, trace_id) SETTINGS index_granularity = 8192, min_bytes_for_wide_part = 0; `, }) - await client.command({ - query: ` - CREATE TABLE IF NOT EXISTS log_attribute_keys - ( - deployment_id LowCardinality(String), - key_path String, - data_type LowCardinality(String), - last_seen DateTime DEFAULT now() - ) - ENGINE = ReplacingMergeTree(last_seen) - ORDER BY (deployment_id, key_path); - `, - }) - - await client.command({ - query: ` - CREATE MATERIALIZED VIEW IF NOT EXISTS log_key_extractor_mv - TO log_attribute_keys - AS - SELECT - deployment_id, - -- On utilise l'alias du tuple .1 pour accéder au chemin - arrayStringConcat(json_pair.1, '.') AS key_path, - multiIf( - -- On utilise l'alias du tuple .2 pour accéder à la valeur - JSONType(json_pair.2) = 'Object', 'Object', - JSONType(json_pair.2) = 'Array', 'Array', - JSONType(json_pair.2) = 'String', 'String', - JSONType(json_pair.2) = 'Number', 'Number', - JSONType(json_pair.2) = 'Bool', 'Boolean', - 'Unknown' - ) AS data_type, - now() AS last_seen - FROM logs - -- LA CORRECTION EST ICI : on donne un seul alias au tuple - ARRAY JOIN JSONAllPaths(log) AS json_pair - `, - }) - console.log('deployment_logs table is ready') + log.info('deployment_logs table is ready') } catch (error) { - console.error('Error creating ClickHouse table:', error) + log.error('Error creating ClickHouse table:', { error }) + Deno.exit(1) } -async function getAvailableLogKeys( - deploymentId: string, -): Promise<{ key: string; type: string }[]> { - const resultSet = await client.query({ - query: ` - SELECT key_path AS key, data_type AS type - FROM log_attribute_keys - WHERE deployment_id = {deploymentId:String} - ORDER BY key_path - `, - query_params: { deploymentId }, - format: 'JSONEachRow', - }) - return await resultSet.json() +async function insertLogs( + resource: string, + data: LogsInput, +) { + const logsToInsert = Array.isArray(data) ? data : [data] + if (logsToInsert.length === 0) { + throw respond.NoContent() + } + + const values = logsToInsert.map((log) => ({ + ...log, + resource, + })) + + try { + await client.insert({ + table: 'logs', + values, + format: 'JSONEachRow', + }) + return respond.OK() + } catch (error) { + log.error("Erreur lors de l'insertion des logs dans ClickHouse:", { error }) + throw respond.InternalServerError() + } } -export { client } +export { client, insertLogs, LogSchema, LogsInputSchema } diff --git a/api/lib/context.ts b/api/lib/context.ts index c42df24..40fb18b 100644 --- a/api/lib/context.ts +++ b/api/lib/context.ts @@ -19,6 +19,7 @@ export type RequestContext = { readonly user: User | undefined readonly trace: number readonly span: number | undefined + resource: string | undefined } // we set default values so we don't have to check everytime if they exists @@ -33,6 +34,7 @@ export const makeContext = ( cookies: {}, user: undefined, span: undefined, + resource: undefined, url, req, ...extra, diff --git a/api/lib/validator.ts b/api/lib/validator.ts index 3c35963..993c126 100644 --- a/api/lib/validator.ts +++ b/api/lib/validator.ts @@ -29,6 +29,15 @@ type DefList = { assert: (value: unknown) => T[number] } +type DefUnion = { + type: 'union' + of: T + report: Validator> + optional?: boolean + description?: string + assert: (value: unknown) => ReturnType +} + type DefObject> = { type: 'object' properties: { [K in keyof T]: T[K] } @@ -69,6 +78,7 @@ export type DefBase = | DefArray | DefObject> | DefList + | DefUnion type OptionalAssert = ( value: unknown, @@ -258,6 +268,32 @@ export const LIST = ( description, }) +export const UNION = (...types: T): DefUnion => ({ + type: 'union', + of: types, + report: (value: unknown, path: (string | number)[] = []) => { + const failures: ValidatorFailure>[] = [] + for (const type of types) { + const result = type.report(value, path) + if (result.length === 0) return [] + failures.push(...result) + } + return failures + }, + assert: (value: unknown): ReturnType => { + for (const type of types) { + try { + return type.assert(value) + } catch { + // Ignore + } + } + throw new Error( + `Invalid value. Expected one of: ${types.map((t) => t.type).join(', ')}`, + ) + }, +}) + // const Article = OBJ({ // id: NUM("Unique identifier for the article"), // title: STR("Title of the article"), diff --git a/api/routes.ts b/api/routes.ts index 3f142a6..cfdf82d 100644 --- a/api/routes.ts +++ b/api/routes.ts @@ -14,6 +14,8 @@ import { ARR, BOOL, OBJ, optional, STR } from './lib/validator.ts' import { respond } from './lib/response.ts' import { deleteCookie } from 'jsr:@std/http/cookie' import { getPicture } from '/api/picture.ts' +import { insertLogs, LogsInputSchema } from './click-house-client.ts' +import { decryptMessage } from './user.ts' const withUserSession = ({ user }: RequestContext) => { if (!user) throw Error('Missing user session') @@ -23,6 +25,14 @@ const withAdminSession = ({ user }: RequestContext) => { if (!user || !user.isAdmin) throw Error('Admin access required') } +const withDeploymentSession = async (ctx: RequestContext) => { + const token = ctx.req.headers.get('Authorization')?.replace(/^Bearer /i, '') + if (!token) throw respond.Unauthorized({ message: 'Missing token' }) + const data = await decryptMessage(token) + if (!data) throw respond.Unauthorized({ message: 'Invalid token' }) + ctx.resource = 'default' +} + const defs = { 'GET/api/health': route({ fn: () => new Response('OK'), @@ -183,6 +193,15 @@ const defs = { output: BOOL('Indicates if the project was deleted'), description: 'Delete a project by ID', }), + 'POST/api/logs': route({ + authorize: withDeploymentSession, + fn: (ctx, logs) => { + if (!ctx.resource) throw respond.InternalServerError() + return insertLogs(ctx.resource, logs) + }, + input: LogsInputSchema, + description: 'Insert logs into ClickHouse', + }), } as const export type RouteDefinitions = typeof defs diff --git a/api/server.ts b/api/server.ts index b594ada..3460aef 100644 --- a/api/server.ts +++ b/api/server.ts @@ -64,6 +64,7 @@ export const fetch = async (req: Request) => { trace: cookies.trace ? Number(cookies.trace) : now(), user: await decodeSession(cookies.session), span: now(), + resource: undefined, } const res = await requestContext.run(ctx, handleRequest, ctx) diff --git a/api/user.ts b/api/user.ts index b314471..64ab44c 100644 --- a/api/user.ts +++ b/api/user.ts @@ -22,7 +22,7 @@ async function encryptMessage(message: string) { } // Decrypting a message -async function decryptMessage(encryptedMessage: string) { +export async function decryptMessage(encryptedMessage: string) { const encryptedData = decodeBase64Url(encryptedMessage) const iv = encryptedData.slice(0, IV_SIZE) const decryptedMessage = await crypto.subtle.decrypt( From 014ad5ef3fda466498d3111f00ed3f8ed6352c69 Mon Sep 17 00:00:00 2001 From: Abdou TOP Date: Sat, 30 Aug 2025 16:27:10 +0000 Subject: [PATCH 3/7] feat(deployment): Add deployment management routes and enhance user session handling --- .env.test | 6 +- api/click-house-client.ts | 31 +------- api/routes.ts | 151 ++++++++++++++++++++++++++++++++++++-- api/user.ts | 2 +- deno.json | 8 +- tasks/clickhouse.ts | 33 +++++++++ 6 files changed, 190 insertions(+), 41 deletions(-) create mode 100644 tasks/clickhouse.ts diff --git a/.env.test b/.env.test index de70d63..2fabf76 100644 --- a/.env.test +++ b/.env.test @@ -2,4 +2,8 @@ APP_ENV=test PORT=3021 GOOGLE_CLIENT_ID=...test.apps.googleusercontent.com CLIENT_SECRET=GOC...test -REDIRECT_URI=http://localhost:7737/api/auth/google \ No newline at end of file +REDIRECT_URI=http://localhost:7737/api/auth/google + +CLICKHOUSE_HOST=http://localhost:8443 +CLICKHOUSE_USER=default +CLICKHOUSE_PASSWORD=token_pass \ No newline at end of file diff --git a/api/click-house-client.ts b/api/click-house-client.ts index 30edffa..fc88fc0 100644 --- a/api/click-house-client.ts +++ b/api/click-house-client.ts @@ -34,35 +34,6 @@ const client = createClient({ }, }) -try { - await client.ping() - - await client.command({ - query: ` - CREATE TABLE IF NOT EXISTS logs ( - resource String, - timestamp DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), - observed_timestamp DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), - trace_id UInt64, - span_id UInt64, - severity_text String, - severity_number UInt8, - attributes JSON, - event_name String - ) - ENGINE = MergeTree - PARTITION BY toYYYYMMDD(timestamp) - ORDER BY (resource, timestamp, trace_id) - SETTINGS index_granularity = 8192, min_bytes_for_wide_part = 0; - `, - }) - - log.info('deployment_logs table is ready') -} catch (error) { - log.error('Error creating ClickHouse table:', { error }) - Deno.exit(1) -} - async function insertLogs( resource: string, data: LogsInput, @@ -85,7 +56,7 @@ async function insertLogs( }) return respond.OK() } catch (error) { - log.error("Erreur lors de l'insertion des logs dans ClickHouse:", { error }) + log.error('Error inserting logs into ClickHouse:', { error }) throw respond.InternalServerError() } } diff --git a/api/routes.ts b/api/routes.ts index cfdf82d..988be80 100644 --- a/api/routes.ts +++ b/api/routes.ts @@ -2,6 +2,8 @@ import { makeRouter, route } from '/api/lib/router.ts' import type { RequestContext } from '/api/lib/context.ts' import { handleGoogleCallback, initiateGoogleAuth } from './auth.ts' import { + DeploymentDef, + DeploymentsCollection, ProjectDef, ProjectsCollection, TeamDef, @@ -10,12 +12,12 @@ import { UserDef, UsersCollection, } from './schema.ts' -import { ARR, BOOL, OBJ, optional, STR } from './lib/validator.ts' +import { ARR, BOOL, NUM, OBJ, optional, STR } from './lib/validator.ts' import { respond } from './lib/response.ts' import { deleteCookie } from 'jsr:@std/http/cookie' import { getPicture } from '/api/picture.ts' import { insertLogs, LogsInputSchema } from './click-house-client.ts' -import { decryptMessage } from './user.ts' +import { decryptMessage, encryptMessage } from './user.ts' const withUserSession = ({ user }: RequestContext) => { if (!user) throw Error('Missing user session') @@ -28,11 +30,28 @@ const withAdminSession = ({ user }: RequestContext) => { const withDeploymentSession = async (ctx: RequestContext) => { const token = ctx.req.headers.get('Authorization')?.replace(/^Bearer /i, '') if (!token) throw respond.Unauthorized({ message: 'Missing token' }) - const data = await decryptMessage(token) - if (!data) throw respond.Unauthorized({ message: 'Invalid token' }) - ctx.resource = 'default' + const message = await decryptMessage(token) + if (!message) throw respond.Unauthorized({ message: 'Invalid token' }) + const data = JSON.parse(message) + const dep = DeploymentsCollection.get(data?.url) + if (!dep || dep.tokenSalt !== data?.tokenSalt) { + throw respond.Unauthorized({ message: 'Invalid token' }) + } + ctx.resource = dep?.url } +const deploymentOutput = OBJ({ + projectId: STR('The ID of the project'), + url: STR('The URL of the deployment'), + logsEnabled: BOOL('Whether logging is enabled'), + databaseEnabled: BOOL('Whether the database is enabled'), + sqlEndpoint: optional(STR('The SQL endpoint')), + sqlToken: optional(STR('The SQL token')), + createdAt: optional(NUM('The creation date of the deployment')), + updatedAt: optional(NUM('The last update date of the deployment')), + token: optional(STR('The deployment token')), +}) + const defs = { 'GET/api/health': route({ fn: () => new Response('OK'), @@ -193,6 +212,126 @@ const defs = { output: BOOL('Indicates if the project was deleted'), description: 'Delete a project by ID', }), + 'GET/api/project/deployments': route({ + authorize: withUserSession, + fn: (_ctx, { project }) => { + const deployments = DeploymentsCollection.filter((d) => + d.projectId === project + ) + if (!deployments.length) { + throw respond.NotFound({ message: 'Deployments not found' }) + } + return deployments.map(({ tokenSalt: _, ...d }) => { + return { + ...d, + createdAt: d.createdAt, + updatedAt: d.updatedAt, + token: undefined, + sqlToken: undefined, + sqlEndpoint: undefined, + } + }) + }, + input: OBJ({ project: STR('The ID of the project') }), + output: ARR(deploymentOutput, 'List of deployments'), + description: 'Get deployments by project ID', + }), + 'GET/api/deployment': route({ + authorize: withAdminSession, + fn: async (_ctx, url) => { + const dep = DeploymentsCollection.get(url) + if (!dep) throw respond.NotFound() + const { tokenSalt, ...deployment } = dep + const token = await encryptMessage( + JSON.stringify({ url: deployment.url, tokenSalt }), + ) + return { + ...deployment, + createdAt: deployment.createdAt, + updatedAt: deployment.updatedAt, + token, + } + }, + input: STR(), + output: deploymentOutput, + description: 'Get a deployment by ID', + }), + 'POST/api/deployment': route({ + authorize: withAdminSession, + fn: async (_ctx, input) => { + const tokenSalt = performance.now().toString() + const { tokenSalt: _, ...deployment } = await DeploymentsCollection + .insert({ + ...input, + tokenSalt, + }) + const token = await encryptMessage( + JSON.stringify({ url: deployment.url, tokenSalt }), + ) + return { + ...deployment, + createdAt: deployment.createdAt, + updatedAt: deployment.updatedAt, + token, + } + }, + input: DeploymentDef, + output: deploymentOutput, + description: 'Create a new deployment', + }), + 'PUT/api/deployment': route({ + authorize: withAdminSession, + fn: async (_ctx, input) => { + const { tokenSalt, ...deployment } = await DeploymentsCollection + .update(input.url, input) + const token = await encryptMessage( + JSON.stringify({ url: deployment.url, tokenSalt }), + ) + return { + ...deployment, + createdAt: deployment.createdAt, + updatedAt: deployment.updatedAt, + token, + } + }, + input: DeploymentDef, + output: deploymentOutput, + description: 'Update a deployment by ID', + }), + 'GET/api/deployment/token/regenerate': route({ + authorize: withAdminSession, + fn: async (_ctx, input) => { + const dep = DeploymentsCollection.get(input) + if (!dep) throw respond.NotFound() + const tokenSalt = performance.now().toString() + + const { tokenSalt: _, ...deployment } = await DeploymentsCollection + .update(input, { ...dep, tokenSalt }) + const token = await encryptMessage( + JSON.stringify({ url: deployment.url, tokenSalt }), + ) + return { + ...deployment, + createdAt: deployment.createdAt, + updatedAt: deployment.updatedAt, + token, + } + }, + input: STR(), + output: deploymentOutput, + description: 'Regenerate a deployment token', + }), + 'DELETE/api/deployment': route({ + authorize: withAdminSession, + fn: async (_ctx, input) => { + const dep = DeploymentsCollection.get(input) + if (!dep) throw respond.NotFound() + await DeploymentsCollection.delete(input) + return respond.NoContent() + }, + input: STR(), + description: 'Delete a deployment', + }), 'POST/api/logs': route({ authorize: withDeploymentSession, fn: (ctx, logs) => { @@ -200,7 +339,7 @@ const defs = { return insertLogs(ctx.resource, logs) }, input: LogsInputSchema, - description: 'Insert logs into ClickHouse', + description: 'Insert logs into ClickHouse NB: a Bearer token is required', }), } as const diff --git a/api/user.ts b/api/user.ts index 64ab44c..4ed035a 100644 --- a/api/user.ts +++ b/api/user.ts @@ -7,7 +7,7 @@ const encoder = new TextEncoder() const decoder = new TextDecoder() const IV_SIZE = 12 // Initialization vector (12 bytes for AES-GCM) -async function encryptMessage(message: string) { +export async function encryptMessage(message: string) { const iv = crypto.getRandomValues(new Uint8Array(IV_SIZE)) const encryptedMessage = await crypto.subtle.encrypt( { name: 'AES-GCM', iv }, diff --git a/deno.json b/deno.json index 7a956ad..43dd3d5 100644 --- a/deno.json +++ b/deno.json @@ -2,13 +2,15 @@ "tasks": { "api:dev": "deno run -A --env-file=.env.dev api/server.ts --env=dev --port=3021", "vite:dev": "deno run -A --env-file=.env.dev tasks/vite.js --env=dev", - "dev": { "dependencies": ["api:dev", "vite:dev"] }, + "clickhouse:dev": "deno run -A --env-file=.env.dev tasks/clickhouse.ts --env=dev", + "dev": { "dependencies": ["clickhouse:dev", "api:dev", "vite:dev"] }, "seed": "deno run -A --env-file=.env.dev tasks/seed.ts", "dev:with-seed": "deno task seed && deno task dev", "api:prod": "deno compile -A --env-file=.env.prod --no-check --output dist/api --target x86_64-unknown-linux-gnu --include dist/web api/server.ts --env=prod", "vite:prod": "deno run -A --env-file=.env.prod tasks/vite.js --build --env=prod", + "clickhouse:prod": "deno run -A --env-file=.env.prod tasks/clickhouse.ts --env=prod", "prod": "deno task vite:prod && deno task api:prod", - "start:prod": "dist/api --env=prod", + "start:prod": "deno task clickhouse:prod && dist/api --env=prod", "fmt": "deno fmt", "lint": "deno lint", "check": "deno check", @@ -34,7 +36,7 @@ "preact": "npm:preact@^10.26.9", "@preact/preset-vite": "npm:@preact/preset-vite@^2.10.2", "@preact/signals": "npm:@preact/signals", - "@@clickhouse/client": "npm:@clickhouse/client", + "@clickhouse/client": "npm:@clickhouse/client", "@tailwindcss/vite": "npm:@tailwindcss/vite@^4.1.11", "tailwindcss": "npm:tailwindcss@^4.1.11", "daisyui": "npm:daisyui@^5.0.46", diff --git a/tasks/clickhouse.ts b/tasks/clickhouse.ts new file mode 100644 index 0000000..dbae0a0 --- /dev/null +++ b/tasks/clickhouse.ts @@ -0,0 +1,33 @@ +import { client } from '../api/click-house-client.ts' +import { log } from '../api/lib/log.ts' + +if (import.meta.main) { + try { + await client.ping() + + await client.command({ + query: ` + CREATE TABLE IF NOT EXISTS logs ( + resource String, + timestamp DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), + observed_timestamp DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), + trace_id UInt64, + span_id UInt64, + severity_text String, + severity_number UInt8, + attributes JSON, + event_name String + ) + ENGINE = MergeTree + PARTITION BY toYYYYMMDD(timestamp) + ORDER BY (resource, timestamp, trace_id) + SETTINGS index_granularity = 8192, min_bytes_for_wide_part = 0; + `, + }) + + log.info('logs table is ready') + } catch (error) { + log.error('Error creating ClickHouse table:', { error }) + Deno.exit(1) + } +} From 770c06c8b3b6a7a17a25f47b752a1b05e57ea6d3 Mon Sep 17 00:00:00 2001 From: Abdou TOP Date: Mon, 1 Sep 2025 17:32:17 +0000 Subject: [PATCH 4/7] feat(schema): Update log schema to include context and remove severity_text --- api/click-house-client.ts | 2 +- tasks/clickhouse.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/click-house-client.ts b/api/click-house-client.ts index fc88fc0..484ecb1 100644 --- a/api/click-house-client.ts +++ b/api/click-house-client.ts @@ -13,10 +13,10 @@ const LogSchema = OBJ({ timestamp: STR(), trace_id: STR(), span_id: STR(), - severity_text: STR(), severity_number: NUM(), attributes: OBJ({}), event_name: STR(), + context: OBJ({}), }) const LogsInputSchema = UNION(LogSchema, ARR(LogSchema)) diff --git a/tasks/clickhouse.ts b/tasks/clickhouse.ts index dbae0a0..6018122 100644 --- a/tasks/clickhouse.ts +++ b/tasks/clickhouse.ts @@ -1,5 +1,5 @@ -import { client } from '../api/click-house-client.ts' -import { log } from '../api/lib/log.ts' +import { client } from '/api/click-house-client.ts' +import { log } from '/api/lib/log.ts' if (import.meta.main) { try { @@ -13,10 +13,10 @@ if (import.meta.main) { observed_timestamp DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), trace_id UInt64, span_id UInt64, - severity_text String, severity_number UInt8, attributes JSON, - event_name String + event_name String, + context JSON ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(timestamp) From b3e0b67f6bbc45964bc26262559ca47589369fae Mon Sep 17 00:00:00 2001 From: Abdou TOP Date: Tue, 2 Sep 2025 19:55:32 +0000 Subject: [PATCH 5/7] feat(logging): Add getLogs endpoint to retrieve logs from ClickHouse and update log insertion functionality --- api/click-house-client.ts | 88 ++++++++++++++++++++++++++++++++++++++- api/routes.ts | 39 ++++++++++++++--- 2 files changed, 119 insertions(+), 8 deletions(-) diff --git a/api/click-house-client.ts b/api/click-house-client.ts index 484ecb1..30da6e1 100644 --- a/api/click-house-client.ts +++ b/api/click-house-client.ts @@ -21,7 +21,7 @@ const LogSchema = OBJ({ const LogsInputSchema = UNION(LogSchema, ARR(LogSchema)) -type Log = Asserted +export type Log = Asserted type LogsInput = Asserted const client = createClient({ @@ -61,4 +61,88 @@ async function insertLogs( } } -export { client, insertLogs, LogSchema, LogsInputSchema } +function toClickhouseDateTime(iso: string) { + // "2025-09-11T17:35:00.000Z" → "2025-09-11 17:35:00" + const match = iso.match( + /^(\d{4}-\d{2}-\d{2})T(\d{2}):(\d{2})(?::(\d{2}))?(\.\d+)?Z?$/, + ) + if (!match) return iso.replace('T', ' ').replace(/(\.\d+)?Z$/, '') + const [_, date, h, m, s] = match + return `${date} ${h}:${m}:${s ?? '00'}` +} + +async function getLogs({ + resource, + level, + startDate, + endDate, + sortBy, + sortOrder, + search, +}: { + resource: string + level?: string + startDate?: string + endDate?: string + sortBy?: string + sortOrder?: 'ASC' | 'DESC' + search?: Record +}) { + const queryParts: string[] = [] + const queryParams: Record = { resource } + + queryParts.push('resource = {resource:String}') + + if (level) { + queryParts.push('severity_number = {level:UInt8}') + queryParams.level = level + } + + if (startDate) { + queryParts.push('timestamp >= {startDate:DateTime}') + queryParams.startDate = toClickhouseDateTime(startDate) + } + + if (endDate) { + queryParts.push('timestamp <= {endDate:DateTime}') + queryParams.endDate = toClickhouseDateTime(endDate) + } + + if (search) { + if (search.trace_id) { + queryParts.push('trace_id = {trace_id:String}') + queryParams.trace_id = search.trace_id + } + if (search.span_id) { + queryParts.push('span_id = {span_id:String}') + queryParams.span_id = search.span_id + } + if (search.event_name) { + queryParts.push('event_name = {event_name:String}') + queryParams.event_name = search.event_name + } + } + + const query = ` + SELECT * + FROM logs + WHERE ${queryParts.join(' AND ')} + ${sortBy ? `ORDER BY ${sortBy} ${sortOrder || 'DESC'}` : ''} + LIMIT 1000 + ` + + try { + const resultSet = await client.query({ + query, + query_params: queryParams, + format: 'JSONEachRow', + }) + + return resultSet.json() + } catch (error) { + log.error('Error querying logs from ClickHouse:', { error }) + throw respond.InternalServerError() + } +} + +export { client, getLogs, insertLogs, LogSchema, LogsInputSchema } diff --git a/api/routes.ts b/api/routes.ts index 988be80..9218de6 100644 --- a/api/routes.ts +++ b/api/routes.ts @@ -12,11 +12,16 @@ import { UserDef, UsersCollection, } from './schema.ts' -import { ARR, BOOL, NUM, OBJ, optional, STR } from './lib/validator.ts' +import { ARR, BOOL, LIST, NUM, OBJ, optional, STR } from './lib/validator.ts' import { respond } from './lib/response.ts' import { deleteCookie } from 'jsr:@std/http/cookie' import { getPicture } from '/api/picture.ts' -import { insertLogs, LogsInputSchema } from './click-house-client.ts' +import { + getLogs, + insertLogs, + LogSchema, + LogsInputSchema, +} from './click-house-client.ts' import { decryptMessage, encryptMessage } from './user.ts' const withUserSession = ({ user }: RequestContext) => { @@ -300,13 +305,15 @@ const defs = { }), 'GET/api/deployment/token/regenerate': route({ authorize: withAdminSession, - fn: async (_ctx, input) => { - const dep = DeploymentsCollection.get(input) + fn: async (_ctx, { url }) => { + console.log('Regenerating token for deployment:', { url }) + const dep = DeploymentsCollection.get(url) + console.log('Regenerating token for deployment:', { dep }) if (!dep) throw respond.NotFound() const tokenSalt = performance.now().toString() const { tokenSalt: _, ...deployment } = await DeploymentsCollection - .update(input, { ...dep, tokenSalt }) + .update(url, { ...dep, tokenSalt }) const token = await encryptMessage( JSON.stringify({ url: deployment.url, tokenSalt }), ) @@ -317,7 +324,7 @@ const defs = { token, } }, - input: STR(), + input: OBJ({ url: STR('The URL of the deployment') }), output: deploymentOutput, description: 'Regenerate a deployment token', }), @@ -341,6 +348,26 @@ const defs = { input: LogsInputSchema, description: 'Insert logs into ClickHouse NB: a Bearer token is required', }), + 'GET/api/logs': route({ + authorize: withUserSession, + fn: async (_ctx, params) => { + const logs = await getLogs(params) + return logs.flat() + }, + input: OBJ({ + resource: STR('The resource to fetch logs for'), + level: optional(STR('The log level to filter by')), + start_date: optional(STR('The start date for the date range filter')), + end_date: optional(STR('The end date for the date range filter')), + sort_by: optional(STR('The field to sort by')), + sort_order: optional( + LIST(['ASC', 'DESC'], 'The sort order (ASC or DESC)'), + ), + search: optional(OBJ({}, 'A map of fields to search by')), + }), + output: ARR(LogSchema, 'List of logs'), + description: 'Get logs from ClickHouse', + }), } as const export type RouteDefinitions = typeof defs From 54ee31c5962b01c1f40857de95c312ac912f1f58 Mon Sep 17 00:00:00 2001 From: Abdou TOP Date: Sat, 6 Sep 2025 12:09:33 +0000 Subject: [PATCH 6/7] feat(schema): Enhance log schema with additional fields and improve data types for better logging structure --- api/click-house-client.ts | 85 +++++++++++++++++++++++++-------------- api/lib/json_store.ts | 24 +++++------ api/routes.ts | 49 +++++++++++++--------- api/schema.ts | 10 ++--- tasks/clickhouse.ts | 25 +++++++++--- 5 files changed, 120 insertions(+), 73 deletions(-) diff --git a/api/click-house-client.ts b/api/click-house-client.ts index 30da6e1..7a5c528 100644 --- a/api/click-house-client.ts +++ b/api/click-house-client.ts @@ -6,22 +6,25 @@ import { } from './lib/env.ts' import { respond } from './lib/response.ts' import { log } from './lib/log.ts' -import { ARR, NUM, OBJ, STR, UNION } from './lib/validator.ts' +import { ARR, NUM, OBJ, optional, STR, UNION } from './lib/validator.ts' import { Asserted } from './lib/router.ts' const LogSchema = OBJ({ - timestamp: STR(), - trace_id: STR(), - span_id: STR(), - severity_number: NUM(), - attributes: OBJ({}), - event_name: STR(), - context: OBJ({}), -}) - -const LogsInputSchema = UNION(LogSchema, ARR(LogSchema)) - -export type Log = Asserted + timestamp: NUM('The timestamp of the log event'), + trace_id: NUM('A float64 representation of the trace ID'), + span_id: optional(NUM('A float64 representation of the span ID')), + severity_number: NUM('The severity number of the log event'), + attributes: optional(OBJ({}, 'A map of attributes')), + event_name: STR('The name of the event'), + service_version: optional(STR('Service version')), + service_instance_id: optional(STR('Service instance ID')), +}, 'A log event') +const LogsInputSchema = UNION( + LogSchema, + ARR(LogSchema, 'An array of log events'), +) + +type Log = Asserted type LogsInput = Asserted const client = createClient({ @@ -34,26 +37,47 @@ const client = createClient({ }, }) +export function float64ToId128( + { id }: { id: number }, +) { + const id128 = new Uint8Array(8) + const view = new DataView(id128.buffer) + view.setFloat64(0, id, false) + return id128 +} + +export function bytesToHex(bytes: Uint8Array) { + return Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join('') +} +const escapeSql = (s: unknown) => + String(s ?? '').replace(/\\/g, '\\\\').replace(/'/g, "''") + async function insertLogs( - resource: string, + service_name: string, data: LogsInput, ) { const logsToInsert = Array.isArray(data) ? data : [data] - if (logsToInsert.length === 0) { - throw respond.NoContent() - } + if (logsToInsert.length === 0) throw respond.NoContent() + + const rows = logsToInsert.map((log) => { + const traceHex = bytesToHex(float64ToId128({ id: log.trace_id })) + const spanHex = bytesToHex( + float64ToId128({ id: log.span_id ?? log.trace_id }), + ) + return { + ...log, + timestamp: toClickhouseDateTime(new Date(log.timestamp).toISOString()), + attributes: log.attributes ?? {}, + service_name: escapeSql(service_name), + trace_id: traceHex, + span_id: spanHex, + } + }) - const values = logsToInsert.map((log) => ({ - ...log, - resource, - })) + log.debug('Inserting logs into ClickHouse', { rows }) try { - await client.insert({ - table: 'logs', - values, - format: 'JSONEachRow', - }) + await client.insert({ table: 'logs', values: rows, format: 'JSONEachRow' }) return respond.OK() } catch (error) { log.error('Error inserting logs into ClickHouse:', { error }) @@ -89,9 +113,9 @@ async function getLogs({ search?: Record }) { const queryParts: string[] = [] - const queryParams: Record = { resource } + const queryParams: Record = { service_name: resource } - queryParts.push('resource = {resource:String}') + queryParts.push('service_name = {service_name:String}') if (level) { queryParts.push('severity_number = {level:UInt8}') @@ -134,11 +158,12 @@ async function getLogs({ try { const resultSet = await client.query({ query, + query_params: queryParams, - format: 'JSONEachRow', + format: 'JSON', }) - return resultSet.json() + return (await resultSet.json()).data } catch (error) { log.error('Error querying logs from ClickHouse:', { error }) throw respond.InternalServerError() diff --git a/api/lib/json_store.ts b/api/lib/json_store.ts index 0ae4d4c..497219d 100644 --- a/api/lib/json_store.ts +++ b/api/lib/json_store.ts @@ -32,10 +32,10 @@ const batch = async ( await Promise.all(pool) } -export type BaseRecord = { createdAt?: number; updatedAt?: number } +export type BaseRecord = { createdAt: number; updatedAt: number } export async function createCollection< - T extends Record & BaseRecord, + T extends Record, K extends keyof T, >({ name, primaryKey }: CollectionOptions) { const dir = join(DB_DIR, name) @@ -66,34 +66,34 @@ export async function createCollection< const id = data[primaryKey] if (!id) throw Error(`Missing primary key ${primaryKey}`) if (records.has(id)) throw Error(`${id} already exists`) - Object.assign(data, { createdAt: Date.now() }) + Object.assign(data, { createdAt: Date.now(), updatedAt: Date.now() }) records.set(id, data) await saveRecord(data) - return data + return data as T & BaseRecord }, [Symbol.iterator]: () => records[Symbol.iterator], keys: () => records.keys(), - values: () => records.values(), - entries: () => records.entries(), - get: (id: T[K]) => records.get(id), + values: () => records.values() as MapIterator, + entries: () => records.entries() as MapIterator<[T[K], T & BaseRecord]>, + get: (id: T[K]) => records.get(id) as T & BaseRecord | undefined, assert: (id: T[K]) => { const match = records.get(id) - if (match) return match + if (match) return match as T & BaseRecord throw new Deno.errors.NotFound(`record ${id} not found`) }, find: (predicate: (record: T) => unknown) => - records.values().find(predicate), + records.values().find(predicate) as T & BaseRecord | undefined, filter: (predicate: (record: T) => unknown) => - records.values().filter(predicate).toArray(), + records.values().filter(predicate).toArray() as (T & BaseRecord)[], async update(id: T[K], changes: Partial>) { const record = records.get(id) if (!record) throw new Deno.errors.NotFound(`record ${id} not found`) - const updated = { ...record, ...changes, _updatedAt: Date.now() } as T + const updated = { ...record, ...changes, updatedAt: Date.now() } as T records.set(id, updated) await saveRecord(updated) - return updated + return updated as T & BaseRecord }, async delete(id: T[K]) { diff --git a/api/routes.ts b/api/routes.ts index 9218de6..3658d48 100644 --- a/api/routes.ts +++ b/api/routes.ts @@ -23,6 +23,7 @@ import { LogsInputSchema, } from './click-house-client.ts' import { decryptMessage, encryptMessage } from './user.ts' +import { log } from './lib/log.ts' const withUserSession = ({ user }: RequestContext) => { if (!user) throw Error('Missing user session') @@ -35,14 +36,19 @@ const withAdminSession = ({ user }: RequestContext) => { const withDeploymentSession = async (ctx: RequestContext) => { const token = ctx.req.headers.get('Authorization')?.replace(/^Bearer /i, '') if (!token) throw respond.Unauthorized({ message: 'Missing token' }) - const message = await decryptMessage(token) - if (!message) throw respond.Unauthorized({ message: 'Invalid token' }) - const data = JSON.parse(message) - const dep = DeploymentsCollection.get(data?.url) - if (!dep || dep.tokenSalt !== data?.tokenSalt) { + try { + const message = await decryptMessage(token) + if (!message) throw respond.Unauthorized({ message: 'Invalid token' }) + const data = JSON.parse(message) + const dep = DeploymentsCollection.get(data?.url) + if (!dep || dep.tokenSalt !== data?.tokenSalt) { + throw respond.Unauthorized({ message: 'Invalid token' }) + } + ctx.resource = dep?.url + } catch (error) { + log.error('Error validating deployment token:', { error }) throw respond.Unauthorized({ message: 'Invalid token' }) } - ctx.resource = dep?.url } const deploymentOutput = OBJ({ @@ -229,8 +235,6 @@ const defs = { return deployments.map(({ tokenSalt: _, ...d }) => { return { ...d, - createdAt: d.createdAt, - updatedAt: d.updatedAt, token: undefined, sqlToken: undefined, sqlEndpoint: undefined, @@ -252,8 +256,6 @@ const defs = { ) return { ...deployment, - createdAt: deployment.createdAt, - updatedAt: deployment.updatedAt, token, } }, @@ -275,8 +277,6 @@ const defs = { ) return { ...deployment, - createdAt: deployment.createdAt, - updatedAt: deployment.updatedAt, token, } }, @@ -294,8 +294,6 @@ const defs = { ) return { ...deployment, - createdAt: deployment.createdAt, - updatedAt: deployment.updatedAt, token, } }, @@ -319,8 +317,6 @@ const defs = { ) return { ...deployment, - createdAt: deployment.createdAt, - updatedAt: deployment.updatedAt, token, } }, @@ -350,9 +346,22 @@ const defs = { }), 'GET/api/logs': route({ authorize: withUserSession, - fn: async (_ctx, params) => { - const logs = await getLogs(params) - return logs.flat() + fn: (ctx, params) => { + const deployments = DeploymentsCollection.filter((d) => + d.projectId === params.resource + ) + if (deployments.length === 0) { + throw respond.NotFound({ message: 'Deployments not found' }) + } + const project = ProjectsCollection.get(deployments[0].projectId) + if (!project) throw respond.NotFound({ message: 'Project not found' }) + if (!project.isPublic && !ctx.user?.isAdmin) { + const team = TeamsCollection.find((t) => t.teamId === project.teamId) + if (!team?.teamMembers.includes(ctx.user?.userEmail || '')) { + throw respond.Forbidden({ message: 'Access to project logs denied' }) + } + } + return getLogs(params) }, input: OBJ({ resource: STR('The resource to fetch logs for'), @@ -363,7 +372,7 @@ const defs = { sort_order: optional( LIST(['ASC', 'DESC'], 'The sort order (ASC or DESC)'), ), - search: optional(OBJ({}, 'A map of fields to search by')), + // search: optional(OBJ({}, 'A map of fields to search by')), }), output: ARR(LogSchema, 'List of logs'), description: 'Get logs from ClickHouse', diff --git a/api/schema.ts b/api/schema.ts index e03a00d..78da210 100644 --- a/api/schema.ts +++ b/api/schema.ts @@ -1,6 +1,6 @@ import { ARR, BOOL, OBJ, optional, STR } from './lib/validator.ts' import { Asserted } from './lib/router.ts' -import { BaseRecord, createCollection } from './lib/json_store.ts' +import { createCollection } from './lib/json_store.ts' export const UserDef = OBJ({ userEmail: STR('The user email address'), @@ -8,7 +8,7 @@ export const UserDef = OBJ({ userPicture: optional(STR('The user profile picture URL')), isAdmin: BOOL('Is the user an admin?'), }, 'The user schema definition') -export type User = Asserted & BaseRecord +export type User = Asserted export const TeamDef = OBJ({ teamId: STR('The unique identifier for the team'), @@ -18,7 +18,7 @@ export const TeamDef = OBJ({ 'The list of user emails who are members of the team', ), }, 'The team schema definition') -export type Team = Asserted & BaseRecord +export type Team = Asserted export const ProjectDef = OBJ({ slug: STR('The unique identifier for the project'), @@ -27,7 +27,7 @@ export const ProjectDef = OBJ({ isPublic: BOOL('Is the project public?'), repositoryUrl: optional(STR('The URL of the project repository')), }, 'The project schema definition') -export type Project = Asserted & BaseRecord +export type Project = Asserted export const DeploymentDef = OBJ({ projectId: STR('The ID of the project this deployment belongs to'), @@ -37,7 +37,7 @@ export const DeploymentDef = OBJ({ sqlEndpoint: optional(STR('The SQL execution endpoint for the database')), sqlToken: optional(STR('The security token for the SQL endpoint')), }, 'The deployment schema definition') -export type Deployment = Asserted & BaseRecord +export type Deployment = Asserted export const UsersCollection = await createCollection( { name: 'users', primaryKey: 'userEmail' }, diff --git a/tasks/clickhouse.ts b/tasks/clickhouse.ts index 6018122..ee1ab1a 100644 --- a/tasks/clickhouse.ts +++ b/tasks/clickhouse.ts @@ -8,19 +8,32 @@ if (import.meta.main) { await client.command({ query: ` CREATE TABLE IF NOT EXISTS logs ( - resource String, + -- Flattened resource fields + service_name LowCardinality(String), + service_version LowCardinality(String), + service_instance_id String, + timestamp DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), observed_timestamp DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), - trace_id UInt64, - span_id UInt64, + trace_id FixedString(16), + span_id FixedString(16), severity_number UInt8, + -- derived column, computed by DB from severity_number + severity_text String MATERIALIZED CASE + WHEN severity_number > 4 AND severity_number <= 8 THEN 'DEBUG' + WHEN severity_number > 8 AND severity_number <= 12 THEN 'INFO' + WHEN severity_number > 12 AND severity_number <= 16 THEN 'WARN' + WHEN severity_number > 20 AND severity_number <= 24 THEN 'FATAL' + ELSE 'ERROR' + END, + -- Often empty, but kept for OTEL spec compliance + body Nullable(String), attributes JSON, - event_name String, - context JSON + event_name String ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(timestamp) - ORDER BY (resource, timestamp, trace_id) + ORDER BY (service_name, timestamp, trace_id) SETTINGS index_granularity = 8192, min_bytes_for_wide_part = 0; `, }) From a0e50df2ba865ef3f32cc31e2c81308360ee7e55 Mon Sep 17 00:00:00 2001 From: Abdou TOP Date: Fri, 12 Sep 2025 10:24:21 +0000 Subject: [PATCH 7/7] feat(logging): Refactor number conversion functions for log trace and span IDs --- api/click-house-client.ts | 95 +++++++++++++--------------- api/routes.ts | 32 ++++++---- api/user.ts | 2 +- deno.json | 5 ++ tasks/clickhouse.ts | 4 +- web/pages/ProjectPage.tsx | 27 ++------ web/pages/ProjectsPage.tsx | 14 ++-- web/pages/project/DeploymentPage.tsx | 11 ++-- web/pages/project/SettingsPage.tsx | 11 ++-- 9 files changed, 100 insertions(+), 101 deletions(-) diff --git a/api/click-house-client.ts b/api/click-house-client.ts index 7a5c528..03b1ae9 100644 --- a/api/click-house-client.ts +++ b/api/click-house-client.ts @@ -35,22 +35,27 @@ const client = createClient({ request: true, response: true, }, + clickhouse_settings: { + date_time_input_format: 'best_effort', + }, }) -export function float64ToId128( - { id }: { id: number }, -) { - const id128 = new Uint8Array(8) - const view = new DataView(id128.buffer) - view.setFloat64(0, id, false) - return id128 -} - -export function bytesToHex(bytes: Uint8Array) { - return Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join('') -} -const escapeSql = (s: unknown) => - String(s ?? '').replace(/\\/g, '\\\\').replace(/'/g, "''") +const numberToHex128 = (() => { + const alphabet = new TextEncoder().encode('0123456789abcdef') + const output = new Uint8Array(16) + const view = new DataView(new Uint8Array(8).buffer) + const dec = new TextDecoder() + return (id: number) => { + view.setFloat64(0, id, false) + let i = -1 + while (++i < 8) { + const x = view.getUint8(i) + output[i * 2] = alphabet[x >> 4] + output[i * 2 + 1] = alphabet[x & 0xF] + } + return dec.decode(output) + } +})() async function insertLogs( service_name: string, @@ -60,15 +65,13 @@ async function insertLogs( if (logsToInsert.length === 0) throw respond.NoContent() const rows = logsToInsert.map((log) => { - const traceHex = bytesToHex(float64ToId128({ id: log.trace_id })) - const spanHex = bytesToHex( - float64ToId128({ id: log.span_id ?? log.trace_id }), - ) + const traceHex = numberToHex128(log.trace_id) + const spanHex = numberToHex128(log.span_id ?? log.trace_id) return { ...log, - timestamp: toClickhouseDateTime(new Date(log.timestamp).toISOString()), + timestamp: new Date(log.timestamp), attributes: log.attributes ?? {}, - service_name: escapeSql(service_name), + service_name: service_name, trace_id: traceHex, span_id: spanHex, } @@ -85,51 +88,42 @@ async function insertLogs( } } -function toClickhouseDateTime(iso: string) { - // "2025-09-11T17:35:00.000Z" → "2025-09-11 17:35:00" - const match = iso.match( - /^(\d{4}-\d{2}-\d{2})T(\d{2}):(\d{2})(?::(\d{2}))?(\.\d+)?Z?$/, - ) - if (!match) return iso.replace('T', ' ').replace(/(\.\d+)?Z$/, '') - const [_, date, h, m, s] = match - return `${date} ${h}:${m}:${s ?? '00'}` -} - async function getLogs({ resource, - level, - startDate, - endDate, - sortBy, - sortOrder, + severity_number, + start_date, + end_date, + sort_by, + sort_order, search, }: { resource: string - level?: string - startDate?: string - endDate?: string - sortBy?: string - sortOrder?: 'ASC' | 'DESC' + severity_number?: string + start_date?: string + end_date?: string + sort_by?: string + sort_order?: 'ASC' | 'DESC' search?: Record }) { const queryParts: string[] = [] const queryParams: Record = { service_name: resource } queryParts.push('service_name = {service_name:String}') + queryParams.service_name = resource - if (level) { - queryParts.push('severity_number = {level:UInt8}') - queryParams.level = level + if (severity_number) { + queryParts.push('severity_number = {severity_number:UInt8}') + queryParams.severity_number = severity_number } - if (startDate) { - queryParts.push('timestamp >= {startDate:DateTime}') - queryParams.startDate = toClickhouseDateTime(startDate) + if (start_date) { + queryParts.push('timestamp >= {start_date:DateTime}') + queryParams.start_date = new Date(start_date) } - if (endDate) { - queryParts.push('timestamp <= {endDate:DateTime}') - queryParams.endDate = toClickhouseDateTime(endDate) + if (end_date) { + queryParts.push('timestamp <= {end_date:DateTime}') + queryParams.end_date = new Date(end_date) } if (search) { @@ -151,14 +145,13 @@ async function getLogs({ SELECT * FROM logs WHERE ${queryParts.join(' AND ')} - ${sortBy ? `ORDER BY ${sortBy} ${sortOrder || 'DESC'}` : ''} + ${sort_by ? `ORDER BY ${sort_by} ${sort_order || 'DESC'}` : ''} LIMIT 1000 ` try { const resultSet = await client.query({ query, - query_params: queryParams, format: 'JSON', }) diff --git a/api/routes.ts b/api/routes.ts index 3658d48..58f838d 100644 --- a/api/routes.ts +++ b/api/routes.ts @@ -4,7 +4,6 @@ import { handleGoogleCallback, initiateGoogleAuth } from './auth.ts' import { DeploymentDef, DeploymentsCollection, - ProjectDef, ProjectsCollection, TeamDef, TeamsCollection, @@ -63,6 +62,16 @@ const deploymentOutput = OBJ({ token: optional(STR('The deployment token')), }) +const projectOutput = OBJ({ + slug: STR('The unique identifier for the project'), + name: STR('The name of the project'), + teamId: STR('The ID of the team that owns the project'), + isPublic: BOOL('Is the project public?'), + repositoryUrl: optional(STR('The URL of the project repository')), + createdAt: optional(NUM('The creation date of the project')), + updatedAt: optional(NUM('The last update date of the project')), +}) + const defs = { 'GET/api/health': route({ fn: () => new Response('OK'), @@ -171,7 +180,7 @@ const defs = { 'GET/api/projects': route({ authorize: withUserSession, fn: () => ProjectsCollection.values().toArray(), - output: ARR(ProjectDef, 'List of projects'), + output: ARR(projectOutput, 'List of projects'), description: 'Get all projects', }), 'POST/api/project': route({ @@ -184,7 +193,7 @@ const defs = { isPublic: BOOL('Is the project public?'), repositoryUrl: optional(STR('The URL of the project repository')), }, 'Create a new project'), - output: ProjectDef, + output: projectOutput, description: 'Create a new project', }), 'GET/api/project': route({ @@ -195,7 +204,7 @@ const defs = { return project }, input: OBJ({ slug: STR('The slug of the project') }), - output: ProjectDef, + output: projectOutput, description: 'Get a project by ID', }), 'PUT/api/project': route({ @@ -208,7 +217,7 @@ const defs = { isPublic: BOOL('Is the project public?'), repositoryUrl: optional(STR('The URL of the project repository')), }), - output: ProjectDef, + output: projectOutput, description: 'Update a project by ID', }), 'DELETE/api/project': route({ @@ -347,13 +356,11 @@ const defs = { 'GET/api/logs': route({ authorize: withUserSession, fn: (ctx, params) => { - const deployments = DeploymentsCollection.filter((d) => - d.projectId === params.resource - ) - if (deployments.length === 0) { - throw respond.NotFound({ message: 'Deployments not found' }) + const deployment = DeploymentsCollection.get(params.resource) + if (!deployment) { + throw respond.NotFound({ message: 'Deployment not found' }) } - const project = ProjectsCollection.get(deployments[0].projectId) + const project = ProjectsCollection.get(deployment.projectId) if (!project) throw respond.NotFound({ message: 'Project not found' }) if (!project.isPublic && !ctx.user?.isAdmin) { const team = TeamsCollection.find((t) => t.teamId === project.teamId) @@ -361,11 +368,12 @@ const defs = { throw respond.Forbidden({ message: 'Access to project logs denied' }) } } + return getLogs(params) }, input: OBJ({ resource: STR('The resource to fetch logs for'), - level: optional(STR('The log level to filter by')), + severity_number: optional(STR('The log level to filter by')), start_date: optional(STR('The start date for the date range filter')), end_date: optional(STR('The end date for the date range filter')), sort_by: optional(STR('The field to sort by')), diff --git a/api/user.ts b/api/user.ts index 4ed035a..a6243e7 100644 --- a/api/user.ts +++ b/api/user.ts @@ -36,7 +36,7 @@ export async function decryptMessage(encryptedMessage: string) { const key = await crypto.subtle.importKey( 'raw', - decodeBase64Url(SECRET), + decodeBase64Url(SECRET) as ArrayBufferView, { name: 'AES-GCM' }, true, // The key should be extractable ['encrypt', 'decrypt'], diff --git a/deno.json b/deno.json index 43dd3d5..7b21ecc 100644 --- a/deno.json +++ b/deno.json @@ -50,6 +50,11 @@ "singleQuote": true, "exclude": ["dist/", "metafile.json"] }, + "lint": { + "rules": { + "exclude": ["no-import-prefix", "no-unversioned-import"] + } + }, "nodeModulesDir": "auto", "compilerOptions": { "jsx": "react-jsx", diff --git a/tasks/clickhouse.ts b/tasks/clickhouse.ts index ee1ab1a..6c8f019 100644 --- a/tasks/clickhouse.ts +++ b/tasks/clickhouse.ts @@ -19,7 +19,7 @@ if (import.meta.main) { span_id FixedString(16), severity_number UInt8, -- derived column, computed by DB from severity_number - severity_text String MATERIALIZED CASE + severity_text LowCardinality(String) MATERIALIZED CASE WHEN severity_number > 4 AND severity_number <= 8 THEN 'DEBUG' WHEN severity_number > 8 AND severity_number <= 12 THEN 'INFO' WHEN severity_number > 12 AND severity_number <= 16 THEN 'WARN' @@ -29,7 +29,7 @@ if (import.meta.main) { -- Often empty, but kept for OTEL spec compliance body Nullable(String), attributes JSON, - event_name String + event_name LowCardinality(String) ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(timestamp) diff --git a/web/pages/ProjectPage.tsx b/web/pages/ProjectPage.tsx index ca4e66a..5b11e87 100644 --- a/web/pages/ProjectPage.tsx +++ b/web/pages/ProjectPage.tsx @@ -9,7 +9,6 @@ import { TasksPage } from './project/TaskPage.tsx' import { SettingsPage } from './project/SettingsPage.tsx' import { api } from '../lib/api.ts' import { effect } from '@preact/signals' -import { Deployment } from '../../api/schema.ts' const pageMap = { deployment: DeploymentPage, @@ -17,32 +16,16 @@ const pageMap = { settings: SettingsPage, } -export const deployments: Deployment[] = [ - { - projectId: 'my-awesome-project', - url: 'https://my-app.fly.dev', - logsEnabled: true, - databaseEnabled: false, - sqlEndpoint: undefined, - sqlToken: undefined, - }, - { - projectId: 'my-awesome-project', - url: 'https://staging.my-app.fly.dev', - logsEnabled: false, - databaseEnabled: true, - sqlEndpoint: 'https://db.my-app.com/sql', - sqlToken: 'super-secret-token', - }, -] +export const deployments = api['GET/api/project/deployments'].signal() const project = api['GET/api/project'].signal() effect(() => { const path = url.path - const projectSlug = path.split('/')[2] - if (projectSlug) { - project.fetch({ slug: projectSlug }) + const slug = path.split('/')[2] + if (slug) { + project.fetch({ slug }) + deployments.fetch({ project: slug }) } }) diff --git a/web/pages/ProjectsPage.tsx b/web/pages/ProjectsPage.tsx index 5027c93..96a72f6 100644 --- a/web/pages/ProjectsPage.tsx +++ b/web/pages/ProjectsPage.tsx @@ -15,11 +15,12 @@ import { Dialog, DialogModal } from '../components/Dialog.tsx' import { url } from '../lib/router.tsx' import { JSX } from 'preact' import { user } from '../lib/session.ts' -import { api } from '../lib/api.ts' +import { api, ApiOutput } from '../lib/api.ts' import { PageContent, PageHeader, PageLayout } from '../components/Layout.tsx' -import type { Project as ApiProject, Team, User } from '../../api/schema.ts' -type Project = ApiProject +type Project = ApiOutput['GET/api/projects'][number] +type User = ApiOutput['GET/api/users'][number] +type Team = ApiOutput['GET/api/teams'][number] const users = api['GET/api/users'].signal() users.fetch() @@ -199,8 +200,11 @@ const EmptyState = ( ) const ProjectCard = ( - { project, team }: { project: ApiProject; team: Team }, + { project, team }: { project: Project; team: Team }, ) => { + console.log(project) + console.log(team) + const isMember = team.teamMembers.includes(user.data?.userEmail || '') return ( ( ) -const TeamProjectsRow = ({ project }: { project: ApiProject }) => ( +const TeamProjectsRow = ({ project }: { project: Project }) => ( {project.name} {project.slug} diff --git a/web/pages/project/DeploymentPage.tsx b/web/pages/project/DeploymentPage.tsx index 64c3427..941ff20 100644 --- a/web/pages/project/DeploymentPage.tsx +++ b/web/pages/project/DeploymentPage.tsx @@ -3,8 +3,11 @@ import { url } from '../../lib/router.tsx' import { A, navigate } from '../../lib/router.tsx' import { Calendar, Database, Logs, Search } from 'lucide-preact' -import { Deployment, Project } from '../../../api/schema.ts' import { deployments } from '../ProjectPage.tsx' +import { ApiOutput } from '../../lib/api.ts' + +type Deployment = ApiOutput['GET/api/project/deployments'][number] +type Project = ApiOutput['GET/api/projects'][number] const DeploymentCard = ({ dep }: { dep: Deployment }) => { const created = dep.createdAt ? new Date(dep.createdAt) : null @@ -75,7 +78,7 @@ export const DeploymentPage = ({}: { project: Project }) => { const { deployment, deptab } = url.params const selectedDeployment = deployment - ? deployments.find((d) => d.url === deployment) + ? deployments.data?.find((d) => d.url === deployment) : null if ( @@ -121,7 +124,7 @@ export const DeploymentPage = ({}: { project: Project }) => { class='select' > - {deployments.map((dep) => ( + {deployments.data?.map((dep) => ( @@ -169,7 +172,7 @@ export const DeploymentPage = ({}: { project: Project }) => { ? tab : (
- {deployments.map((dep) => ( + {deployments.data?.map((dep) => ( ))}
diff --git a/web/pages/project/SettingsPage.tsx b/web/pages/project/SettingsPage.tsx index 8cf3d83..fbc431c 100644 --- a/web/pages/project/SettingsPage.tsx +++ b/web/pages/project/SettingsPage.tsx @@ -1,13 +1,16 @@ -import { Deployment, Project, User } from '../../../api/schema.ts' import { PageContent, PageHeader } from '../../components/Layout.tsx' import { Button, Card, Input, Switch } from '../../components/forms.tsx' import { useSignal } from '@preact/signals' import { navigate, url } from '../../lib/router.tsx' import { JSX } from 'preact' -import { api } from '../../lib/api.ts' +import { api, ApiOutput } from '../../lib/api.ts' import { deployments } from '../ProjectPage.tsx' import { user } from '../../lib/session.ts' +type Project = ApiOutput['GET/api/projects'][number] +type Deployment = ApiOutput['GET/api/project/deployments'][number] +type User = ApiOutput['GET/api/users'][number] + const users = api['GET/api/users'].signal() users.fetch() @@ -244,10 +247,10 @@ export const SettingsPage = ({ project }: { project: Project }) => { : action === 'edit' && id ? ( d.url === id)} + deployment={deployments.data?.find((d) => d.url === id)} /> ) - : + : ) : view === 'users' ?