diff --git a/.env.test b/.env.test index de70d63..2fabf76 100644 --- a/.env.test +++ b/.env.test @@ -2,4 +2,8 @@ APP_ENV=test PORT=3021 GOOGLE_CLIENT_ID=...test.apps.googleusercontent.com CLIENT_SECRET=GOC...test -REDIRECT_URI=http://localhost:7737/api/auth/google \ No newline at end of file +REDIRECT_URI=http://localhost:7737/api/auth/google + +CLICKHOUSE_HOST=http://localhost:8443 +CLICKHOUSE_USER=default +CLICKHOUSE_PASSWORD=token_pass \ No newline at end of file diff --git a/api/click-house-client.ts b/api/click-house-client.ts new file mode 100644 index 0000000..03b1ae9 --- /dev/null +++ b/api/click-house-client.ts @@ -0,0 +1,166 @@ +import { createClient } from 'npm:@clickhouse/client' +import { + CLICKHOUSE_HOST, + CLICKHOUSE_PASSWORD, + CLICKHOUSE_USER, +} from './lib/env.ts' +import { respond } from './lib/response.ts' +import { log } from './lib/log.ts' +import { ARR, NUM, OBJ, optional, STR, UNION } from './lib/validator.ts' +import { Asserted } from './lib/router.ts' + +const LogSchema = OBJ({ + timestamp: NUM('The timestamp of the log event'), + trace_id: NUM('A float64 representation of the trace ID'), + span_id: optional(NUM('A float64 representation of the span ID')), + severity_number: NUM('The severity number of the log event'), + attributes: optional(OBJ({}, 'A map of attributes')), + event_name: STR('The name of the event'), + service_version: optional(STR('Service version')), + service_instance_id: optional(STR('Service instance ID')), +}, 'A log event') +const LogsInputSchema = UNION( + LogSchema, + ARR(LogSchema, 'An array of log events'), +) + +type Log = Asserted +type LogsInput = Asserted + +const client = createClient({ + url: CLICKHOUSE_HOST, + username: CLICKHOUSE_USER, + password: CLICKHOUSE_PASSWORD, + compression: { + request: true, + response: true, + }, + clickhouse_settings: { + date_time_input_format: 'best_effort', + }, +}) + +const numberToHex128 = (() => { + const alphabet = new TextEncoder().encode('0123456789abcdef') + const output = new Uint8Array(16) + const view = new DataView(new Uint8Array(8).buffer) + const dec = new TextDecoder() + return (id: number) => { + view.setFloat64(0, id, false) + let i = -1 + while (++i < 8) { + const x = view.getUint8(i) + output[i * 2] = alphabet[x >> 4] + output[i * 2 + 1] = alphabet[x & 0xF] + } + return dec.decode(output) + } +})() + +async function insertLogs( + service_name: string, + data: LogsInput, +) { + const logsToInsert = Array.isArray(data) ? data : [data] + if (logsToInsert.length === 0) throw respond.NoContent() + + const rows = logsToInsert.map((log) => { + const traceHex = numberToHex128(log.trace_id) + const spanHex = numberToHex128(log.span_id ?? log.trace_id) + return { + ...log, + timestamp: new Date(log.timestamp), + attributes: log.attributes ?? {}, + service_name: service_name, + trace_id: traceHex, + span_id: spanHex, + } + }) + + log.debug('Inserting logs into ClickHouse', { rows }) + + try { + await client.insert({ table: 'logs', values: rows, format: 'JSONEachRow' }) + return respond.OK() + } catch (error) { + log.error('Error inserting logs into ClickHouse:', { error }) + throw respond.InternalServerError() + } +} + +async function getLogs({ + resource, + severity_number, + start_date, + end_date, + sort_by, + sort_order, + search, +}: { + resource: string + severity_number?: string + start_date?: string + end_date?: string + sort_by?: string + sort_order?: 'ASC' | 'DESC' + search?: Record +}) { + const queryParts: string[] = [] + const queryParams: Record = { service_name: resource } + + queryParts.push('service_name = {service_name:String}') + queryParams.service_name = resource + + if (severity_number) { + queryParts.push('severity_number = {severity_number:UInt8}') + queryParams.severity_number = severity_number + } + + if (start_date) { + queryParts.push('timestamp >= {start_date:DateTime}') + queryParams.start_date = new Date(start_date) + } + + if (end_date) { + queryParts.push('timestamp <= {end_date:DateTime}') + queryParams.end_date = new Date(end_date) + } + + if (search) { + if (search.trace_id) { + queryParts.push('trace_id = {trace_id:String}') + queryParams.trace_id = search.trace_id + } + if (search.span_id) { + queryParts.push('span_id = {span_id:String}') + queryParams.span_id = search.span_id + } + if (search.event_name) { + queryParts.push('event_name = {event_name:String}') + queryParams.event_name = search.event_name + } + } + + const query = ` + SELECT * + FROM logs + WHERE ${queryParts.join(' AND ')} + ${sort_by ? `ORDER BY ${sort_by} ${sort_order || 'DESC'}` : ''} + LIMIT 1000 + ` + + try { + const resultSet = await client.query({ + query, + query_params: queryParams, + format: 'JSON', + }) + + return (await resultSet.json()).data + } catch (error) { + log.error('Error querying logs from ClickHouse:', { error }) + throw respond.InternalServerError() + } +} + +export { client, getLogs, insertLogs, LogSchema, LogsInputSchema } diff --git a/api/lib/context.ts b/api/lib/context.ts index c42df24..40fb18b 100644 --- a/api/lib/context.ts +++ b/api/lib/context.ts @@ -19,6 +19,7 @@ export type RequestContext = { readonly user: User | undefined readonly trace: number readonly span: number | undefined + resource: string | undefined } // we set default values so we don't have to check everytime if they exists @@ -33,6 +34,7 @@ export const makeContext = ( cookies: {}, user: undefined, span: undefined, + resource: undefined, url, req, ...extra, diff --git a/api/lib/env.ts b/api/lib/env.ts index e56d5e5..b52700a 100644 --- a/api/lib/env.ts +++ b/api/lib/env.ts @@ -26,3 +26,16 @@ export const ORIGIN = new URL(REDIRECT_URI).origin export const SECRET = env.SECRET || 'iUokBru8WPSMAuMspijlt7F-Cnpqyg84F36b1G681h0' + +export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST +if (!CLICKHOUSE_HOST) { + throw Error('CLICKHOUSE_HOST: field required in the env') +} +export const CLICKHOUSE_USER = env.CLICKHOUSE_USER +if (!CLICKHOUSE_USER) { + throw Error('CLICKHOUSE_USER: field required in the env') +} +export const CLICKHOUSE_PASSWORD = env.CLICKHOUSE_PASSWORD +if (!CLICKHOUSE_PASSWORD) { + throw Error('CLICKHOUSE_PASSWORD: field required in the env') +} diff --git a/api/lib/json_store.ts b/api/lib/json_store.ts index 0ae4d4c..497219d 100644 --- a/api/lib/json_store.ts +++ b/api/lib/json_store.ts @@ -32,10 +32,10 @@ const batch = async ( await Promise.all(pool) } -export type BaseRecord = { createdAt?: number; updatedAt?: number } +export type BaseRecord = { createdAt: number; updatedAt: number } export async function createCollection< - T extends Record & BaseRecord, + T extends Record, K extends keyof T, >({ name, primaryKey }: CollectionOptions) { const dir = join(DB_DIR, name) @@ -66,34 +66,34 @@ export async function createCollection< const id = data[primaryKey] if (!id) throw Error(`Missing primary key ${primaryKey}`) if (records.has(id)) throw Error(`${id} already exists`) - Object.assign(data, { createdAt: Date.now() }) + Object.assign(data, { createdAt: Date.now(), updatedAt: Date.now() }) records.set(id, data) await saveRecord(data) - return data + return data as T & BaseRecord }, [Symbol.iterator]: () => records[Symbol.iterator], keys: () => records.keys(), - values: () => records.values(), - entries: () => records.entries(), - get: (id: T[K]) => records.get(id), + values: () => records.values() as MapIterator, + entries: () => records.entries() as MapIterator<[T[K], T & BaseRecord]>, + get: (id: T[K]) => records.get(id) as T & BaseRecord | undefined, assert: (id: T[K]) => { const match = records.get(id) - if (match) return match + if (match) return match as T & BaseRecord throw new Deno.errors.NotFound(`record ${id} not found`) }, find: (predicate: (record: T) => unknown) => - records.values().find(predicate), + records.values().find(predicate) as T & BaseRecord | undefined, filter: (predicate: (record: T) => unknown) => - records.values().filter(predicate).toArray(), + records.values().filter(predicate).toArray() as (T & BaseRecord)[], async update(id: T[K], changes: Partial>) { const record = records.get(id) if (!record) throw new Deno.errors.NotFound(`record ${id} not found`) - const updated = { ...record, ...changes, _updatedAt: Date.now() } as T + const updated = { ...record, ...changes, updatedAt: Date.now() } as T records.set(id, updated) await saveRecord(updated) - return updated + return updated as T & BaseRecord }, async delete(id: T[K]) { diff --git a/api/lib/validator.ts b/api/lib/validator.ts index 3c35963..993c126 100644 --- a/api/lib/validator.ts +++ b/api/lib/validator.ts @@ -29,6 +29,15 @@ type DefList = { assert: (value: unknown) => T[number] } +type DefUnion = { + type: 'union' + of: T + report: Validator> + optional?: boolean + description?: string + assert: (value: unknown) => ReturnType +} + type DefObject> = { type: 'object' properties: { [K in keyof T]: T[K] } @@ -69,6 +78,7 @@ export type DefBase = | DefArray | DefObject> | DefList + | DefUnion type OptionalAssert = ( value: unknown, @@ -258,6 +268,32 @@ export const LIST = ( description, }) +export const UNION = (...types: T): DefUnion => ({ + type: 'union', + of: types, + report: (value: unknown, path: (string | number)[] = []) => { + const failures: ValidatorFailure>[] = [] + for (const type of types) { + const result = type.report(value, path) + if (result.length === 0) return [] + failures.push(...result) + } + return failures + }, + assert: (value: unknown): ReturnType => { + for (const type of types) { + try { + return type.assert(value) + } catch { + // Ignore + } + } + throw new Error( + `Invalid value. Expected one of: ${types.map((t) => t.type).join(', ')}`, + ) + }, +}) + // const Article = OBJ({ // id: NUM("Unique identifier for the article"), // title: STR("Title of the article"), diff --git a/api/routes.ts b/api/routes.ts index 3f142a6..58f838d 100644 --- a/api/routes.ts +++ b/api/routes.ts @@ -2,7 +2,8 @@ import { makeRouter, route } from '/api/lib/router.ts' import type { RequestContext } from '/api/lib/context.ts' import { handleGoogleCallback, initiateGoogleAuth } from './auth.ts' import { - ProjectDef, + DeploymentDef, + DeploymentsCollection, ProjectsCollection, TeamDef, TeamsCollection, @@ -10,10 +11,18 @@ import { UserDef, UsersCollection, } from './schema.ts' -import { ARR, BOOL, OBJ, optional, STR } from './lib/validator.ts' +import { ARR, BOOL, LIST, NUM, OBJ, optional, STR } from './lib/validator.ts' import { respond } from './lib/response.ts' import { deleteCookie } from 'jsr:@std/http/cookie' import { getPicture } from '/api/picture.ts' +import { + getLogs, + insertLogs, + LogSchema, + LogsInputSchema, +} from './click-house-client.ts' +import { decryptMessage, encryptMessage } from './user.ts' +import { log } from './lib/log.ts' const withUserSession = ({ user }: RequestContext) => { if (!user) throw Error('Missing user session') @@ -23,6 +32,46 @@ const withAdminSession = ({ user }: RequestContext) => { if (!user || !user.isAdmin) throw Error('Admin access required') } +const withDeploymentSession = async (ctx: RequestContext) => { + const token = ctx.req.headers.get('Authorization')?.replace(/^Bearer /i, '') + if (!token) throw respond.Unauthorized({ message: 'Missing token' }) + try { + const message = await decryptMessage(token) + if (!message) throw respond.Unauthorized({ message: 'Invalid token' }) + const data = JSON.parse(message) + const dep = DeploymentsCollection.get(data?.url) + if (!dep || dep.tokenSalt !== data?.tokenSalt) { + throw respond.Unauthorized({ message: 'Invalid token' }) + } + ctx.resource = dep?.url + } catch (error) { + log.error('Error validating deployment token:', { error }) + throw respond.Unauthorized({ message: 'Invalid token' }) + } +} + +const deploymentOutput = OBJ({ + projectId: STR('The ID of the project'), + url: STR('The URL of the deployment'), + logsEnabled: BOOL('Whether logging is enabled'), + databaseEnabled: BOOL('Whether the database is enabled'), + sqlEndpoint: optional(STR('The SQL endpoint')), + sqlToken: optional(STR('The SQL token')), + createdAt: optional(NUM('The creation date of the deployment')), + updatedAt: optional(NUM('The last update date of the deployment')), + token: optional(STR('The deployment token')), +}) + +const projectOutput = OBJ({ + slug: STR('The unique identifier for the project'), + name: STR('The name of the project'), + teamId: STR('The ID of the team that owns the project'), + isPublic: BOOL('Is the project public?'), + repositoryUrl: optional(STR('The URL of the project repository')), + createdAt: optional(NUM('The creation date of the project')), + updatedAt: optional(NUM('The last update date of the project')), +}) + const defs = { 'GET/api/health': route({ fn: () => new Response('OK'), @@ -131,7 +180,7 @@ const defs = { 'GET/api/projects': route({ authorize: withUserSession, fn: () => ProjectsCollection.values().toArray(), - output: ARR(ProjectDef, 'List of projects'), + output: ARR(projectOutput, 'List of projects'), description: 'Get all projects', }), 'POST/api/project': route({ @@ -144,7 +193,7 @@ const defs = { isPublic: BOOL('Is the project public?'), repositoryUrl: optional(STR('The URL of the project repository')), }, 'Create a new project'), - output: ProjectDef, + output: projectOutput, description: 'Create a new project', }), 'GET/api/project': route({ @@ -155,7 +204,7 @@ const defs = { return project }, input: OBJ({ slug: STR('The slug of the project') }), - output: ProjectDef, + output: projectOutput, description: 'Get a project by ID', }), 'PUT/api/project': route({ @@ -168,7 +217,7 @@ const defs = { isPublic: BOOL('Is the project public?'), repositoryUrl: optional(STR('The URL of the project repository')), }), - output: ProjectDef, + output: projectOutput, description: 'Update a project by ID', }), 'DELETE/api/project': route({ @@ -183,6 +232,159 @@ const defs = { output: BOOL('Indicates if the project was deleted'), description: 'Delete a project by ID', }), + 'GET/api/project/deployments': route({ + authorize: withUserSession, + fn: (_ctx, { project }) => { + const deployments = DeploymentsCollection.filter((d) => + d.projectId === project + ) + if (!deployments.length) { + throw respond.NotFound({ message: 'Deployments not found' }) + } + return deployments.map(({ tokenSalt: _, ...d }) => { + return { + ...d, + token: undefined, + sqlToken: undefined, + sqlEndpoint: undefined, + } + }) + }, + input: OBJ({ project: STR('The ID of the project') }), + output: ARR(deploymentOutput, 'List of deployments'), + description: 'Get deployments by project ID', + }), + 'GET/api/deployment': route({ + authorize: withAdminSession, + fn: async (_ctx, url) => { + const dep = DeploymentsCollection.get(url) + if (!dep) throw respond.NotFound() + const { tokenSalt, ...deployment } = dep + const token = await encryptMessage( + JSON.stringify({ url: deployment.url, tokenSalt }), + ) + return { + ...deployment, + token, + } + }, + input: STR(), + output: deploymentOutput, + description: 'Get a deployment by ID', + }), + 'POST/api/deployment': route({ + authorize: withAdminSession, + fn: async (_ctx, input) => { + const tokenSalt = performance.now().toString() + const { tokenSalt: _, ...deployment } = await DeploymentsCollection + .insert({ + ...input, + tokenSalt, + }) + const token = await encryptMessage( + JSON.stringify({ url: deployment.url, tokenSalt }), + ) + return { + ...deployment, + token, + } + }, + input: DeploymentDef, + output: deploymentOutput, + description: 'Create a new deployment', + }), + 'PUT/api/deployment': route({ + authorize: withAdminSession, + fn: async (_ctx, input) => { + const { tokenSalt, ...deployment } = await DeploymentsCollection + .update(input.url, input) + const token = await encryptMessage( + JSON.stringify({ url: deployment.url, tokenSalt }), + ) + return { + ...deployment, + token, + } + }, + input: DeploymentDef, + output: deploymentOutput, + description: 'Update a deployment by ID', + }), + 'GET/api/deployment/token/regenerate': route({ + authorize: withAdminSession, + fn: async (_ctx, { url }) => { + console.log('Regenerating token for deployment:', { url }) + const dep = DeploymentsCollection.get(url) + console.log('Regenerating token for deployment:', { dep }) + if (!dep) throw respond.NotFound() + const tokenSalt = performance.now().toString() + + const { tokenSalt: _, ...deployment } = await DeploymentsCollection + .update(url, { ...dep, tokenSalt }) + const token = await encryptMessage( + JSON.stringify({ url: deployment.url, tokenSalt }), + ) + return { + ...deployment, + token, + } + }, + input: OBJ({ url: STR('The URL of the deployment') }), + output: deploymentOutput, + description: 'Regenerate a deployment token', + }), + 'DELETE/api/deployment': route({ + authorize: withAdminSession, + fn: async (_ctx, input) => { + const dep = DeploymentsCollection.get(input) + if (!dep) throw respond.NotFound() + await DeploymentsCollection.delete(input) + return respond.NoContent() + }, + input: STR(), + description: 'Delete a deployment', + }), + 'POST/api/logs': route({ + authorize: withDeploymentSession, + fn: (ctx, logs) => { + if (!ctx.resource) throw respond.InternalServerError() + return insertLogs(ctx.resource, logs) + }, + input: LogsInputSchema, + description: 'Insert logs into ClickHouse NB: a Bearer token is required', + }), + 'GET/api/logs': route({ + authorize: withUserSession, + fn: (ctx, params) => { + const deployment = DeploymentsCollection.get(params.resource) + if (!deployment) { + throw respond.NotFound({ message: 'Deployment not found' }) + } + const project = ProjectsCollection.get(deployment.projectId) + if (!project) throw respond.NotFound({ message: 'Project not found' }) + if (!project.isPublic && !ctx.user?.isAdmin) { + const team = TeamsCollection.find((t) => t.teamId === project.teamId) + if (!team?.teamMembers.includes(ctx.user?.userEmail || '')) { + throw respond.Forbidden({ message: 'Access to project logs denied' }) + } + } + + return getLogs(params) + }, + input: OBJ({ + resource: STR('The resource to fetch logs for'), + severity_number: optional(STR('The log level to filter by')), + start_date: optional(STR('The start date for the date range filter')), + end_date: optional(STR('The end date for the date range filter')), + sort_by: optional(STR('The field to sort by')), + sort_order: optional( + LIST(['ASC', 'DESC'], 'The sort order (ASC or DESC)'), + ), + // search: optional(OBJ({}, 'A map of fields to search by')), + }), + output: ARR(LogSchema, 'List of logs'), + description: 'Get logs from ClickHouse', + }), } as const export type RouteDefinitions = typeof defs diff --git a/api/schema.ts b/api/schema.ts index e03a00d..78da210 100644 --- a/api/schema.ts +++ b/api/schema.ts @@ -1,6 +1,6 @@ import { ARR, BOOL, OBJ, optional, STR } from './lib/validator.ts' import { Asserted } from './lib/router.ts' -import { BaseRecord, createCollection } from './lib/json_store.ts' +import { createCollection } from './lib/json_store.ts' export const UserDef = OBJ({ userEmail: STR('The user email address'), @@ -8,7 +8,7 @@ export const UserDef = OBJ({ userPicture: optional(STR('The user profile picture URL')), isAdmin: BOOL('Is the user an admin?'), }, 'The user schema definition') -export type User = Asserted & BaseRecord +export type User = Asserted export const TeamDef = OBJ({ teamId: STR('The unique identifier for the team'), @@ -18,7 +18,7 @@ export const TeamDef = OBJ({ 'The list of user emails who are members of the team', ), }, 'The team schema definition') -export type Team = Asserted & BaseRecord +export type Team = Asserted export const ProjectDef = OBJ({ slug: STR('The unique identifier for the project'), @@ -27,7 +27,7 @@ export const ProjectDef = OBJ({ isPublic: BOOL('Is the project public?'), repositoryUrl: optional(STR('The URL of the project repository')), }, 'The project schema definition') -export type Project = Asserted & BaseRecord +export type Project = Asserted export const DeploymentDef = OBJ({ projectId: STR('The ID of the project this deployment belongs to'), @@ -37,7 +37,7 @@ export const DeploymentDef = OBJ({ sqlEndpoint: optional(STR('The SQL execution endpoint for the database')), sqlToken: optional(STR('The security token for the SQL endpoint')), }, 'The deployment schema definition') -export type Deployment = Asserted & BaseRecord +export type Deployment = Asserted export const UsersCollection = await createCollection( { name: 'users', primaryKey: 'userEmail' }, diff --git a/api/server.ts b/api/server.ts index b594ada..3460aef 100644 --- a/api/server.ts +++ b/api/server.ts @@ -64,6 +64,7 @@ export const fetch = async (req: Request) => { trace: cookies.trace ? Number(cookies.trace) : now(), user: await decodeSession(cookies.session), span: now(), + resource: undefined, } const res = await requestContext.run(ctx, handleRequest, ctx) diff --git a/api/user.ts b/api/user.ts index b314471..a6243e7 100644 --- a/api/user.ts +++ b/api/user.ts @@ -7,7 +7,7 @@ const encoder = new TextEncoder() const decoder = new TextDecoder() const IV_SIZE = 12 // Initialization vector (12 bytes for AES-GCM) -async function encryptMessage(message: string) { +export async function encryptMessage(message: string) { const iv = crypto.getRandomValues(new Uint8Array(IV_SIZE)) const encryptedMessage = await crypto.subtle.encrypt( { name: 'AES-GCM', iv }, @@ -22,7 +22,7 @@ async function encryptMessage(message: string) { } // Decrypting a message -async function decryptMessage(encryptedMessage: string) { +export async function decryptMessage(encryptedMessage: string) { const encryptedData = decodeBase64Url(encryptedMessage) const iv = encryptedData.slice(0, IV_SIZE) const decryptedMessage = await crypto.subtle.decrypt( @@ -36,7 +36,7 @@ async function decryptMessage(encryptedMessage: string) { const key = await crypto.subtle.importKey( 'raw', - decodeBase64Url(SECRET), + decodeBase64Url(SECRET) as ArrayBufferView, { name: 'AES-GCM' }, true, // The key should be extractable ['encrypt', 'decrypt'], diff --git a/deno.json b/deno.json index a2b08fb..7b21ecc 100644 --- a/deno.json +++ b/deno.json @@ -2,13 +2,15 @@ "tasks": { "api:dev": "deno run -A --env-file=.env.dev api/server.ts --env=dev --port=3021", "vite:dev": "deno run -A --env-file=.env.dev tasks/vite.js --env=dev", - "dev": { "dependencies": ["api:dev", "vite:dev"] }, + "clickhouse:dev": "deno run -A --env-file=.env.dev tasks/clickhouse.ts --env=dev", + "dev": { "dependencies": ["clickhouse:dev", "api:dev", "vite:dev"] }, "seed": "deno run -A --env-file=.env.dev tasks/seed.ts", "dev:with-seed": "deno task seed && deno task dev", "api:prod": "deno compile -A --env-file=.env.prod --no-check --output dist/api --target x86_64-unknown-linux-gnu --include dist/web api/server.ts --env=prod", "vite:prod": "deno run -A --env-file=.env.prod tasks/vite.js --build --env=prod", + "clickhouse:prod": "deno run -A --env-file=.env.prod tasks/clickhouse.ts --env=prod", "prod": "deno task vite:prod && deno task api:prod", - "start:prod": "dist/api --env=prod", + "start:prod": "deno task clickhouse:prod && dist/api --env=prod", "fmt": "deno fmt", "lint": "deno lint", "check": "deno check", @@ -34,6 +36,7 @@ "preact": "npm:preact@^10.26.9", "@preact/preset-vite": "npm:@preact/preset-vite@^2.10.2", "@preact/signals": "npm:@preact/signals", + "@clickhouse/client": "npm:@clickhouse/client", "@tailwindcss/vite": "npm:@tailwindcss/vite@^4.1.11", "tailwindcss": "npm:tailwindcss@^4.1.11", "daisyui": "npm:daisyui@^5.0.46", @@ -47,6 +50,11 @@ "singleQuote": true, "exclude": ["dist/", "metafile.json"] }, + "lint": { + "rules": { + "exclude": ["no-import-prefix", "no-unversioned-import"] + } + }, "nodeModulesDir": "auto", "compilerOptions": { "jsx": "react-jsx", diff --git a/deno.lock b/deno.lock index 14d9f10..a074fbb 100644 --- a/deno.lock +++ b/deno.lock @@ -24,6 +24,7 @@ "jsr:@std/path@^1.1.1": "1.1.1", "jsr:@std/streams@^1.0.8": "1.0.8", "jsr:@std/testing@*": "1.0.15", + "npm:@clickhouse/client@*": "1.12.1", "npm:@preact/preset-vite@^2.10.2": "2.10.2_@babel+core@7.28.0_vite@7.0.4__picomatch@4.0.2__@types+node@22.15.15_preact@10.26.9_@types+node@22.15.15", "npm:@preact/signals@*": "2.2.1_preact@10.26.9", "npm:@tailwindcss/vite@^4.1.11": "4.1.11_vite@7.0.4__picomatch@4.0.2__@types+node@22.15.15_@types+node@22.15.15", @@ -278,6 +279,15 @@ "@babel/helper-validator-identifier" ] }, + "@clickhouse/client-common@1.12.1": { + "integrity": "sha512-ccw1N6hB4+MyaAHIaWBwGZ6O2GgMlO99FlMj0B0UEGfjxM9v5dYVYql6FpP19rMwrVAroYs/IgX2vyZEBvzQLg==" + }, + "@clickhouse/client@1.12.1": { + "integrity": "sha512-7ORY85rphRazqHzImNXMrh4vsaPrpetFoTWpZYueCO2bbO6PXYDXp/GQ4DgxnGIqbWB/Di1Ai+Xuwq2o7DJ36A==", + "dependencies": [ + "@clickhouse/client-common" + ] + }, "@emnapi/core@1.4.4": { "integrity": "sha512-A9CnAbC6ARNMKcIcrQwq6HeHCjpcBZ5wSx4U01WXCqEKlrzB9F9315WDNHkrs2xbx7YjjSxbUYxuN6EQzpcY2g==", "dependencies": [ @@ -1228,6 +1238,7 @@ "workspace": { "dependencies": [ "jsr:@std/assert@1", + "npm:@clickhouse/client@*", "npm:@preact/preset-vite@^2.10.2", "npm:@preact/signals@*", "npm:@tailwindcss/vite@^4.1.11", diff --git a/tasks/clickhouse.ts b/tasks/clickhouse.ts new file mode 100644 index 0000000..6c8f019 --- /dev/null +++ b/tasks/clickhouse.ts @@ -0,0 +1,46 @@ +import { client } from '/api/click-house-client.ts' +import { log } from '/api/lib/log.ts' + +if (import.meta.main) { + try { + await client.ping() + + await client.command({ + query: ` + CREATE TABLE IF NOT EXISTS logs ( + -- Flattened resource fields + service_name LowCardinality(String), + service_version LowCardinality(String), + service_instance_id String, + + timestamp DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), + observed_timestamp DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), + trace_id FixedString(16), + span_id FixedString(16), + severity_number UInt8, + -- derived column, computed by DB from severity_number + severity_text LowCardinality(String) MATERIALIZED CASE + WHEN severity_number > 4 AND severity_number <= 8 THEN 'DEBUG' + WHEN severity_number > 8 AND severity_number <= 12 THEN 'INFO' + WHEN severity_number > 12 AND severity_number <= 16 THEN 'WARN' + WHEN severity_number > 20 AND severity_number <= 24 THEN 'FATAL' + ELSE 'ERROR' + END, + -- Often empty, but kept for OTEL spec compliance + body Nullable(String), + attributes JSON, + event_name LowCardinality(String) + ) + ENGINE = MergeTree + PARTITION BY toYYYYMMDD(timestamp) + ORDER BY (service_name, timestamp, trace_id) + SETTINGS index_granularity = 8192, min_bytes_for_wide_part = 0; + `, + }) + + log.info('logs table is ready') + } catch (error) { + log.error('Error creating ClickHouse table:', { error }) + Deno.exit(1) + } +} diff --git a/web/pages/ProjectPage.tsx b/web/pages/ProjectPage.tsx index ca4e66a..5b11e87 100644 --- a/web/pages/ProjectPage.tsx +++ b/web/pages/ProjectPage.tsx @@ -9,7 +9,6 @@ import { TasksPage } from './project/TaskPage.tsx' import { SettingsPage } from './project/SettingsPage.tsx' import { api } from '../lib/api.ts' import { effect } from '@preact/signals' -import { Deployment } from '../../api/schema.ts' const pageMap = { deployment: DeploymentPage, @@ -17,32 +16,16 @@ const pageMap = { settings: SettingsPage, } -export const deployments: Deployment[] = [ - { - projectId: 'my-awesome-project', - url: 'https://my-app.fly.dev', - logsEnabled: true, - databaseEnabled: false, - sqlEndpoint: undefined, - sqlToken: undefined, - }, - { - projectId: 'my-awesome-project', - url: 'https://staging.my-app.fly.dev', - logsEnabled: false, - databaseEnabled: true, - sqlEndpoint: 'https://db.my-app.com/sql', - sqlToken: 'super-secret-token', - }, -] +export const deployments = api['GET/api/project/deployments'].signal() const project = api['GET/api/project'].signal() effect(() => { const path = url.path - const projectSlug = path.split('/')[2] - if (projectSlug) { - project.fetch({ slug: projectSlug }) + const slug = path.split('/')[2] + if (slug) { + project.fetch({ slug }) + deployments.fetch({ project: slug }) } }) diff --git a/web/pages/ProjectsPage.tsx b/web/pages/ProjectsPage.tsx index 5027c93..96a72f6 100644 --- a/web/pages/ProjectsPage.tsx +++ b/web/pages/ProjectsPage.tsx @@ -15,11 +15,12 @@ import { Dialog, DialogModal } from '../components/Dialog.tsx' import { url } from '../lib/router.tsx' import { JSX } from 'preact' import { user } from '../lib/session.ts' -import { api } from '../lib/api.ts' +import { api, ApiOutput } from '../lib/api.ts' import { PageContent, PageHeader, PageLayout } from '../components/Layout.tsx' -import type { Project as ApiProject, Team, User } from '../../api/schema.ts' -type Project = ApiProject +type Project = ApiOutput['GET/api/projects'][number] +type User = ApiOutput['GET/api/users'][number] +type Team = ApiOutput['GET/api/teams'][number] const users = api['GET/api/users'].signal() users.fetch() @@ -199,8 +200,11 @@ const EmptyState = ( ) const ProjectCard = ( - { project, team }: { project: ApiProject; team: Team }, + { project, team }: { project: Project; team: Team }, ) => { + console.log(project) + console.log(team) + const isMember = team.teamMembers.includes(user.data?.userEmail || '') return ( ( ) -const TeamProjectsRow = ({ project }: { project: ApiProject }) => ( +const TeamProjectsRow = ({ project }: { project: Project }) => ( {project.name} {project.slug} diff --git a/web/pages/project/DeploymentPage.tsx b/web/pages/project/DeploymentPage.tsx index 64c3427..941ff20 100644 --- a/web/pages/project/DeploymentPage.tsx +++ b/web/pages/project/DeploymentPage.tsx @@ -3,8 +3,11 @@ import { url } from '../../lib/router.tsx' import { A, navigate } from '../../lib/router.tsx' import { Calendar, Database, Logs, Search } from 'lucide-preact' -import { Deployment, Project } from '../../../api/schema.ts' import { deployments } from '../ProjectPage.tsx' +import { ApiOutput } from '../../lib/api.ts' + +type Deployment = ApiOutput['GET/api/project/deployments'][number] +type Project = ApiOutput['GET/api/projects'][number] const DeploymentCard = ({ dep }: { dep: Deployment }) => { const created = dep.createdAt ? new Date(dep.createdAt) : null @@ -75,7 +78,7 @@ export const DeploymentPage = ({}: { project: Project }) => { const { deployment, deptab } = url.params const selectedDeployment = deployment - ? deployments.find((d) => d.url === deployment) + ? deployments.data?.find((d) => d.url === deployment) : null if ( @@ -121,7 +124,7 @@ export const DeploymentPage = ({}: { project: Project }) => { class='select' > - {deployments.map((dep) => ( + {deployments.data?.map((dep) => ( @@ -169,7 +172,7 @@ export const DeploymentPage = ({}: { project: Project }) => { ? tab : (
- {deployments.map((dep) => ( + {deployments.data?.map((dep) => ( ))}
diff --git a/web/pages/project/SettingsPage.tsx b/web/pages/project/SettingsPage.tsx index 8cf3d83..fbc431c 100644 --- a/web/pages/project/SettingsPage.tsx +++ b/web/pages/project/SettingsPage.tsx @@ -1,13 +1,16 @@ -import { Deployment, Project, User } from '../../../api/schema.ts' import { PageContent, PageHeader } from '../../components/Layout.tsx' import { Button, Card, Input, Switch } from '../../components/forms.tsx' import { useSignal } from '@preact/signals' import { navigate, url } from '../../lib/router.tsx' import { JSX } from 'preact' -import { api } from '../../lib/api.ts' +import { api, ApiOutput } from '../../lib/api.ts' import { deployments } from '../ProjectPage.tsx' import { user } from '../../lib/session.ts' +type Project = ApiOutput['GET/api/projects'][number] +type Deployment = ApiOutput['GET/api/project/deployments'][number] +type User = ApiOutput['GET/api/users'][number] + const users = api['GET/api/users'].signal() users.fetch() @@ -244,10 +247,10 @@ export const SettingsPage = ({ project }: { project: Project }) => { : action === 'edit' && id ? ( d.url === id)} + deployment={deployments.data?.find((d) => d.url === id)} /> ) - : + : ) : view === 'users' ?