feat: Demo project

README.md

@@ -31,7 +31,7 @@
 
 [Swetrix](https://swetrix.com) is an open source, privacy-focused and cookie-less alternative to Google Analytics. Swetrix is designed to be easy to use while providing all the features you need to understand your website users. With Swetrix you can track your site's traffic, monitor your site's speed, analyse user sessions and page flows, see user flows and much more!
 
-Swetrix is made in the 🇬🇧 United Kingdom, and is hosted on Hetzner in 🇩🇪 Germany. Here's our [live demo with our own website statistics](https://swetrix.com/projects/STEzHcB1rALV).
+Swetrix is made in the 🇬🇧 United Kingdom, and is hosted on Hetzner in 🇩🇪 Germany. Here's our [live demo with our own website statistics](https://swetrix.com/demo).
 
 We are a bootstrapped company that is passionate about privacy and open source, funded solely by our subscribers.
 

backend/.env.example

@@ -141,3 +141,9 @@ SWETRIX_CDN_TOKEN=
 # Leave blank if you don't run the managed reverse proxy edge - the endpoints
 # fail closed (404) when this is unset.
 MANAGED_PROXY_EDGE_API_KEY=
+
+# Public demo project synthetic data generator
+DEMO_DATA_ENABLED=false
+# S3 prefix containing demo replay folders, for example demo-replays/1/*.json.gz.
+# Every direct replay folder under this prefix is grouped into one demo replay.
+DEMO_REPLAY_OBJECT_PREFIX=

backend/apps/cloud/src/analytics/session-replay-s3.service.ts

@@ -24,6 +24,12 @@ const hmac = (key: Buffer | string, value: string) =>
 const encodeKeyPath = (key: string) =>
   key.split('/').map(encodeURIComponent).join('/')
 
+const encodeQueryValue = (value: string) =>
+  encodeURIComponent(value).replace(
+    /[!'()*]/g,
+    (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`,
+  )
+
 const normalizeEndpoint = (endpoint: string) => {
   const value = endpoint.trim().replace(/\/+$/, '')
 
@@ -41,6 +47,14 @@ const inferHetznerRegion = (endpoint: string) => {
   return hostname.slice(0, -suffix.length).split('.')[0] || ''
 }
 
+const decodeXmlValue = (value: string) =>
+  value
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&amp;/g, '&')
+
 @Injectable()
 export class SessionReplayS3Service {
   private getConfig(): S3Config | null {
@@ -149,12 +163,57 @@ export class SessionReplayS3Service {
     }
   }
 
+  async listObjects(prefix: string): Promise<string[]> {
+    const keys: string[] = []
+    let continuationToken: string | undefined
+
+    do {
+      const response = await this.signedFetch(
+        'GET',
+        '',
+        undefined,
+        {},
+        undefined,
+        {
+          'list-type': '2',
+          prefix,
+          'max-keys': '1000',
+          'continuation-token': continuationToken,
+        },
+      )
+
+      if (!response.ok) {
+        throw new Error(`Hetzner S3 LIST failed with status ${response.status}`)
+      }
+
+      const xml = await response.text()
+      const keyRegex = /<Key>([\s\S]*?)<\/Key>/g
+      let keyMatch = keyRegex.exec(xml)
+
+      while (keyMatch) {
+        keys.push(decodeXmlValue(keyMatch[1]))
+        keyMatch = keyRegex.exec(xml)
+      }
+
+      continuationToken = xml.match(
+        /<NextContinuationToken>([\s\S]*?)<\/NextContinuationToken>/,
+      )?.[1]
+
+      if (continuationToken) {
+        continuationToken = decodeXmlValue(continuationToken)
+      }
+    } while (continuationToken)
+
+    return keys
+  }
+
   private async signedFetch(
     method: 'PUT' | 'GET' | 'DELETE',
     key: string,
     body?: PutObjectBody,
     extraHeaders: HeaderMap = {},
     payloadHashOverride?: string,
+    queryParams: Record<string, string | undefined> = {},
   ): Promise<Response> {
     const config = this.getConfig()
     if (!config) {
@@ -173,8 +232,18 @@ export class SessionReplayS3Service {
     const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, '')
     const dateStamp = amzDate.slice(0, 8)
     const endpoint = new URL(config.endpoint)
+    const canonicalQuery = Object.entries(queryParams)
+      .filter(([, value]) => value !== undefined)
+      .sort(([left], [right]) => left.localeCompare(right))
+      .map(
+        ([key, value]) =>
+          `${encodeQueryValue(key)}=${encodeQueryValue(value || '')}`,
+      )
+      .join('&')
+
     endpoint.hostname = `${config.bucket}.${endpoint.hostname}`
     endpoint.pathname = `/${encodeKeyPath(key)}`
+    endpoint.search = canonicalQuery ? `?${canonicalQuery}` : ''
 
     const headers: HeaderMap = {
       host: endpoint.host,
@@ -193,7 +262,7 @@ export class SessionReplayS3Service {
     const canonicalRequest = [
       method,
       endpoint.pathname,
-      '',
+      canonicalQuery,
       canonicalHeaders,
       signedHeaders,
       payloadHash,

backend/apps/cloud/src/common/integrations/clickhouse.ts

@@ -71,6 +71,9 @@ const clickhouse = createClient({
     enable_http_compression: 0,
     log_queries: 0,
 
+    compile_expressions: 0,
+    compile_aggregate_expressions: 0,
+
     // Used for analytics & captcha stuff.
     // https://clickhouse.com/docs/en/optimize/asynchronous-inserts
     wait_for_async_insert: 0, // Return ACK (await) when row was added to the buffer, not flushed to the database

backend/apps/cloud/src/demo-data/demo-data.module.ts

@@ -0,0 +1,30 @@
+import { Module } from '@nestjs/common'
+import { TypeOrmModule } from '@nestjs/typeorm'
+import { SessionReplayS3Service } from '../analytics/session-replay-s3.service'
+import { Experiment } from '../experiment/entity/experiment.entity'
+import { ExperimentVariant } from '../experiment/entity/experiment-variant.entity'
+import { FeatureFlag } from '../feature-flag/entity/feature-flag.entity'
+import { Goal } from '../goal/entity/goal.entity'
+import { Annotation, Funnel, Project } from '../project/entity'
+import { ProjectViewCustomEventEntity } from '../project/entity/project-view-custom-event.entity'
+import { ProjectViewEntity } from '../project/entity/project-view.entity'
+import { DemoDataService } from './demo-data.service'
+
+@Module({
+  imports: [
+    TypeOrmModule.forFeature([
+      Annotation,
+      Experiment,
+      ExperimentVariant,
+      FeatureFlag,
+      Funnel,
+      Goal,
+      Project,
+      ProjectViewCustomEventEntity,
+      ProjectViewEntity,
+    ]),
+  ],
+  providers: [DemoDataService, SessionReplayS3Service],
+  exports: [DemoDataService],
+})
+export class DemoDataModule {}