diff --git a/.cspell.json b/.cspell.json
index b29827d886b..663e78eba25 100644
--- a/.cspell.json
+++ b/.cspell.json
@@ -444,7 +444,6 @@
"novuapp",
"novugo",
"novuhq",
- "novui",
"novutest",
"npmignore",
"npmjs",
@@ -475,8 +474,6 @@
"OTLP",
"outdir",
"Outgoers",
- "pandabox",
- "pandacss",
"Paramtype",
"parens",
"partnerintegration",
@@ -861,10 +858,6 @@
"apps/api/src/app/workflows-v2/usecases/validate-content/validate-placeholders/validate-placeholder.usecase.ts",
"apps/api/src/app/workflows-v2/util/json-schema-mock.ts",
"libs/internal-sdk/**/*",
- "apps/web/env.sh",
- "apps/web/playwright.config.ts",
- "apps/web/src/pages/playground/web-container-configuration/sandbox-vite/*.ts",
- "apps/web/src/studio/components/workflows/step-editor/editor/files.ts",
"apps/widget/public/iframeResizer.contentWindow.js",
"apps/worker/README.md",
"apps/ws/src/.env.test",
diff --git a/.cursor/rules/novu.mdc b/.cursor/rules/novu.mdc
index 34fa670b7a2..74d1a768034 100644
--- a/.cursor/rules/novu.mdc
+++ b/.cursor/rules/novu.mdc
@@ -36,7 +36,7 @@ When writing FrontEnd code, you are an expert on modern dev tool UI and UX desig
- Optimize images: use WebP format, include size data, implement lazy loading.
### Git Rules
-- when adding commit titles, use proper scope (dashboard,web,api,worker,shared,etc...)
+- when adding commit titles, use proper scope (dashboard,api,worker,shared,etc...)
### Key Conventions
- Linting: Add blank lines before return statements
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 1f577752071..2f13fb6560b 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -18,7 +18,7 @@
// Use 'forwardPorts' to make a list of ports inside the container available locally.
"forwardPorts": [4200, 3000, 27017],
- "onCreateCommand": "npm run setup:project -- --exclude=@novu/api-service,@novu/worker,@novu/web",
+ "onCreateCommand": "npm run setup:project -- --exclude=@novu/api-service,@novu/worker",
// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
"remoteUser": "node",
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 0bdb616cf38..a62d28d9271 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -10,6 +10,3 @@
.husky @novuhq/novu-infra
docker @novuhq/novu-infra
**/Dockerfile @novuhq/novu-infra
-
-# Novu community team ownership
-apps/web @novuhq/novu-web
diff --git a/.github/labeler.yml b/.github/labeler.yml
index 768dffe137f..b73ef775611 100644
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -2,8 +2,6 @@
- apps/api/**/*
'@novu/worker':
- apps/worker/**/*
-'@novu/web':
- - apps/web/**/*
'@novu/dashboard':
- apps/dashboard/**/*
'@novu/ws':
diff --git a/.github/workflows/dev-deploy-dashboard.yml b/.github/workflows/dev-deploy-dashboard.yml
index b4c5eaa4277..ec9cb9ffea7 100644
--- a/.github/workflows/dev-deploy-dashboard.yml
+++ b/.github/workflows/dev-deploy-dashboard.yml
@@ -9,7 +9,6 @@ on:
- next
- main
paths:
- - 'apps/web/**'
- 'apps/dashboard/**'
env:
diff --git a/.github/workflows/dev-deploy-web.yml b/.github/workflows/dev-deploy-web.yml
deleted file mode 100644
index 24dcafcfd0b..00000000000
--- a/.github/workflows/dev-deploy-web.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-name: Deploy DEV WEB
-
-# Controls when the action will run. Triggers the workflow on push or pull request
-# events but only for the master branch
-on:
- workflow_dispatch:
- inputs:
- skip_tests:
- description: "Skip E2E tests"
- required: false
- type: boolean
- default: false
-
-env:
- NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
-jobs:
- test_web:
- if: ${{ !inputs.skip_tests }}
- uses: ./.github/workflows/reusable-web-e2e.yml
- with:
- ee: true
- secrets: inherit
-
- deploy_web:
- needs: test_web
- if: ${{ !contains(github.event.head_commit.message, 'ci skip') && (success() || inputs.skip_tests) }}
- uses: ./.github/workflows/reusable-web-deploy.yml
- with:
- environment: Development
- react_app_api_url: https://api.novu-staging.co
- react_app_ws_url: https://dev.ws.novu.co
- react_app_webhook_url: https://dev.webhook.novu.co
- react_app_widget_embed_path: https://dev.embed.novu.co/embed.umd.min.js
- react_app_sentry_dsn: https://2b5160da86384949be4cc66679c54e79@o1161119.ingest.sentry.io/6250907
- react_app_environment: dev
- react_app_mail_server_domain: dev.inbound-mail.novu.co
- react_app_hubspot_embed: 44416662
- netlify_deploy_message: Dev deployment
- netlify_alias: dev
- netlify_gh_env: development
- netlify_site_id: 45396446-dc86-4ad6-81e4-86d3eb78d06f
- clerk_publishable_key: pk_live_Y2xlcmsubm92dS1zdGFnaW5nLmNvJA
- clerk_is_ee_auth_enabled: true
- new_dashboard_url: https://dashboard.novu-staging.co
- extra_env_variables: ${{ vars.STAGING_EXTRA_VARIABLES || '' }}
- secrets: inherit
diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml
index aa09579cb0e..1cfe2c74e99 100644
--- a/.github/workflows/on-pr.yml
+++ b/.github/workflows/on-pr.yml
@@ -30,7 +30,7 @@ jobs:
- name: Run Spell Check
uses: streetsidesoftware/cspell-action@v6
with:
- root: 'apps/web'
+ root: 'apps/dashboard'
files: '**/*'
incremental_files_only: true
@@ -271,16 +271,6 @@ jobs:
ee: true
secrets: inherit
- # TODO: Uncomment this when we have update on the web app
- # test_e2e_web:
- # name: E2E test Web app
- # needs: [get-affected]
- # if: ${{ contains(fromJson(needs.get-affected.outputs.test-e2e), '@novu/web') }}
- # uses: ./.github/workflows/reusable-web-e2e.yml
- # secrets: inherit
- # with:
- # ee: true
-
#test_e2e_dashboard:
# name: E2E test Dashboard app
# needs: [get-affected]
diff --git a/.github/workflows/prod-deploy-web.yml b/.github/workflows/prod-deploy-web.yml
deleted file mode 100644
index e7ec7d0bd22..00000000000
--- a/.github/workflows/prod-deploy-web.yml
+++ /dev/null
@@ -1,55 +0,0 @@
-name: Deploy PROD WEB
-
-# Controls when the action will run. Triggers the workflow on push or pull request
-# events but only for the master branch
-on:
- workflow_dispatch:
-
-env:
- NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
-jobs:
- deploy_web_eu:
- uses: ./.github/workflows/reusable-web-deploy.yml
- with:
- environment: Production
- react_app_api_url: https://eu.api.novu.co
- react_app_ws_url: https://eu.ws.novu.co
- react_app_webhook_url: https://eu.webhook.novu.co
- react_app_widget_embed_path: https://eu.embed.novu.co/embed.umd.min.js
- react_app_sentry_dsn: https://2b5160da86384949be4cc66679c54e79@o1161119.ingest.sentry.io/6250907
- react_app_environment: production
- react_app_mail_server_domain: eu.inbound-mail.novu.co
- react_app_hubspot_embed: 44416662
- netlify_deploy_message: Prod deployment
- netlify_alias: prod
- netlify_gh_env: Production
- netlify_site_id: d2e8b860-7016-4202-9256-ebca0f13259a
- clerk_publishable_key: pk_live_Y2xlcmsuZXUuZGFzaGJvYXJkLm5vdnUuY28k
- clerk_is_ee_auth_enabled: true
- new_dashboard_url: https://eu.dashboard.novu.co
- extra_env_variables: ${{ vars.EU_EXTRA_VARIABLES || '' }}
- secrets: inherit
-
- deploy_web_us:
- uses: ./.github/workflows/reusable-web-deploy.yml
- with:
- environment: Production
- react_app_api_url: https://api.novu.co
- react_app_ws_url: https://ws.novu.co
- react_app_webhook_url: https://webhook.novu.co
- react_app_widget_embed_path: https://embed.novu.co/embed.umd.min.js
- react_app_sentry_dsn: https://2b5160da86384949be4cc66679c54e79@o1161119.ingest.sentry.io/6250907
- react_app_environment: production
- react_app_mail_server_domain: inbound-mail.novu.co
- react_app_hubspot_embed: 44416662
- netlify_deploy_message: Prod deployment
- netlify_alias: prod
- netlify_gh_env: Production
- netlify_site_id: 8639d8b9-81f9-44c3-b885-585a7fd2b5ff
- clerk_publishable_key: pk_live_Y2xlcmsuZGFzaGJvYXJkLm5vdnUuY28k
- clerk_is_ee_auth_enabled: true
- new_dashboard_url: https://dashboard.novu.co
- extra_env_variables: ${{ vars.US_EXTRA_VARIABLES || '' }}
- secrets: inherit
diff --git a/.github/workflows/reusable-web-deploy.yml b/.github/workflows/reusable-web-deploy.yml
deleted file mode 100644
index 990f97313ac..00000000000
--- a/.github/workflows/reusable-web-deploy.yml
+++ /dev/null
@@ -1,139 +0,0 @@
-name: Deploy Web to Netlify
-
-env:
- NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-
-# Controls when the action will run. Triggers the workflow on push or pull request
-on:
- workflow_call:
- inputs:
- environment:
- required: true
- type: string
- react_app_api_url:
- required: false
- type: string
- react_app_ws_url:
- required: true
- type: string
- react_app_webhook_url:
- required: true
- type: string
- react_app_widget_embed_path:
- required: true
- type: string
- react_app_sentry_dsn:
- required: true
- type: string
- react_app_environment:
- required: true
- type: string
- react_app_mail_server_domain:
- required: true
- type: string
- react_app_hubspot_embed:
- required: false
- type: string
- # Netlify inputs
- netlify_deploy_message:
- required: true
- type: string
- netlify_alias:
- required: true
- type: string
- netlify_gh_env:
- required: true
- type: string
- netlify_site_id:
- required: true
- type: string
- clerk_publishable_key:
- required: true
- type: string
- clerk_is_ee_auth_enabled:
- required: true
- type: string
- new_dashboard_url:
- required: false
- type: string
- extra_env_variables:
- required: false
- type: string
- description: "Multi-line string of additional env variables (e.g., for multi-region configs)"
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
-jobs:
- reusable_web_deploy:
- runs-on: blacksmith-4vcpu-ubuntu-2404
- timeout-minutes: 80
- environment: ${{ inputs.environment }}
- permissions:
- contents: read
- packages: write
- deployments: write
- id-token: write
- steps:
- - uses: actions/checkout@v5
- name: Checkout with submodules
- with:
- submodules: true
- token: ${{ secrets.SUBMODULES_TOKEN }}
-
- - uses: ./.github/actions/setup-project
- with:
- slim: "true"
- submodules: true
-
- - name: Create env file
- working-directory: apps/web
- run: |
- touch .env
- echo REACT_APP_SEGMENT_KEY=${{ secrets.WEB_SEGMENT_KEY }} >> .env
- echo REACT_APP_INTERCOM_APP_ID=${{ secrets.INTERCOM_APP_ID }} >> .env
- echo REACT_APP_API_URL=${{ inputs.react_app_api_url }} >> .env
- echo REACT_APP_WS_URL=${{ inputs.react_app_ws_url }} >> .env
- echo REACT_APP_WEBHOOK_URL=${{ inputs.react_app_webhook_url }} >> .env
- echo REACT_APP_WIDGET_EMBED_PATH=${{ inputs.react_app_widget_embed_path }} >> .env
- echo REACT_APP_NOVU_APP_ID=${{ secrets.NOVU_APP_ID }} >> .env
- echo REACT_APP_SENTRY_DSN=${{ inputs.react_app_sentry_dsn }} >> .env
- echo REACT_APP_ENVIRONMENT=${{ inputs.react_app_environment }} >> .env
- echo REACT_APP_MAIL_SERVER_DOMAIN=${{ inputs.react_app_mail_server_domain }} >> .env
- echo REACT_APP_LAUNCH_DARKLY_CLIENT_SIDE_ID=${{ secrets.LAUNCH_DARKLY_CLIENT_SIDE_ID }} >> .env
- echo REACT_APP_HUBSPOT_EMBED=${{ inputs.react_app_hubspot_embed }} >> .env
- echo REACT_APP_STRIPE_CLIENT_KEY=${{ secrets.STRIPE_CLIENT_KEY }} >> .env
- echo REACT_APP_NOVU_GTM_ID=${{ secrets.REACT_APP_NOVU_GTM_ID }} >> .env
- echo REACT_APP_MIXPANEL_KEY=${{ secrets.MIXPANEL_TOKEN }} >> .env
- echo REACT_APP_CLERK_PUBLISHABLE_KEY=${{ inputs.clerk_publishable_key }} >> .env
- echo REACT_APP_IS_EE_AUTH_ENABLED=${{ inputs.clerk_is_ee_auth_enabled }} >> .env
- echo REACT_APP_NEW_DASHBOARD_URL=${{ inputs.new_dashboard_url }} >> .env
- echo REACT_APP_PLAIN_SUPPORT_CHAT_APP_ID=${{secrets.REACT_APP_PLAIN_SUPPORT_CHAT_APP_ID}} >> .env
- # Append extra environment variables if provided
- if [ -n "${{ inputs.extra_env_variables }}" ]; then
- echo "Adding extra environment variables..."
- IFS='|' read -ra VARS <<< "${{ inputs.extra_env_variables }}"
- for var in "${VARS[@]}"; do
- echo "$var" >> .env
- done
- fi
- - name: Envsetup
- working-directory: apps/web
- run: npm run envsetup
-
- - name: Build
- run: CI='' pnpm build:web --skip-nx-cache
-
- - name: Deploy WEB
- uses: nwtgck/actions-netlify@v1.2
- with:
- publish-dir: apps/web/build
- github-token: ${{ secrets.GITHUB_TOKEN }}
- deploy-message: ${{ inputs.netlify_deploy_message }}
- production-deploy: true
- alias: ${{ inputs.netlify_alias }}
- github-deployment-environment: ${{ inputs.netlify_gh_env }}
- github-deployment-description: Web Deployment
- netlify-config-path: apps/web/netlify.toml
- env:
- NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
- NETLIFY_SITE_ID: ${{ inputs.netlify_site_id }}
- timeout-minutes: 1
diff --git a/.github/workflows/reusable-web-e2e.yml b/.github/workflows/reusable-web-e2e.yml
deleted file mode 100644
index 4d0fb7fa111..00000000000
--- a/.github/workflows/reusable-web-e2e.yml
+++ /dev/null
@@ -1,155 +0,0 @@
-name: Test WEB
-
-env:
- NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
-
-# Controls when the action will run. Triggers the workflow on push or pull request
-on:
- workflow_dispatch:
- inputs:
- ee:
- description: 'use the ee version of worker'
- required: false
- default: true
- type: boolean
- workflow_call:
- inputs:
- ee:
- description: 'use the ee version of worker'
- required: false
- default: false
- type: boolean
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
-jobs:
- # This workflow contains a single job called "build"
- e2e_web:
- strategy:
- fail-fast: false
- matrix:
- # run 4 copies of the current job in parallel
- containers: [1, 2, 3, 4]
- total: [4]
-
- # The type of runner that the job will run on
- runs-on: ubuntu-latest
- timeout-minutes: 80
-
- permissions:
- contents: read
- packages: write
- deployments: write
- id-token: write
-
- steps:
- - id: determine_run_type
- name: Determing community vs enterprise run
- run: |
- if ! [[ -z "${{ secrets.SUBMODULES_TOKEN }}" ]]; then
- echo "enterprise_run=true" >> $GITHUB_OUTPUT
- else
- echo "enterprise_run=false" >> $GITHUB_OUTPUT
- fi
-
- - id: checkout-enterprise-code
- name: Checkout enterprise code from the submodule
- uses: actions/checkout@v5
- if: steps.determine_run_type.outputs.enterprise_run == 'true'
- with:
- submodules: true
- token: ${{ secrets.SUBMODULES_TOKEN }}
-
- - id: checkout-community-code
- name: Checkout community code
- uses: actions/checkout@v5
- if: steps.determine_run_type.outputs.enterprise_run != 'true'
-
- - uses: ./.github/actions/setup-project
- with:
- submodules: true
-
- - uses: mansagroup/nrwl-nx-action@v3
- with:
- targets: build
- projects: '@novu/web,@novu/api-service,@novu/worker'
-
- - uses: ./.github/actions/start-localstack
- - uses: ./.github/actions/setup-redis-cluster
-
- - uses: ./.github/actions/run-backend
- with:
- cypress_github_oauth_client_id: ${{ secrets.CYPRESS_GITHUB_OAUTH_CLIENT_ID }}
- cypress_github_oauth_client_secret: ${{ secrets.CYPRESS_GITHUB_OAUTH_CLIENT_SECRET }}
- launch_darkly_sdk_key: ${{ secrets.LAUNCH_DARKLY_SDK_KEY }}
- ci_ee_test: ${{ steps.determine_run_type.outputs.enterprise_run }}
-
- - name: Start WS
- run: |
- cd apps/ws && pnpm start:test &
-
- - name: Start Novu web app
- working-directory: apps/web
- env:
- REACT_APP_API_URL: http://127.0.0.1:1336
- REACT_APP_WS_URL: http://127.0.0.1:1340
- REACT_APP_WEBHOOK_URL: http://127.0.0.1:1341
- # Disable LaunchDarkly client-side SDK in the test environment to reduce E2E flakiness
- REACT_APP_LAUNCH_DARKLY_CLIENT_SIDE_ID: ''
- NOVU_ENTERPRISE: ${{ steps.determine_run_type.outputs.enterprise_run }}
- run: |
- pnpm run envsetup:docker
- pnpm start:static:build &
-
- - name: Wait on Services
- run: wait-on --timeout=180000 http://127.0.0.1:1340/v1/health-check http://127.0.0.1:4200/
-
- - name: Install Playwright
- working-directory: apps/web
- run: pnpm test:e2e:install
-
- - name: Run E2E tests
- working-directory: apps/web
- env:
- NOVU_ENTERPRISE: ${{ steps.determine_run_type.outputs.enterprise_run }}
- run: pnpm test:e2e --shard=${{ matrix.containers }}/${{ matrix.total }}
-
- - uses: actions/upload-artifact@v4
- if: ${{ !cancelled() }}
- with:
- name: blob-report-${{ matrix.containers }}
- path: apps/web/blob-report
- retention-days: 1
-
- merge-reports:
- # Merge reports after playwright-tests, even if some shards have failed
- if: ${{ !cancelled() }}
- needs: [e2e_web]
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v5
- - uses: actions/setup-node@v4
- with:
- node-version: 20.19.0
-
- - name: Download blob reports from GitHub Actions Artifacts
- uses: actions/download-artifact@v4
- with:
- path: all-blob-reports
- pattern: blob-report-*
- merge-multiple: true
-
- - name: Merge into HTML Report
- run: npx playwright merge-reports --reporter html ./all-blob-reports
-
- - name: Upload HTML report
- uses: actions/upload-artifact@v4
- with:
- name: html-report--attempt-${{ github.run_attempt }}
- path: playwright-report
- retention-days: 14
-
- - name: Send Slack notifications
- uses: ./.github/actions/slack-notify-on-failure
- if: failure()
- with:
- slackWebhookURL: ${{ secrets.SLACK_WEBHOOK_URL_ENG_FEED_GITHUB }}
diff --git a/.github/workflows/tag-images.yml b/.github/workflows/tag-images.yml
deleted file mode 100644
index 779c160740a..00000000000
--- a/.github/workflows/tag-images.yml
+++ /dev/null
@@ -1,127 +0,0 @@
-name: Tag Docker Version
-
-# Controls when the action will run. Triggers the workflow on push or pull request
-# events but only for the master branch
-on:
- workflow_dispatch:
- inputs:
- version:
- description: 'The version to tag docker images'
- required: true
- type: string
-
-jobs:
- tag_images:
- runs-on: blacksmith-4vcpu-ubuntu-2404
- timeout-minutes: 80
- environment: Production
- permissions:
- contents: read
- packages: write
- deployments: write
- steps:
- # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- - uses: actions/checkout@v5
- - name: Setup kernel for react native, increase watchers
- run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
- - uses: useblacksmith/setup-node@v5
- with:
- node-version: '20.19.0'
-
- - name: Login to docker
- env:
- GH_ACTOR: ${{ github.actor }}
- GH_PASSWORD: ${{ secrets.GH_PACKAGES }}
- run: |
- echo $GH_PASSWORD | docker login ghcr.io -u $GH_ACTOR --password-stdin
-
- - name: Tag API
- env:
- REGISTRY_OWNER: novuhq
- DOCKER_NAME: novu/api
- IMAGE_TAG: ${{ github.sha }}
- GH_ACTOR: ${{ github.actor }}
- GH_PASSWORD: ${{ secrets.GH_PACKAGES }}
- DOCKER_VERSION: ${{ inputs.version }}
- run: |
- docker pull ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod
- docker tag ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
- docker push ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
-
- - name: Tag Worker
- env:
- REGISTRY_OWNER: novuhq
- DOCKER_NAME: novu/worker
- IMAGE_TAG: ${{ github.sha }}
- GH_ACTOR: ${{ github.actor }}
- GH_PASSWORD: ${{ secrets.GH_PACKAGES }}
- DOCKER_VERSION: ${{ inputs.version }}
- run: |
- docker pull ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod
- docker tag ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
- docker push ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
-
- - name: Tag WS
- env:
- REGISTRY_OWNER: novuhq
- DOCKER_NAME: novu/ws
- IMAGE_TAG: ${{ github.sha }}
- GH_ACTOR: ${{ github.actor }}
- GH_PASSWORD: ${{ secrets.GH_PACKAGES }}
- DOCKER_VERSION: ${{ inputs.version }}
- run: |
- docker pull ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod
- docker tag ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
- docker push ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
-
- - name: Tag EMBED
- env:
- REGISTRY_OWNER: novuhq
- DOCKER_NAME: novu/embed
- IMAGE_TAG: ${{ github.sha }}
- GH_ACTOR: ${{ github.actor }}
- GH_PASSWORD: ${{ secrets.GH_PACKAGES }}
- DOCKER_VERSION: ${{ inputs.version }}
- run: |
- docker pull ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod
- docker tag ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
- docker push ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
-
- - name: Tag WIDGET
- env:
- REGISTRY_OWNER: novuhq
- DOCKER_NAME: novu/widget
- IMAGE_TAG: ${{ github.sha }}
- GH_ACTOR: ${{ github.actor }}
- GH_PASSWORD: ${{ secrets.GH_PACKAGES }}
- DOCKER_VERSION: ${{ inputs.version }}
- run: |
- docker pull ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod
- docker tag ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
- docker push ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
-
- - name: Tag WEB
- env:
- REGISTRY_OWNER: novuhq
- DOCKER_NAME: novu/web
- IMAGE_TAG: ${{ github.sha }}
- GH_ACTOR: ${{ github.actor }}
- GH_PASSWORD: ${{ secrets.GH_PACKAGES }}
- DOCKER_VERSION: ${{ inputs.version }}
- run: |
- docker pull ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod
- docker tag ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
- docker push ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
-
- - name: Tag SDK
- env:
- REGISTRY_OWNER: novuhq
- DOCKER_NAME: novu/sdk
- IMAGE_TAG: ${{ github.sha }}
- GH_ACTOR: ${{ github.actor }}
- GH_PASSWORD: ${{ secrets.GH_PACKAGES }}
- DOCKER_VERSION: ${{ inputs.version }}
- run: |
- docker pull ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod
- docker tag ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:prod ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
- docker push ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$DOCKER_VERSION
diff --git a/.idea/novu.iml b/.idea/novu.iml
index f3d0f7f72dc..ad99c417478 100644
--- a/.idea/novu.iml
+++ b/.idea/novu.iml
@@ -7,20 +7,15 @@
-
-
-
-
-
diff --git a/.idea/webResources.xml b/.idea/webResources.xml
deleted file mode 100644
index 9c504eef033..00000000000
--- a/.idea/webResources.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/.source b/.source
index 914686cece0..e38b53ee33b 160000
--- a/.source
+++ b/.source
@@ -1 +1 @@
-Subproject commit 914686cece0ed8f21a9d6be60db5d721cbf55f0c
+Subproject commit e38b53ee33bda663c1948ad3a8271c1f91c140b6
diff --git a/.vscode/settings.json b/.vscode/settings.json
index cae065da195..b8da8711b95 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -27,7 +27,10 @@
"editor.defaultFormatter": "biomejs.biome"
},
"files.exclude": {
- "**/.source": true
+ "**/.source": true,
+ "**/dist/**": true,
+ "**/build/**": true,
+ "**/*.map": true
},
"search.exclude": {
"**/.source": true
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
index d396346aee3..43bbd2a5a5b 100644
--- a/.vscode/tasks.json
+++ b/.vscode/tasks.json
@@ -78,26 +78,6 @@
}
}
},
- {
- "type": "npm",
- "script": "start",
- "isBackground": true,
- "label": "WEB",
- "path": "/apps/web",
- "icon": {
- "id": "browser",
- "color": "terminal.ansiGreen"
- },
- "problemMatcher": {
- "base": "$tsc-watch",
- "owner": "typescript",
- "background": {
- "activeOnStart": true,
- "beginsPattern": "Compiling...",
- "endsPattern": "webpack compiled successfully"
- }
- }
- },
{
"type": "npm",
"script": "start",
@@ -122,14 +102,6 @@
"path": "/libs/dal",
"problemMatcher": "$tsc-watch"
},
- {
- "type": "npm",
- "script": "start",
- "isBackground": true,
- "label": "STATELESS",
- "path": "/packages/stateless",
- "problemMatcher": "$tsc-watch"
- },
{
"type": "npm",
"script": "start",
@@ -178,20 +150,6 @@
"path": "/enterprise/packages/shared-services",
"problemMatcher": "$tsc-watch"
},
- {
- "type": "npm",
- "script": "build",
- "label": "DESIGN SYSTEM",
- "path": "/libs/design-system",
- "problemMatcher": "$tsc-watch"
- },
- {
- "type": "npm",
- "script": "setup",
- "label": "NOVUI",
- "path": "/libs/novui",
- "problemMatcher": "$tsc"
- },
{
"type": "npm",
"script": "build",
diff --git a/CLAUDE.md b/CLAUDE.md
index b47faea0af9..10a6db8ca46 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -16,7 +16,6 @@ Novu is a notification infrastructure platform built as a **monorepo using Nx**
**Key Applications** (`apps/`):
- `api` - Core NestJS backend service (REST API, authentication, business logic)
- `dashboard` - Modern React dashboard built with Vite (primary UI)
-- `web` - Legacy React dashboard (CRA-based, maintained for compatibility)
- `worker` - Background job processing service (Bull queues)
- `ws` - WebSocket service for real-time updates
- `webhook` - Webhook delivery service
@@ -25,7 +24,6 @@ Novu is a notification infrastructure platform built as a **monorepo using Nx**
**Key Libraries** (`libs/`):
- `application-generic` - Common business logic, CQRS patterns, auth decorators
- `dal` - Data Access Layer (MongoDB models, repositories)
-- `design-system` - Shared UI components and themes
- `internal-sdk` - TypeScript SDK with auto-generated types
**NPM Packages** (`packages/`):
@@ -51,7 +49,6 @@ pnpm start # or: pnpm jarvis
# Core development stack
pnpm start:api:dev # API service with hot reload
pnpm start:dashboard # New React dashboard
-pnpm start:web # Legacy web app
pnpm start:worker # Background worker
pnpm start:ws # WebSocket service
```
@@ -70,7 +67,6 @@ cd apps/api && pnpm test # Unit tests
cd apps/api && pnpm test:e2e:novu-v2 # E2E tests
# Frontend E2E tests
-cd apps/web && pnpm test:e2e # Playwright tests
cd apps/dashboard && pnpm test:e2e # Dashboard E2E tests
```
@@ -91,7 +87,7 @@ pnpm typecheck # Run TypeScript checks
- Prefer interfaces over types (backend), types over interfaces (frontend)
- Add blank lines before return statements
- Import "motion-react" from "motion/react"
-- Git commits: use proper scope (dashboard, web, api, worker, shared, etc.)
+- Git commits: use proper scope (dashboard, api, worker, shared, etc.)
**Dashboard-specific (.cursor/rules/dashboard.mdc):**
- Do not attempt to build/run dashboard (user will be running it)
@@ -146,8 +142,8 @@ pnpm typecheck # Run TypeScript checks
- **pnpm 10+** required as package manager
- **MongoDB & Redis** required for local development (available via Docker Compose)
- **Environment files** are set up automatically by setup scripts
-- **Dashboard is primary UI** - web app is legacy but still maintained
+- **Dashboard is primary UI**
## Development Guidance
-- No need to run npm typecheck commands, as we will see it ourself
\ No newline at end of file
+- No need to run npm typecheck commands, as we will see it ourself
diff --git a/apps/web/src/ee/LICENSE b/EE-PACKAGES-LICENSE
similarity index 100%
rename from apps/web/src/ee/LICENSE
rename to EE-PACKAGES-LICENSE
diff --git a/LICENSE-ENTERPRISE b/LICENSE-ENTERPRISE
index 1da8557fd7c..de2879c1f4c 100644
--- a/LICENSE-ENTERPRISE
+++ b/LICENSE-ENTERPRISE
@@ -1,7 +1,6 @@
Portions of this software are licensed as follows:
-* All content that resides under https://github.com/novuhq/novu/tree/next/enterprise/packages and
-https://github.com/novuhq/novu/tree/next/apps/web/src/ee directory of this repository (Commercial License) is licensed under the license defined in "https://github.com/novuhq/novu/blob/next/apps/web/src/ee/LICENSE".
+* All content that resides under https://github.com/novuhq/novu/tree/next/enterprise/packages and is licensed under the license defined in "https://github.com/novuhq/novu/blob/next/EE-PACKAGES-LICENSE".
* All third party components incorporated into the Novu Software are licensed under the original license provided by the owner of the applicable component.
* Content outside of the above mentioned directories or restrictions above is available under the "MIT" license as defined below.
diff --git a/apps/api/migrations/clickhouse-migrations/3_analytics_tables.sql b/apps/api/migrations/clickhouse-migrations/3_analytics_tables.sql
new file mode 100644
index 00000000000..e109b36823a
--- /dev/null
+++ b/apps/api/migrations/clickhouse-migrations/3_analytics_tables.sql
@@ -0,0 +1,69 @@
+-- Delivery trend counts table
+-- Pre-aggregates completed step runs by step_type and date for efficient delivery trend queries
+-- Handles message delivery volume per channel type from step_runs table
+
+CREATE TABLE IF NOT EXISTS delivery_trend_counts (
+ date Date,
+ organization_id String,
+ environment_id String,
+ workflow_id String DEFAULT '',
+ step_type LowCardinality(String),
+ count UInt64
+)
+ENGINE = SummingMergeTree(count)
+PARTITION BY toYYYYMM(date)
+ORDER BY (organization_id, environment_id, date, workflow_id, step_type);
+
+-- Materialized view populates from step_runs table (completed messaging steps)
+CREATE MATERIALIZED VIEW IF NOT EXISTS delivery_trend_counts_mv
+TO delivery_trend_counts
+AS SELECT
+ toDate(created_at) AS date,
+ organization_id,
+ environment_id,
+ ifNull(workflow_id, '') AS workflow_id,
+ step_type,
+ 1 AS count
+FROM step_runs
+WHERE
+ status = 'completed'
+ AND step_type IN ('in_app', 'email', 'sms', 'chat', 'push');
+
+-- Add provider_id column to traces table
+-- This column stores the provider ID that was used to send the message
+-- Must be added before creating the materialized view that references it
+ALTER TABLE traces
+ADD COLUMN IF NOT EXISTS provider_id String DEFAULT '';
+
+-- Trace rollup table
+-- Handles both message counts and subscriber activity from traces table
+-- Captures message_sent events and interaction events (seen, read, snoozed, archived)
+CREATE TABLE IF NOT EXISTS trace_rollup (
+ date Date,
+ organization_id String,
+ environment_id String,
+ workflow_id String,
+ external_subscriber_id String DEFAULT '',
+ event_type LowCardinality(String),
+ provider_id String DEFAULT '',
+ count UInt64
+)
+ENGINE = SummingMergeTree(count)
+PARTITION BY toYYYYMM(date)
+ORDER BY (organization_id, environment_id, event_type, date, workflow_id, external_subscriber_id, provider_id);
+
+-- Materialized view populates from traces table
+-- Captures both message_sent events and interaction events
+CREATE MATERIALIZED VIEW IF NOT EXISTS trace_rollup_mv
+TO trace_rollup
+AS SELECT
+ toDate(created_at) AS date,
+ organization_id,
+ environment_id,
+ ifNull(workflow_id, '') AS workflow_id,
+ ifNull(external_subscriber_id, '') AS external_subscriber_id,
+ event_type,
+ ifNull(provider_id, '') AS provider_id,
+ 1 AS count
+FROM traces
+WHERE event_type IN ('message_sent', 'message_seen', 'message_read', 'message_snoozed', 'message_archived');
diff --git a/apps/api/package.json b/apps/api/package.json
index a363ce249ff..046a853a9e2 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -32,6 +32,8 @@
"test:e2e:novu-v0": "cross-env TS_NODE_COMPILER_OPTIONS='{\"strictNullChecks\": false}' NODE_ENV=test mocha --timeout 15000 --retries 3 --grep '#novu-v0' --require ts-node/register --exit --file e2e/setup.ts src/**/*.e2e{,-ee}.ts",
"test:e2e:novu-v2": "cross-env NODE_ENV=test CI_EE_TEST=true CLERK_ENABLED=true NODE_OPTIONS=--max_old_space_size=8192 mocha --timeout 30000 --retries 3 --grep '#novu-v2' --require ./swc-register.js --exit --file e2e/setup.ts 'src/**/*.e2e{,-ee}.ts' 'e2e/enterprise/**/*.e2e.ts'",
"migration": "cross-env NODE_ENV=local MIGRATION=true ts-node --transpileOnly",
+ "seed:clickhouse": "cross-env NODE_ENV=local ts-node --transpileOnly scripts/seed-clickhouse.ts",
+ "seed:triggers": "cross-env NODE_ENV=local ts-node --transpileOnly scripts/seed-triggers.ts",
"clickhouse:migrate:local": "clickhouse-migrations migrate --host=http://localhost:8123 --user=default --password= --db=novu-local --migrations-home=./migrations/clickhouse-migrations",
"clickhouse:migrate:prod": "clickhouse-migrations migrate --migrations-home=./migrations/clickhouse-migrations",
"link:submodules": "pnpm link ../../enterprise/packages/auth && pnpm link ../../enterprise/packages/translation && pnpm link ../../enterprise/packages/billing",
diff --git a/apps/api/scripts/clickhouse-seeder/README.md b/apps/api/scripts/clickhouse-seeder/README.md
new file mode 100644
index 00000000000..50e5ac56f32
--- /dev/null
+++ b/apps/api/scripts/clickhouse-seeder/README.md
@@ -0,0 +1,384 @@
+# ClickHouse Data Seeding Script
+
+A comprehensive TypeScript script to populate ClickHouse observability tables with realistic mock data for load testing and development.
+
+## Overview
+
+This seeding script generates realistic Novu usage data across multiple organizations, environments, and workflows. It creates hierarchical data that mimics real-world scenarios including:
+
+- Multiple organization profiles (Enterprise, Large, Medium)
+- Multiple environments per organization (Production, Staging, Development)
+- Various workflow types with different channel combinations
+- Realistic time distribution patterns (business hours, weekends, peak patterns)
+- Workflow runs, step runs, and trace events
+
+## Architecture
+
+```
+Organization
+ └── Environment(s)
+ ├── Workflow(s)
+ │ └── Workflow Run(s)
+ │ └── Step Run(s)
+ │ └── Trace(s)
+ └── Subscriber(s)
+```
+
+## Prerequisites
+
+1. ClickHouse instance running and accessible
+2. Environment variables set:
+ ```bash
+ CLICK_HOUSE_URL=http://localhost:8123
+ CLICK_HOUSE_DATABASE=novu
+ CLICK_HOUSE_USER=default
+ CLICK_HOUSE_PASSWORD=
+ ```
+
+3. ClickHouse tables and materialized views created (run migrations first)
+
+## Usage
+
+### Basic Usage
+
+From the `apps/api` directory:
+
+```bash
+pnpm seed:clickhouse
+```
+
+Or from the root directory:
+
+```bash
+pnpm --filter @novu/api-service seed:clickhouse
+```
+
+This will generate:
+- 10 organizations (3 Enterprise, 4 Large, 3 Medium)
+- 30 days of data
+- Realistic volume based on organization profile
+- ~500K+ total records
+
+### High Volume Load Testing
+
+```bash
+pnpm seed:clickhouse -- --scale=10 --organizations=50
+```
+
+This will generate:
+- 50 organizations
+- 10x the normal data volume per organization
+- 30 days of data
+- ~25M+ total records
+
+### Custom Configuration
+
+```bash
+pnpm seed:clickhouse -- \
+ --organizations=20 \
+ --days=7 \
+ --scale=5 \
+ --batch-size=5000 \
+ --start-date=2024-01-01
+```
+
+### Single Environment Mode with Specific IDs
+
+For testing with existing organization, environment, workflow, and subscriber IDs:
+
+```bash
+pnpm seed:clickhouse -- \
+ --single-env \
+ --workflow=693ab23238cf527f6dc645d6 \
+ --subscriber=69395055051b1b19ff9e1b4c \
+ --org-id=69395056051b1b19ff9e1b52 \
+ --env-id=69395056c66fd6620f4521ba \
+ --days=30 \
+ --runs-per-day=5000
+```
+
+This will generate data for:
+- A single specified workflow
+- A single specified subscriber
+- Using the exact IDs provided
+- 30 days of data with 5000 runs per day
+
+## Configuration Options
+
+### Multi-Organization Mode Options
+
+| Option | Short | Default | Description |
+|--------|-------|---------|-------------|
+| `--organizations` | `-o` | 10 | Number of organizations to create |
+| `--days` | `-d` | 30 | Days of data to generate |
+| `--scale` | `-s` | 1.0 | Data volume multiplier for load testing |
+| `--batch-size` | `-b` | 10000 | Records per ClickHouse insert batch |
+| `--start-date` | - | Last month | Start date for data generation (YYYY-MM-DD) |
+| `--help` | `-h` | - | Show help message |
+
+### Single Environment Mode Options
+
+| Option | Short | Default | Description |
+|--------|-------|---------|-------------|
+| `--single-env` | - | - | Enable single environment mode |
+| `--org-id` | - | auto-generated | Organization ID to use |
+| `--env-id` | - | auto-generated | Environment ID to use |
+| `--workflows` | `-w` | 5 | Number of workflows to create |
+| `--workflow` | - | - | Specific workflow ID (sets workflows to 1) |
+| `--subscribers` | - | 1000 | Number of subscribers to create |
+| `--subscriber` | - | - | Specific subscriber ID (sets subscribers to 1) |
+| `--runs-per-day` | `-r` | 5000 | Workflow runs per day |
+| `--days` | `-d` | 30 | Days of data to generate |
+| `--batch-size` | `-b` | 10000 | Records per ClickHouse insert batch |
+| `--start-date` | - | Last month | Start date for data generation (YYYY-MM-DD) |
+
+## Data Volume Estimates
+
+### At scale=1 (default)
+
+| Profile | Count | Runs/Day | Total/Month |
+|---------|-------|----------|-------------|
+| Enterprise | 3 | 20K-50K | 1.8M-4.5M |
+| Large | 4 | 5K-15K | 600K-1.8M |
+| Medium | 3 | 500-2K | 45K-180K |
+
+**Total**: ~2.5M-6.5M workflow runs per month
+
+### Derived Data
+
+- **Step Runs**: 2-5x workflow runs (based on channels)
+- **Traces**: 2-8x step runs (based on events)
+- **Total Records**: 15M-50M+ per month at scale=1
+
+### At scale=10
+
+Multiply all numbers by 10x for load testing scenarios.
+
+## Organization Profiles
+
+### Enterprise (High Volume)
+- **Runs/Day**: 20,000-50,000
+- **Workflows**: 8-15
+- **Subscribers**: 5,000-10,000
+- **Environments**: 2-3
+
+### Large
+- **Runs/Day**: 5,000-15,000
+- **Workflows**: 5-10
+- **Subscribers**: 1,000-5,000
+- **Environments**: 2-3
+
+### Medium
+- **Runs/Day**: 500-2,000
+- **Workflows**: 3-5
+- **Subscribers**: 100-500
+- **Environments**: 1-2
+
+## Workflow Templates
+
+The script generates realistic workflow patterns:
+
+| Type | Channels | Weight | Example |
+|------|----------|--------|---------|
+| Transactional | email + in_app | 40% | Order Confirmation |
+| Marketing | email | 25% | Newsletter |
+| Alerts | push + sms | 15% | Critical Alert |
+| Multi-channel | email + in_app + push | 20% | Campaign Update |
+
+## Time Distribution
+
+### Business Hours Weighting
+- **9am-6pm**: 2.5x normal volume
+- **7am-9am, 6pm-9pm**: 1.2x normal volume
+- **9pm-7am**: 0.3x normal volume
+
+### Weekend Reduction
+- **Weekends**: 30% of weekday volume
+
+### Peak Patterns
+- **Monthly peaks**: 1st and 15th of the month
+- **Weekly peaks**: Tuesday 10am, Thursday 2pm
+
+## Data Tables Populated
+
+### Primary Tables (Direct Insert)
+1. **workflow_runs**: Workflow execution records
+2. **step_runs**: Individual step executions
+3. **traces**: Event traces and logs
+
+### Materialized Views (Auto-Populated)
+1. **workflow_runs_daily**: Daily aggregations of workflow runs
+2. **step_runs_daily**: Daily aggregations of step runs
+3. **traces_daily**: Daily aggregations of trace events
+
+The materialized views are automatically populated by ClickHouse as data is inserted into the primary tables.
+
+## Status Distributions
+
+### Workflow Runs
+- `completed`: 85%
+- `processing`: 5%
+- `error`: 10%
+
+### Step Runs
+- `completed`: 88%
+- `failed`: 7%
+- `skipped`: 3%
+- `delayed`: 2%
+
+### Delivery Lifecycle
+- `delivered`: 70%
+- `sent`: 15%
+- `errored`: 8%
+- `skipped`: 4%
+- `canceled`: 2%
+- `merged`: 1%
+
+## Example Output
+
+```
+============================================================
+ClickHouse Data Seeding Script
+============================================================
+
+Configuration:
+ Organizations: 10
+ Days: 30
+ Scale: 1x
+ Batch Size: 10000
+ Start Date: 2024-12-01
+
+✓ Connected to ClickHouse
+
+Phase 1: Generating Organizations and Structure
+------------------------------------------------------------
+✓ Generated 10 organizations
+ Environments: 21
+ Workflows: 147
+ Subscribers: 32,450
+
+ Organization Breakdown:
+ Enterprise: 3
+ Large: 4
+ Medium: 3
+
+Phase 2: Generating Workflow Runs
+------------------------------------------------------------
+✓ Generated 2,847,593 workflow runs
+
+Phase 3: Generating Step Runs
+------------------------------------------------------------
+✓ Generated 7,119,483 step runs
+
+Phase 4: Generating Traces
+------------------------------------------------------------
+✓ Generated 21,358,449 traces
+
+Phase 5: Inserting Data into ClickHouse
+------------------------------------------------------------
+...
+
+============================================================
+Insertion Statistics
+============================================================
+Workflow Runs: 2,847,593
+Step Runs: 7,119,483
+Traces: 21,358,449
+Total Records: 31,325,525
+Duration: 127.34s
+============================================================
+
+Additional Information:
+ Estimated Size: 14.2 GB
+ Records/Second: 246,123
+
+✓ Data seeding completed successfully!
+```
+
+## Troubleshooting
+
+### Connection Errors
+Ensure ClickHouse environment variables are set correctly:
+```bash
+export CLICK_HOUSE_URL=http://localhost:8123
+export CLICK_HOUSE_DATABASE=novu
+```
+
+### Out of Memory
+Reduce batch size for systems with limited memory:
+```bash
+pnpm seed:clickhouse -- --batch-size=5000
+```
+
+### Slow Insertion
+Increase batch size for faster insertion (if memory allows):
+```bash
+pnpm seed:clickhouse -- --batch-size=20000
+```
+
+## Development
+
+### File Structure
+```
+apps/api/scripts/
+├── seed-clickhouse.ts # Main entry point
+└── clickhouse-seeder/
+ ├── config.ts # Configuration and CLI parsing
+ ├── time-distribution.ts # Time pattern generation
+ ├── generators.ts # Data generation logic
+ ├── inserter.ts # Batched ClickHouse insertion
+ └── README.md # This file
+```
+
+### Adding New Organization Profiles
+
+Edit `config.ts` and add to `ORGANIZATION_PROFILES`:
+
+```typescript
+export const ORGANIZATION_PROFILES = {
+ // ... existing profiles
+ startup: {
+ type: 'startup',
+ runsPerDayMin: 10,
+ runsPerDayMax: 100,
+ workflowsMin: 1,
+ workflowsMax: 3,
+ subscribersMin: 10,
+ subscribersMax: 100,
+ environmentsMin: 1,
+ environmentsMax: 1,
+ },
+};
+```
+
+### Adding New Workflow Templates
+
+Edit `config.ts` and add to `WORKFLOW_TEMPLATES`:
+
+```typescript
+export const WORKFLOW_TEMPLATES: WorkflowTemplate[] = [
+ // ... existing templates
+ {
+ type: 'support',
+ name: 'Support Ticket',
+ channels: ['email', 'sms'],
+ weight: 0.1
+ },
+];
+```
+
+## Performance Tips
+
+1. **Use high scale factors** for load testing: `--scale=10` or higher
+2. **Optimize batch size** based on your system's memory
+3. **Run during off-peak hours** to avoid impacting production systems
+4. **Monitor ClickHouse** resources during large imports
+5. **Use async inserts** (already enabled in the script)
+
+## Notes
+
+- Data is generated with realistic time distributions matching business patterns
+- All IDs are randomly generated and unique
+- Subscriber external IDs follow pattern: `user_1`, `user_2`, etc.
+- Materialized views process data asynchronously after insertion
+- TTL settings from schema definitions are respected
diff --git a/apps/api/scripts/clickhouse-seeder/config.ts b/apps/api/scripts/clickhouse-seeder/config.ts
new file mode 100644
index 00000000000..926052b608c
--- /dev/null
+++ b/apps/api/scripts/clickhouse-seeder/config.ts
@@ -0,0 +1,263 @@
+export interface SingleEnvironmentConfig {
+ enabled: boolean;
+ organizationId?: string;
+ environmentId?: string;
+ workflows: number;
+ subscribers: number;
+ runsPerDay: number;
+ workflowId?: string;
+ subscriberId?: string;
+}
+
+export interface SeederConfig {
+ organizations: number;
+ days: number;
+ scale: number;
+ batchSize: number;
+ startDate?: Date;
+ singleEnv?: SingleEnvironmentConfig;
+}
+
+export interface OrganizationProfile {
+ type: 'enterprise' | 'large' | 'medium';
+ runsPerDayMin: number;
+ runsPerDayMax: number;
+ workflowsMin: number;
+ workflowsMax: number;
+ subscribersMin: number;
+ subscribersMax: number;
+ environmentsMin: number;
+ environmentsMax: number;
+}
+
+export const ORGANIZATION_PROFILES: Record = {
+ enterprise: {
+ type: 'enterprise',
+ runsPerDayMin: 20000,
+ runsPerDayMax: 50000,
+ workflowsMin: 8,
+ workflowsMax: 15,
+ subscribersMin: 5000,
+ subscribersMax: 10000,
+ environmentsMin: 2,
+ environmentsMax: 3,
+ },
+ large: {
+ type: 'large',
+ runsPerDayMin: 5000,
+ runsPerDayMax: 15000,
+ workflowsMin: 5,
+ workflowsMax: 10,
+ subscribersMin: 1000,
+ subscribersMax: 5000,
+ environmentsMin: 2,
+ environmentsMax: 3,
+ },
+ medium: {
+ type: 'medium',
+ runsPerDayMin: 500,
+ runsPerDayMax: 2000,
+ workflowsMin: 3,
+ workflowsMax: 5,
+ subscribersMin: 100,
+ subscribersMax: 500,
+ environmentsMin: 1,
+ environmentsMax: 2,
+ },
+};
+
+export const ENTERPRISE_HEAVY_DISTRIBUTION = {
+ enterprise: 3,
+ large: 4,
+ medium: 3,
+};
+
+export interface WorkflowTemplate {
+ type: 'transactional' | 'marketing' | 'alerts' | 'multichannel';
+ name: string;
+ channels: string[];
+ weight: number;
+}
+
+export const WORKFLOW_TEMPLATES: WorkflowTemplate[] = [
+ { type: 'transactional', name: 'Order Confirmation', channels: ['email', 'in_app'], weight: 0.4 },
+ { type: 'marketing', name: 'Newsletter', channels: ['email'], weight: 0.25 },
+ { type: 'alerts', name: 'Critical Alert', channels: ['push', 'sms'], weight: 0.15 },
+ { type: 'multichannel', name: 'Campaign Update', channels: ['email', 'in_app', 'push'], weight: 0.2 },
+];
+
+export const WORKFLOW_RUN_STATUS_DISTRIBUTION = {
+ completed: 0.85,
+ processing: 0.05,
+ error: 0.1,
+};
+
+export const STEP_RUN_STATUS_DISTRIBUTION = {
+ completed: 0.88,
+ failed: 0.07,
+ skipped: 0.03,
+ delayed: 0.02,
+};
+
+export const DELIVERY_LIFECYCLE_STATUS_DISTRIBUTION = {
+ delivered: 0.7,
+ sent: 0.15,
+ errored: 0.08,
+ skipped: 0.04,
+ canceled: 0.02,
+ merged: 0.01,
+};
+
+export const TRACE_EVENT_TYPES = {
+ step_run: ['message_seen', 'message_read', 'message_clicked', 'message_archived'],
+ execution: ['step_created', 'step_queued', 'step_completed', 'step_canceled'],
+ delivery: ['message_sent', 'message_delivered', 'message_bounced', 'message_dropped'],
+};
+
+export const DEFAULT_SINGLE_ENV_CONFIG: SingleEnvironmentConfig = {
+ enabled: false,
+ workflows: 5,
+ subscribers: 1000,
+ runsPerDay: 5000,
+};
+
+export const DEFAULT_CONFIG: SeederConfig = {
+ organizations: 10,
+ days: 30,
+ scale: 1,
+ batchSize: 10000,
+};
+
+export function parseCliArgs(): SeederConfig {
+ const args = process.argv.slice(2);
+ const config: SeederConfig = { ...DEFAULT_CONFIG };
+ const singleEnvConfig: SingleEnvironmentConfig = { ...DEFAULT_SINGLE_ENV_CONFIG };
+
+ for (let i = 0; i < args.length; i++) {
+ let arg = args[i];
+ let value = args[i + 1];
+
+ if (arg.includes('=')) {
+ const [key, val] = arg.split('=');
+ arg = key;
+ value = val;
+ }
+
+ switch (arg) {
+ case '--single-env':
+ singleEnvConfig.enabled = true;
+ break;
+ case '--org-id':
+ singleEnvConfig.organizationId = value;
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--env-id':
+ singleEnvConfig.environmentId = value;
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--workflows':
+ case '-w':
+ singleEnvConfig.workflows = parseInt(value, 10);
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--subscribers':
+ singleEnvConfig.subscribers = parseInt(value, 10);
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--runs-per-day':
+ case '-r':
+ singleEnvConfig.runsPerDay = parseInt(value, 10);
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--workflow':
+ singleEnvConfig.workflowId = value;
+ singleEnvConfig.workflows = 1;
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--subscriber':
+ singleEnvConfig.subscriberId = value;
+ singleEnvConfig.subscribers = 1;
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--organizations':
+ case '-o':
+ config.organizations = parseInt(value, 10);
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--days':
+ case '-d':
+ config.days = parseInt(value, 10);
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--scale':
+ case '-s':
+ config.scale = parseFloat(value);
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--batch-size':
+ case '-b':
+ config.batchSize = parseInt(value, 10);
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--start-date':
+ config.startDate = new Date(value);
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--help':
+ case '-h':
+ printHelp();
+ process.exit(0);
+ }
+ }
+
+ if (!config.startDate) {
+ const now = new Date();
+ const msPerDay = 24 * 60 * 60 * 1000;
+ config.startDate = new Date(now.getTime() - config.days * msPerDay);
+ }
+
+ if (singleEnvConfig.enabled) {
+ config.singleEnv = singleEnvConfig;
+ }
+
+ return config;
+}
+
+function printHelp() {
+ console.log(`
+ClickHouse Data Seeding Script
+
+Usage: pnpm seed:clickhouse [options]
+
+Multi-Organization Mode (default):
+ -o, --organizations Number of organizations to create (default: 10)
+ -s, --scale Multiplier for data volume (default: 1)
+
+Single Environment Mode:
+ --single-env Enable single environment mode
+ --org-id Organization ID (optional, auto-generated if not provided)
+ --env-id Environment ID (optional, auto-generated if not provided)
+ -w, --workflows Number of workflows (default: 5)
+ --subscribers Number of subscribers (default: 1000)
+ -r, --runs-per-day Workflow runs per day (default: 5000)
+ --workflow Specific workflow ID to use (sets workflows to 1)
+ --subscriber Specific subscriber ID to use (sets subscribers to 1)
+
+Common Options:
+ -d, --days Days of data to generate (default: 30)
+ -b, --batch-size Records per ClickHouse insert batch (default: 10000)
+ --start-date Start date for data generation (default: first day of last month)
+ -h, --help Show this help message
+
+Examples:
+ # Multi-org mode (default)
+ pnpm seed:clickhouse
+ pnpm seed:clickhouse --scale=10 --organizations=50
+ pnpm seed:clickhouse --days=7 --scale=5
+
+ # Single environment mode
+ pnpm seed:clickhouse --single-env --days=7 --runs-per-day=10000
+ pnpm seed:clickhouse --single-env --org-id=abc123 --env-id=def456 --workflows=10 --subscribers=5000
+ pnpm seed:clickhouse --single-env --workflow=693ab23238cf527f6dc645d6 --subscriber=69395055051b1b19ff9e1b4c --org-id=69395056051b1b19ff9e1b52 --env-id=69395056c66fd6620f4521ba
+ `);
+}
diff --git a/apps/api/scripts/clickhouse-seeder/generators.ts b/apps/api/scripts/clickhouse-seeder/generators.ts
new file mode 100644
index 00000000000..3412157982c
--- /dev/null
+++ b/apps/api/scripts/clickhouse-seeder/generators.ts
@@ -0,0 +1,703 @@
+import { randomBytes } from 'crypto';
+import {
+ DELIVERY_LIFECYCLE_STATUS_DISTRIBUTION,
+ ENTERPRISE_HEAVY_DISTRIBUTION,
+ ORGANIZATION_PROFILES,
+ OrganizationProfile,
+ SeederConfig,
+ SingleEnvironmentConfig,
+ STEP_RUN_STATUS_DISTRIBUTION,
+ TRACE_EVENT_TYPES,
+ WORKFLOW_RUN_STATUS_DISTRIBUTION,
+ WORKFLOW_TEMPLATES,
+ WorkflowTemplate,
+} from './config';
+import { addRandomJitter, generateRandomTimestampsForDay } from './time-distribution';
+
+export interface Organization {
+ id: string;
+ name: string;
+ profile: OrganizationProfile;
+ environments: Environment[];
+}
+
+export interface Environment {
+ id: string;
+ name: string;
+ organizationId: string;
+ workflows: Workflow[];
+ subscribers: Subscriber[];
+}
+
+export interface Workflow {
+ id: string;
+ name: string;
+ triggerIdentifier: string;
+ environmentId: string;
+ organizationId: string;
+ channels: string[];
+ template: WorkflowTemplate;
+}
+
+export interface Subscriber {
+ id: string;
+ externalId: string;
+ environmentId: string;
+ organizationId: string;
+}
+
+export interface WorkflowRunRecord {
+ id: string;
+ created_at: Date;
+ updated_at: Date;
+ workflow_run_id: string;
+ workflow_id: string;
+ workflow_name: string;
+ organization_id: string;
+ environment_id: string;
+ user_id: string;
+ subscriber_id: string;
+ external_subscriber_id: string;
+ status: string;
+ trigger_identifier: string;
+ transaction_id: string;
+ channels: string;
+ subscriber_to: string;
+ payload: string;
+ control_values: string | null;
+ topics: string | null;
+ is_digest: string;
+ digested_workflow_run_id: string | null;
+ expires_at: Date;
+ delivery_lifecycle_status: string;
+ delivery_lifecycle_detail: string;
+ severity: string;
+ critical: boolean;
+ context_keys: string[];
+}
+
+export interface StepRunRecord {
+ id: string;
+ created_at: Date;
+ updated_at: Date;
+ step_run_id: string;
+ step_id: string;
+ workflow_run_id: string;
+ workflow_id: string;
+ organization_id: string;
+ environment_id: string;
+ user_id: string;
+ subscriber_id: string;
+ external_subscriber_id: string;
+ message_id: string | null;
+ context_keys: string[];
+ step_type: string;
+ step_name: string;
+ provider_id: string | null;
+ status: string;
+ deferred_ms: number | null;
+ error_code: string | null;
+ error_message: string | null;
+ transaction_id: string;
+ expires_at: Date;
+ digest: string | null;
+ schedule_extensions_count: number;
+}
+
+export interface TraceRecord {
+ id: string;
+ created_at: Date;
+ organization_id: string;
+ environment_id: string;
+ user_id: string;
+ external_subscriber_id: string;
+ subscriber_id: string;
+ event_type: string;
+ title: string;
+ message: string | null;
+ raw_data: string | null;
+ status: string;
+ entity_type: string;
+ entity_id: string;
+ expires_at: Date;
+ step_run_type: string;
+ workflow_run_identifier: string;
+ workflow_id: string;
+ provider_id: string | null;
+}
+
+function generateId(): string {
+ return randomBytes(12).toString('hex');
+}
+
+function randomInt(min: number, max: number): number {
+ return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+function randomChoice(items: T[]): T {
+ return items[Math.floor(Math.random() * items.length)];
+}
+
+function weightedRandomChoice(distribution: Record): string {
+ const total = Object.values(distribution).reduce((sum, weight) => sum + weight, 0);
+ let random = Math.random() * total;
+
+ for (const [key, weight] of Object.entries(distribution)) {
+ random -= weight;
+ if (random <= 0) {
+ return key;
+ }
+ }
+
+ return Object.keys(distribution)[0];
+}
+
+export function generateOrganizations(config: SeederConfig): Organization[] {
+ if (config.singleEnv?.enabled) {
+ return generateSingleEnvironment(config.singleEnv);
+ }
+
+ const organizations: Organization[] = [];
+ const distribution = ENTERPRISE_HEAVY_DISTRIBUTION;
+
+ let orgCount = 0;
+
+ for (const [profileType, count] of Object.entries(distribution)) {
+ const scaledCount = Math.ceil(count * (config.organizations / 10));
+
+ for (let i = 0; i < scaledCount && orgCount < config.organizations; i++) {
+ const profile = ORGANIZATION_PROFILES[profileType];
+ const orgId = generateId();
+
+ const org: Organization = {
+ id: orgId,
+ name: `${profile.type.charAt(0).toUpperCase() + profile.type.slice(1)} Organization ${orgCount + 1}`,
+ profile,
+ environments: [],
+ };
+
+ const numEnvironments = randomInt(profile.environmentsMin, profile.environmentsMax);
+ const envNames = ['Production', 'Staging', 'Development'];
+
+ for (let e = 0; e < numEnvironments; e++) {
+ const envId = generateId();
+ const env: Environment = {
+ id: envId,
+ name: envNames[e] || `Environment ${e + 1}`,
+ organizationId: orgId,
+ workflows: [],
+ subscribers: [],
+ };
+
+ const numWorkflows = randomInt(profile.workflowsMin, profile.workflowsMax);
+ for (let w = 0; w < numWorkflows; w++) {
+ const template = selectWorkflowTemplate();
+ const workflowId = generateId();
+
+ env.workflows.push({
+ id: workflowId,
+ name: `${template.name} ${w + 1}`,
+ triggerIdentifier: `${template.type}_${w + 1}`.toLowerCase().replace(/\s+/g, '_'),
+ environmentId: envId,
+ organizationId: orgId,
+ channels: template.channels,
+ template,
+ });
+ }
+
+ const numSubscribers = randomInt(profile.subscribersMin, profile.subscribersMax);
+ for (let s = 0; s < numSubscribers; s++) {
+ env.subscribers.push({
+ id: generateId(),
+ externalId: `user_${s + 1}`,
+ environmentId: envId,
+ organizationId: orgId,
+ });
+ }
+
+ org.environments.push(env);
+ }
+
+ organizations.push(org);
+ orgCount++;
+ }
+ }
+
+ return organizations;
+}
+
+export function generateSingleEnvironment(singleEnvConfig: SingleEnvironmentConfig): Organization[] {
+ const orgId = singleEnvConfig.organizationId || generateId();
+ const envId = singleEnvConfig.environmentId || generateId();
+
+ const customProfile: OrganizationProfile = {
+ type: 'enterprise',
+ runsPerDayMin: singleEnvConfig.runsPerDay,
+ runsPerDayMax: singleEnvConfig.runsPerDay,
+ workflowsMin: singleEnvConfig.workflows,
+ workflowsMax: singleEnvConfig.workflows,
+ subscribersMin: singleEnvConfig.subscribers,
+ subscribersMax: singleEnvConfig.subscribers,
+ environmentsMin: 1,
+ environmentsMax: 1,
+ };
+
+ const env: Environment = {
+ id: envId,
+ name: 'Production',
+ organizationId: orgId,
+ workflows: [],
+ subscribers: [],
+ };
+
+ for (let w = 0; w < singleEnvConfig.workflows; w++) {
+ const template = selectWorkflowTemplate();
+ const workflowId = singleEnvConfig.workflowId || generateId();
+
+ env.workflows.push({
+ id: workflowId,
+ name: `${template.name} ${w + 1}`,
+ triggerIdentifier: `${template.type}_${w + 1}`.toLowerCase().replace(/\s+/g, '_'),
+ environmentId: envId,
+ organizationId: orgId,
+ channels: template.channels,
+ template,
+ });
+ }
+
+ for (let s = 0; s < singleEnvConfig.subscribers; s++) {
+ const subscriberId = singleEnvConfig.subscriberId || generateId();
+ env.subscribers.push({
+ id: subscriberId,
+ externalId: `user_${s + 1}`,
+ environmentId: envId,
+ organizationId: orgId,
+ });
+ }
+
+ const org: Organization = {
+ id: orgId,
+ name: 'Single Environment Organization',
+ profile: customProfile,
+ environments: [env],
+ };
+
+ return [org];
+}
+
+function selectWorkflowTemplate(): WorkflowTemplate {
+ const random = Math.random();
+ let cumulative = 0;
+
+ for (const template of WORKFLOW_TEMPLATES) {
+ cumulative += template.weight;
+ if (random <= cumulative) {
+ return template;
+ }
+ }
+
+ return WORKFLOW_TEMPLATES[0];
+}
+
+export function generateWorkflowRuns(organizations: Organization[], config: SeederConfig): WorkflowRunRecord[] {
+ const allWorkflowRuns: WorkflowRunRecord[] = [];
+ const startDate = config.startDate ?? new Date();
+
+ for (const org of organizations) {
+ for (const env of org.environments) {
+ const runsPerDay = Math.floor(randomInt(org.profile.runsPerDayMin, org.profile.runsPerDayMax) * config.scale);
+
+ for (let day = 0; day < config.days; day++) {
+ const currentDate = new Date(startDate);
+ currentDate.setDate(startDate.getDate() + day);
+
+ const timestamps = generateRandomTimestampsForDay(currentDate, runsPerDay);
+
+ for (const timestamp of timestamps) {
+ const workflow = randomChoice(env.workflows);
+ const subscriber = randomChoice(env.subscribers);
+
+ const workflowRun = createWorkflowRunRecord(org, env, workflow, subscriber, timestamp);
+ allWorkflowRuns.push(workflowRun);
+ }
+ }
+ }
+ }
+
+ return allWorkflowRuns;
+}
+
+export interface GenerationProgress {
+ phase: string;
+ current: number;
+ total: number;
+ percentage: number;
+}
+
+export type ProgressCallback = (progress: GenerationProgress) => void;
+
+export interface DataBatch {
+ workflowRuns: WorkflowRunRecord[];
+ stepRuns: StepRunRecord[];
+ traces: TraceRecord[];
+}
+
+export function estimateTotalWorkflowRuns(organizations: Organization[], config: SeederConfig): number {
+ let total = 0;
+
+ for (const org of organizations) {
+ const envCount = org.environments.length;
+ const avgRunsPerDay = Math.floor(((org.profile.runsPerDayMin + org.profile.runsPerDayMax) / 2) * config.scale);
+ total += avgRunsPerDay * config.days * envCount;
+ }
+
+ return total;
+}
+
+export function* generateDataInBatches(
+ organizations: Organization[],
+ config: SeederConfig,
+ batchSize: number,
+ onProgress?: ProgressCallback
+): Generator {
+ const estimatedTotal = estimateTotalWorkflowRuns(organizations, config);
+ const startDate = config.startDate ?? new Date();
+
+ let processedWorkflowRuns = 0;
+ let pendingWorkflowRuns: WorkflowRunRecord[] = [];
+ let pendingStepRuns: StepRunRecord[] = [];
+ let pendingTraces: TraceRecord[] = [];
+
+ for (const org of organizations) {
+ for (const env of org.environments) {
+ const runsPerDay = Math.floor(randomInt(org.profile.runsPerDayMin, org.profile.runsPerDayMax) * config.scale);
+
+ for (let day = 0; day < config.days; day++) {
+ const currentDate = new Date(startDate);
+ currentDate.setDate(startDate.getDate() + day);
+
+ const timestamps = generateRandomTimestampsForDay(currentDate, runsPerDay);
+
+ for (const timestamp of timestamps) {
+ const workflow = randomChoice(env.workflows);
+ const subscriber = randomChoice(env.subscribers);
+
+ const workflowRun = createWorkflowRunRecord(org, env, workflow, subscriber, timestamp);
+ pendingWorkflowRuns.push(workflowRun);
+
+ const stepRuns = generateStepRunsForWorkflow(workflowRun, workflow);
+ pendingStepRuns.push(...stepRuns);
+
+ for (const stepRun of stepRuns) {
+ const traces = generateTracesForStepRun(stepRun);
+ pendingTraces.push(...traces);
+ }
+
+ processedWorkflowRuns++;
+
+ if (pendingWorkflowRuns.length >= batchSize) {
+ if (onProgress) {
+ onProgress({
+ phase: 'Generating data',
+ current: processedWorkflowRuns,
+ total: estimatedTotal,
+ percentage: Math.min(100, (processedWorkflowRuns / estimatedTotal) * 100),
+ });
+ }
+
+ yield {
+ workflowRuns: pendingWorkflowRuns,
+ stepRuns: pendingStepRuns,
+ traces: pendingTraces,
+ };
+
+ pendingWorkflowRuns = [];
+ pendingStepRuns = [];
+ pendingTraces = [];
+ }
+ }
+ }
+ }
+ }
+
+ if (pendingWorkflowRuns.length > 0) {
+ if (onProgress) {
+ onProgress({
+ phase: 'Generating data',
+ current: processedWorkflowRuns,
+ total: estimatedTotal,
+ percentage: 100,
+ });
+ }
+
+ yield {
+ workflowRuns: pendingWorkflowRuns,
+ stepRuns: pendingStepRuns,
+ traces: pendingTraces,
+ };
+ }
+}
+
+function generateStepRunsForWorkflow(workflowRun: WorkflowRunRecord, workflow: Workflow): StepRunRecord[] {
+ const stepRuns: StepRunRecord[] = [];
+ const channels = workflow.channels;
+
+ for (let i = 0; i < channels.length; i++) {
+ const channel = channels[i];
+ const stepCreatedAt = new Date(workflowRun.created_at.getTime() + i * 100);
+
+ const stepRun = createStepRunRecord(workflowRun, channel, stepCreatedAt, i);
+ stepRuns.push(stepRun);
+ }
+
+ return stepRuns;
+}
+
+function generateTracesForStepRun(stepRun: StepRunRecord): TraceRecord[] {
+ const traces: TraceRecord[] = [];
+ const numTraces = randomInt(2, 5);
+
+ for (let i = 0; i < numTraces; i++) {
+ const traceCreatedAt = new Date(stepRun.created_at.getTime() + i * 50);
+
+ const eventType = selectTraceEventType(i, numTraces, stepRun.status);
+ const trace = createTraceRecord(stepRun, eventType, traceCreatedAt);
+ traces.push(trace);
+ }
+
+ return traces;
+}
+
+function createWorkflowRunRecord(
+ org: Organization,
+ env: Environment,
+ workflow: Workflow,
+ subscriber: Subscriber,
+ createdAt: Date
+): WorkflowRunRecord {
+ const workflowRunId = generateId();
+ const transactionId = generateId();
+ const status = weightedRandomChoice(WORKFLOW_RUN_STATUS_DISTRIBUTION);
+ const deliveryStatus = weightedRandomChoice(DELIVERY_LIFECYCLE_STATUS_DISTRIBUTION);
+
+ const expiresAt = new Date(createdAt);
+ expiresAt.setDate(expiresAt.getDate() + 365);
+
+ return {
+ id: generateId(),
+ created_at: createdAt,
+ updated_at: addRandomJitter(createdAt, 1000),
+ workflow_run_id: workflowRunId,
+ workflow_id: workflow.id,
+ workflow_name: workflow.name,
+ organization_id: org.id,
+ environment_id: env.id,
+ user_id: generateId(),
+ subscriber_id: subscriber.id,
+ external_subscriber_id: subscriber.externalId,
+ status,
+ trigger_identifier: workflow.triggerIdentifier,
+ transaction_id: transactionId,
+ channels: JSON.stringify(workflow.channels),
+ subscriber_to: JSON.stringify({ email: `${subscriber.externalId}@example.com` }),
+ payload: JSON.stringify({ data: 'sample payload' }),
+ control_values: null,
+ topics: null,
+ is_digest: 'false',
+ digested_workflow_run_id: null,
+ expires_at: expiresAt,
+ delivery_lifecycle_status: deliveryStatus,
+ delivery_lifecycle_detail: '',
+ severity: Math.random() > 0.9 ? 'high' : 'none',
+ critical: Math.random() > 0.95,
+ context_keys: [],
+ };
+}
+
+export function generateStepRuns(workflowRuns: WorkflowRunRecord[], organizations: Organization[]): StepRunRecord[] {
+ const allStepRuns: StepRunRecord[] = [];
+
+ const orgMap = new Map();
+ for (const org of organizations) {
+ orgMap.set(org.id, org);
+ }
+
+ const workflowMap = new Map();
+ for (const org of organizations) {
+ for (const env of org.environments) {
+ for (const workflow of env.workflows) {
+ workflowMap.set(workflow.id, workflow);
+ }
+ }
+ }
+
+ for (const workflowRun of workflowRuns) {
+ const workflow = workflowMap.get(workflowRun.workflow_id);
+ if (!workflow) continue;
+
+ const channels = workflow.channels;
+
+ for (let i = 0; i < channels.length; i++) {
+ const channel = channels[i];
+ const stepCreatedAt = new Date(workflowRun.created_at.getTime() + i * 100);
+
+ const stepRun = createStepRunRecord(workflowRun, channel, stepCreatedAt, i);
+ allStepRuns.push(stepRun);
+ }
+ }
+
+ return allStepRuns;
+}
+
+function createStepRunRecord(
+ workflowRun: WorkflowRunRecord,
+ channel: string,
+ createdAt: Date,
+ _index: number
+): StepRunRecord {
+ const stepRunId = generateId();
+ const status = weightedRandomChoice(STEP_RUN_STATUS_DISTRIBUTION);
+
+ const providerMap: Record = {
+ email: ['sendgrid', 'ses', 'mailgun'],
+ sms: ['twilio', 'sns'],
+ push: ['fcm', 'apns'],
+ in_app: ['novu'],
+ chat: ['slack', 'discord'],
+ };
+
+ const providers = providerMap[channel] || ['novu'];
+ const providerId = randomChoice(providers);
+
+ const expiresAt = new Date(createdAt);
+ expiresAt.setDate(expiresAt.getDate() + 365);
+
+ return {
+ id: generateId(),
+ created_at: createdAt,
+ updated_at: addRandomJitter(createdAt, 500),
+ step_run_id: stepRunId,
+ step_id: generateId(),
+ workflow_run_id: workflowRun.workflow_run_id,
+ workflow_id: workflowRun.workflow_id,
+ organization_id: workflowRun.organization_id,
+ environment_id: workflowRun.environment_id,
+ user_id: workflowRun.user_id,
+ subscriber_id: workflowRun.subscriber_id,
+ external_subscriber_id: workflowRun.external_subscriber_id,
+ message_id: status === 'completed' ? generateId() : null,
+ context_keys: [],
+ step_type: channel,
+ step_name: `${channel} notification`,
+ provider_id: providerId,
+ status,
+ deferred_ms: null,
+ error_code: status === 'failed' ? 'PROVIDER_ERROR' : null,
+ error_message: status === 'failed' ? 'Failed to send notification' : null,
+ transaction_id: workflowRun.transaction_id,
+ expires_at: expiresAt,
+ digest: null,
+ schedule_extensions_count: 0,
+ };
+}
+
+export function generateTraces(stepRuns: StepRunRecord[]): TraceRecord[] {
+ const allTraces: TraceRecord[] = [];
+
+ for (const stepRun of stepRuns) {
+ const numTraces = randomInt(2, 5);
+
+ for (let i = 0; i < numTraces; i++) {
+ const traceCreatedAt = new Date(stepRun.created_at.getTime() + i * 50);
+
+ const eventType = selectTraceEventType(i, numTraces, stepRun.status);
+ const trace = createTraceRecord(stepRun, eventType, traceCreatedAt);
+ allTraces.push(trace);
+ }
+ }
+
+ return allTraces;
+}
+
+function selectTraceEventType(index: number, total: number, stepStatus: string): string {
+ if (index === 0) {
+ return 'step_created';
+ }
+
+ if (index === 1) {
+ if (stepStatus === 'completed') {
+ return 'message_sent';
+ }
+
+ return 'step_queued';
+ }
+
+ if (index === 2) {
+ return 'step_queued';
+ }
+
+ if (index === total - 1) {
+ if (stepStatus === 'completed') {
+ return 'step_completed';
+ } else if (stepStatus === 'failed') {
+ return 'step_canceled';
+ } else if (stepStatus === 'canceled') {
+ return 'step_canceled';
+ }
+
+ return 'step_completed';
+ }
+
+ const interactionEvents = TRACE_EVENT_TYPES.step_run;
+
+ return randomChoice(interactionEvents);
+}
+
+function createTraceRecord(stepRun: StepRunRecord, eventType: string, createdAt: Date): TraceRecord {
+ const expiresAt = new Date(createdAt);
+ expiresAt.setDate(expiresAt.getDate() + 365);
+
+ const statusMap: Record = {
+ step_completed: 'success',
+ step_canceled: 'error',
+ step_created: 'success',
+ step_queued: 'success',
+ message_sent: 'success',
+ message_delivered: 'success',
+ message_bounced: 'error',
+ message_dropped: 'error',
+ message_seen: 'success',
+ message_read: 'success',
+ message_clicked: 'success',
+ message_archived: 'success',
+ };
+
+ return {
+ id: generateId(),
+ created_at: createdAt,
+ organization_id: stepRun.organization_id,
+ environment_id: stepRun.environment_id,
+ user_id: stepRun.user_id,
+ external_subscriber_id: stepRun.external_subscriber_id,
+ subscriber_id: stepRun.subscriber_id,
+ event_type: eventType,
+ title: formatEventTitle(eventType),
+ message: null,
+ raw_data: null,
+ status: statusMap[eventType] || 'success',
+ entity_type: 'step_run',
+ entity_id: stepRun.step_run_id,
+ expires_at: expiresAt,
+ step_run_type: stepRun.step_type,
+ workflow_run_identifier: stepRun.workflow_run_id,
+ workflow_id: stepRun.workflow_id,
+ provider_id: stepRun.provider_id,
+ };
+}
+
+function formatEventTitle(eventType: string): string {
+ return eventType
+ .split('_')
+ .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
+ .join(' ');
+}
diff --git a/apps/api/scripts/clickhouse-seeder/inserter.ts b/apps/api/scripts/clickhouse-seeder/inserter.ts
new file mode 100644
index 00000000000..24e1e775f7c
--- /dev/null
+++ b/apps/api/scripts/clickhouse-seeder/inserter.ts
@@ -0,0 +1,173 @@
+import { ClickHouseClient } from '@clickhouse/client';
+
+export interface InsertStats {
+ workflowRuns: number;
+ stepRuns: number;
+ traces: number;
+ duration: number;
+}
+
+function formatDateForClickHouse(date: Date): string {
+ const year = date.getUTCFullYear();
+ const month = String(date.getUTCMonth() + 1).padStart(2, '0');
+ const day = String(date.getUTCDate()).padStart(2, '0');
+ const hours = String(date.getUTCHours()).padStart(2, '0');
+ const minutes = String(date.getUTCMinutes()).padStart(2, '0');
+ const seconds = String(date.getUTCSeconds()).padStart(2, '0');
+ const ms = String(date.getUTCMilliseconds()).padStart(3, '0');
+
+ return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${ms}`;
+}
+
+function transformRecordDates(record: any): any {
+ const transformed = { ...record };
+ for (const key of Object.keys(transformed)) {
+ if (transformed[key] instanceof Date) {
+ transformed[key] = formatDateForClickHouse(transformed[key]);
+ }
+ }
+
+ return transformed;
+}
+
+export class ClickHouseInserter {
+ private stats: InsertStats = {
+ workflowRuns: 0,
+ stepRuns: 0,
+ traces: 0,
+ duration: 0,
+ };
+
+ constructor(
+ private readonly client: ClickHouseClient,
+ private readonly batchSize: number
+ ) {}
+
+ async insertWorkflowRuns(records: any[]): Promise {
+ const startTime = Date.now();
+ await this.insertInBatches('workflow_runs', records, true);
+ this.stats.workflowRuns += records.length;
+ this.stats.duration += Date.now() - startTime;
+ }
+
+ async insertStepRuns(records: any[]): Promise {
+ const startTime = Date.now();
+ await this.insertInBatches('step_runs', records, true);
+ this.stats.stepRuns += records.length;
+ this.stats.duration += Date.now() - startTime;
+ }
+
+ async insertTraces(records: any[]): Promise {
+ const startTime = Date.now();
+ await this.insertInBatches('traces', records, true);
+ this.stats.traces += records.length;
+ this.stats.duration += Date.now() - startTime;
+ }
+
+ async insertWorkflowRunsSilent(records: any[]): Promise {
+ if (records.length === 0) return;
+
+ const startTime = Date.now();
+ await this.insertDirect('workflow_runs', records);
+ this.stats.workflowRuns += records.length;
+ this.stats.duration += Date.now() - startTime;
+ }
+
+ async insertStepRunsSilent(records: any[]): Promise {
+ if (records.length === 0) return;
+
+ const startTime = Date.now();
+ await this.insertDirect('step_runs', records);
+ this.stats.stepRuns += records.length;
+ this.stats.duration += Date.now() - startTime;
+ }
+
+ async insertTracesSilent(records: any[]): Promise {
+ if (records.length === 0) return;
+
+ const startTime = Date.now();
+ await this.insertDirect('traces', records);
+ this.stats.traces += records.length;
+ this.stats.duration += Date.now() - startTime;
+ }
+
+ private async insertDirect(table: string, records: any[]): Promise {
+ const transformedRecords = records.map(transformRecordDates);
+ await this.client.insert({
+ table,
+ values: transformedRecords,
+ format: 'JSONEachRow',
+ clickhouse_settings: {
+ async_insert: 1,
+ wait_for_async_insert: 1,
+ },
+ });
+ }
+
+ private async insertInBatches(table: string, records: any[], logProgress = true): Promise {
+ const totalBatches = Math.ceil(records.length / this.batchSize);
+
+ for (let i = 0; i < records.length; i += this.batchSize) {
+ const batch = records.slice(i, i + this.batchSize).map(transformRecordDates);
+ const currentBatch = Math.floor(i / this.batchSize) + 1;
+
+ await this.client.insert({
+ table,
+ values: batch,
+ format: 'JSONEachRow',
+ clickhouse_settings: {
+ async_insert: 1,
+ wait_for_async_insert: 1,
+ },
+ });
+
+ if (logProgress) {
+ this.logProgress(table, currentBatch, totalBatches, batch.length);
+ }
+ }
+ }
+
+ private logProgress(table: string, currentBatch: number, totalBatches: number, batchSize: number): void {
+ const percentage = ((currentBatch / totalBatches) * 100).toFixed(1);
+ console.log(` [${table}] Batch ${currentBatch}/${totalBatches} (${percentage}%) - ${batchSize} records inserted`);
+ }
+
+ getStats(): InsertStats {
+ return { ...this.stats };
+ }
+
+ printStats(): void {
+ console.log('\n' + '='.repeat(60));
+ console.log('Insertion Statistics');
+ console.log('='.repeat(60));
+ console.log(`Workflow Runs: ${this.stats.workflowRuns.toLocaleString()}`);
+ console.log(`Step Runs: ${this.stats.stepRuns.toLocaleString()}`);
+ console.log(`Traces: ${this.stats.traces.toLocaleString()}`);
+ console.log(
+ `Total Records: ${(this.stats.workflowRuns + this.stats.stepRuns + this.stats.traces).toLocaleString()}`
+ );
+ console.log(`Duration: ${(this.stats.duration / 1000).toFixed(2)}s`);
+ console.log('='.repeat(60) + '\n');
+ }
+}
+
+export function formatBytes(bytes: number): string {
+ if (bytes === 0) return '0 Bytes';
+
+ const k = 1024;
+ const sizes = ['Bytes', 'KB', 'MB', 'GB'];
+ const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+ return parseFloat((bytes / k ** i).toFixed(2)) + ' ' + sizes[i];
+}
+
+export function estimateDataSize(stats: InsertStats): string {
+ const avgWorkflowRunSize = 800;
+ const avgStepRunSize = 500;
+ const avgTraceSize = 400;
+
+ const totalBytes =
+ stats.workflowRuns * avgWorkflowRunSize + stats.stepRuns * avgStepRunSize + stats.traces * avgTraceSize;
+
+ return formatBytes(totalBytes);
+}
diff --git a/apps/api/scripts/clickhouse-seeder/time-distribution.ts b/apps/api/scripts/clickhouse-seeder/time-distribution.ts
new file mode 100644
index 00000000000..0575485987d
--- /dev/null
+++ b/apps/api/scripts/clickhouse-seeder/time-distribution.ts
@@ -0,0 +1,177 @@
+export interface TimeDistributionConfig {
+ businessHoursWeight: number;
+ weekendReduction: number;
+ enablePeakPatterns: boolean;
+}
+
+export const DEFAULT_TIME_CONFIG: TimeDistributionConfig = {
+ businessHoursWeight: 2.5,
+ weekendReduction: 0.3,
+ enablePeakPatterns: true,
+};
+
+export function isWeekend(date: Date): boolean {
+ const day = date.getDay();
+ return day === 0 || day === 6;
+}
+
+export function isBusinessHours(date: Date): boolean {
+ const hour = date.getHours();
+ return hour >= 9 && hour < 18;
+}
+
+export function getHourWeight(hour: number): number {
+ if (hour >= 9 && hour < 18) {
+ return 2.5;
+ }
+
+ if ((hour >= 7 && hour < 9) || (hour >= 18 && hour < 21)) {
+ return 1.2;
+ }
+
+ if (hour >= 21 || hour < 7) {
+ return 0.3;
+ }
+
+ return 1.0;
+}
+
+export function getDayWeight(date: Date, config: TimeDistributionConfig = DEFAULT_TIME_CONFIG): number {
+ let weight = 1.0;
+
+ if (isWeekend(date)) {
+ weight *= config.weekendReduction;
+ }
+
+ return weight;
+}
+
+export function getTimestampWeight(date: Date, config: TimeDistributionConfig = DEFAULT_TIME_CONFIG): number {
+ let weight = 1.0;
+
+ weight *= getDayWeight(date, config);
+
+ const hourWeight = getHourWeight(date.getHours());
+ weight *= hourWeight;
+
+ if (config.enablePeakPatterns) {
+ const peakModifier = getPeakPatternModifier(date);
+ weight *= peakModifier;
+ }
+
+ return weight;
+}
+
+function getPeakPatternModifier(date: Date): number {
+ const hour = date.getHours();
+ const dayOfWeek = date.getDay();
+ const dayOfMonth = date.getDate();
+
+ if (dayOfWeek === 2 && hour === 10) {
+ return 1.8;
+ }
+
+ if (dayOfWeek === 4 && hour === 14) {
+ return 1.5;
+ }
+
+ if (dayOfMonth === 1 && hour >= 8 && hour < 12) {
+ return 2.0;
+ }
+
+ if (dayOfMonth === 15 && hour >= 9 && hour < 11) {
+ return 1.7;
+ }
+
+ return 1.0;
+}
+
+export function generateRandomTimestampsForDay(
+ date: Date,
+ count: number,
+ config: TimeDistributionConfig = DEFAULT_TIME_CONFIG
+): Date[] {
+ const timestamps: Date[] = [];
+ const dayWeight = getDayWeight(date, config);
+ const adjustedCount = Math.floor(count * dayWeight);
+
+ const hourDistribution = calculateHourDistribution(adjustedCount, config);
+
+ for (let hour = 0; hour < 24; hour++) {
+ const countForHour = hourDistribution[hour];
+
+ for (let i = 0; i < countForHour; i++) {
+ const minute = Math.floor(Math.random() * 60);
+ const second = Math.floor(Math.random() * 60);
+ const millisecond = Math.floor(Math.random() * 1000);
+
+ const timestamp = new Date(
+ date.getFullYear(),
+ date.getMonth(),
+ date.getDate(),
+ hour,
+ minute,
+ second,
+ millisecond
+ );
+
+ timestamps.push(timestamp);
+ }
+ }
+
+ timestamps.sort((a, b) => a.getTime() - b.getTime());
+
+ return timestamps;
+}
+
+function calculateHourDistribution(totalCount: number, config: TimeDistributionConfig): number[] {
+ const hourWeights: number[] = [];
+ let totalWeight = 0;
+
+ for (let hour = 0; hour < 24; hour++) {
+ const weight = getHourWeight(hour);
+ hourWeights.push(weight);
+ totalWeight += weight;
+ }
+
+ const distribution: number[] = [];
+ let assignedCount = 0;
+
+ for (let hour = 0; hour < 24; hour++) {
+ const proportion = hourWeights[hour] / totalWeight;
+ let count = Math.floor(totalCount * proportion);
+
+ if (hour === 23) {
+ count = totalCount - assignedCount;
+ }
+
+ distribution.push(count);
+ assignedCount += count;
+ }
+
+ return distribution;
+}
+
+export function addRandomJitter(baseDate: Date, maxJitterMs: number = 5000): Date {
+ const jitter = Math.floor(Math.random() * maxJitterMs * 2) - maxJitterMs;
+ return new Date(baseDate.getTime() + jitter);
+}
+
+export function generateWorkflowRunTimestamps(
+ startDate: Date,
+ days: number,
+ runsPerDay: number,
+ config: TimeDistributionConfig = DEFAULT_TIME_CONFIG
+): Date[] {
+ const allTimestamps: Date[] = [];
+
+ for (let day = 0; day < days; day++) {
+ const currentDate = new Date(startDate);
+ currentDate.setDate(startDate.getDate() + day);
+
+ const timestampsForDay = generateRandomTimestampsForDay(currentDate, runsPerDay, config);
+ allTimestamps.push(...timestampsForDay);
+ }
+
+ return allTimestamps;
+}
diff --git a/apps/api/scripts/seed-clickhouse.ts b/apps/api/scripts/seed-clickhouse.ts
new file mode 100644
index 00000000000..8027d2b5dda
--- /dev/null
+++ b/apps/api/scripts/seed-clickhouse.ts
@@ -0,0 +1,206 @@
+import path from 'node:path';
+import dotenv from 'dotenv';
+
+dotenv.config({ path: path.join(__dirname, '..', 'src', '.env') });
+
+import { ClickHouseClient, createClient } from '@clickhouse/client';
+import { parseCliArgs } from './clickhouse-seeder/config';
+import {
+ estimateTotalWorkflowRuns,
+ GenerationProgress,
+ generateDataInBatches,
+ generateOrganizations,
+ Organization,
+} from './clickhouse-seeder/generators';
+import { ClickHouseInserter, estimateDataSize } from './clickhouse-seeder/inserter';
+
+function formatProgress(progress: GenerationProgress): string {
+ const barLength = 30;
+ const filledLength = Math.round((progress.percentage / 100) * barLength);
+ const bar = '█'.repeat(filledLength) + '░'.repeat(barLength - filledLength);
+
+ return ` [${bar}] ${progress.percentage.toFixed(1)}% (${progress.current.toLocaleString()}/${progress.total.toLocaleString()})`;
+}
+
+async function main() {
+ console.log('\n' + '='.repeat(60));
+ console.log('ClickHouse Data Seeding Script');
+ console.log('='.repeat(60) + '\n');
+
+ const config = parseCliArgs();
+
+ if (config.singleEnv?.enabled) {
+ console.log('Mode: Single Environment');
+ console.log('-'.repeat(60));
+ console.log(` Organization ID: ${config.singleEnv.organizationId || '(auto-generated)'}`);
+ console.log(` Environment ID: ${config.singleEnv.environmentId || '(auto-generated)'}`);
+ if (config.singleEnv.workflowId) {
+ console.log(` Workflow ID: ${config.singleEnv.workflowId}`);
+ } else {
+ console.log(` Workflows: ${config.singleEnv.workflows}`);
+ }
+ if (config.singleEnv.subscriberId) {
+ console.log(` Subscriber ID: ${config.singleEnv.subscriberId}`);
+ } else {
+ console.log(` Subscribers: ${config.singleEnv.subscribers.toLocaleString()}`);
+ }
+ console.log(` Runs/Day: ${config.singleEnv.runsPerDay.toLocaleString()}`);
+ console.log(` Days: ${config.days}`);
+ console.log(` Batch Size: ${config.batchSize.toLocaleString()}`);
+ console.log(` Start Date: ${config.startDate?.toISOString().split('T')[0]}`);
+ } else {
+ console.log('Mode: Multi-Organization');
+ console.log('-'.repeat(60));
+ console.log(` Organizations: ${config.organizations}`);
+ console.log(` Days: ${config.days}`);
+ console.log(` Scale: ${config.scale}x`);
+ console.log(` Batch Size: ${config.batchSize.toLocaleString()}`);
+ console.log(` Start Date: ${config.startDate?.toISOString().split('T')[0]}`);
+ }
+ console.log('');
+
+ if (!process.env.CLICK_HOUSE_URL || !process.env.CLICK_HOUSE_DATABASE) {
+ console.error('Error: ClickHouse environment variables not set');
+ console.error('Required: CLICK_HOUSE_URL, CLICK_HOUSE_DATABASE');
+ process.exit(1);
+ }
+
+ const client: ClickHouseClient = createClient({
+ url: process.env.CLICK_HOUSE_URL,
+ username: process.env.CLICK_HOUSE_USER,
+ password: process.env.CLICK_HOUSE_PASSWORD,
+ database: process.env.CLICK_HOUSE_DATABASE,
+ });
+
+ try {
+ console.log('Testing ClickHouse connection...');
+ await client.ping();
+ console.log('✓ Connected to ClickHouse\n');
+
+ console.log('Phase 1: Generating Organizations and Structure');
+ console.log('-'.repeat(60));
+ const organizations = generateOrganizations(config);
+
+ const totalEnvironments = organizations.reduce((sum, org) => sum + org.environments.length, 0);
+ const totalWorkflows = organizations.reduce(
+ (sum, org) => sum + org.environments.reduce((envSum, env) => envSum + env.workflows.length, 0),
+ 0
+ );
+ const totalSubscribers = organizations.reduce(
+ (sum, org) => sum + org.environments.reduce((envSum, env) => envSum + env.subscribers.length, 0),
+ 0
+ );
+
+ if (config.singleEnv?.enabled) {
+ const org = organizations[0];
+ const env = org.environments[0];
+ console.log(`✓ Generated single environment`);
+ console.log(` Organization ID: ${org.id}`);
+ console.log(` Environment ID: ${env.id}`);
+ console.log(` Workflows: ${totalWorkflows}`);
+ console.log(` Subscribers: ${totalSubscribers.toLocaleString()}`);
+ } else {
+ console.log(`✓ Generated ${organizations.length} organizations`);
+ console.log(` Environments: ${totalEnvironments}`);
+ console.log(` Workflows: ${totalWorkflows}`);
+ console.log(` Subscribers: ${totalSubscribers.toLocaleString()}`);
+ printOrganizationBreakdown(organizations);
+ }
+
+ const estimatedWorkflowRuns = estimateTotalWorkflowRuns(organizations, config);
+ console.log(`\nEstimated records to generate:`);
+ console.log(` Workflow runs: ~${estimatedWorkflowRuns.toLocaleString()}`);
+ console.log(` Step runs: ~${(estimatedWorkflowRuns * 2).toLocaleString()} (avg 2 steps/workflow)`);
+ console.log(
+ ` Traces: ~${Math.floor(estimatedWorkflowRuns * 2 * 3.5).toLocaleString()} (avg 3.5 traces/step)`
+ );
+
+ console.log('\nPhase 2: Generating and Inserting Data (Streaming)');
+ console.log('-'.repeat(60));
+
+ const inserter = new ClickHouseInserter(client, config.batchSize);
+ const startTime = Date.now();
+
+ let lastProgressLog = 0;
+ const progressLogInterval = 5;
+ let batchCount = 0;
+
+ const progressCallback = (progress: GenerationProgress) => {
+ const now = progress.percentage;
+ if (now - lastProgressLog >= progressLogInterval || now >= 100) {
+ process.stdout.write('\r' + formatProgress(progress));
+ lastProgressLog = Math.floor(now / progressLogInterval) * progressLogInterval;
+ }
+ };
+
+ const dataGenerator = generateDataInBatches(organizations, config, config.batchSize, progressCallback);
+
+ for (const batch of dataGenerator) {
+ batchCount++;
+
+ await Promise.all([
+ inserter.insertWorkflowRunsSilent(batch.workflowRuns),
+ inserter.insertStepRunsSilent(batch.stepRuns),
+ inserter.insertTracesSilent(batch.traces),
+ ]);
+ }
+
+ console.log('\n');
+
+ const stats = inserter.getStats();
+ const totalDuration = Date.now() - startTime;
+
+ console.log('✓ Data generation and insertion complete');
+ console.log(` Processed ${batchCount} batches in ${(totalDuration / 1000).toFixed(2)}s`);
+
+ inserter.printStats();
+
+ console.log('Additional Information:');
+ console.log(` Estimated Size: ${estimateDataSize(stats)}`);
+ const totalRecords = stats.workflowRuns + stats.stepRuns + stats.traces;
+ console.log(` Records/Second: ${(totalRecords / (totalDuration / 1000)).toFixed(0)}`);
+
+ console.log('\n✓ Data seeding completed successfully!');
+ console.log('\nNote: Materialized views will automatically populate aggregation tables:');
+ console.log(' - trace_rollup: Pre-aggregated counts by date/event_type/workflow/subscriber/provider');
+ console.log(' - delivery_trend_counts: Pre-aggregated delivery counts by step_type');
+ console.log(
+ ' Query trace_rollup for optimized analytics (message counts, active subscribers, interactions).\n'
+ );
+ } catch (error) {
+ console.error('\n✗ Error during seeding:', error);
+ throw error;
+ } finally {
+ await client.close();
+ }
+}
+
+function printOrganizationBreakdown(organizations: Organization[]) {
+ const breakdown = {
+ enterprise: 0,
+ large: 0,
+ medium: 0,
+ };
+
+ for (const org of organizations) {
+ breakdown[org.profile.type]++;
+ }
+
+ console.log('\n Organization Breakdown:');
+ console.log(` Enterprise: ${breakdown.enterprise}`);
+ console.log(` Large: ${breakdown.large}`);
+ console.log(` Medium: ${breakdown.medium}`);
+}
+
+if (require.main === module) {
+ main()
+ .then(() => {
+ process.exit(0);
+ })
+ .catch((error) => {
+ console.error('Fatal error:', error);
+ process.exit(1);
+ });
+}
+
+export { main };
diff --git a/apps/api/scripts/seed-triggers.ts b/apps/api/scripts/seed-triggers.ts
new file mode 100644
index 00000000000..1e9b8591e91
--- /dev/null
+++ b/apps/api/scripts/seed-triggers.ts
@@ -0,0 +1,313 @@
+import path from 'node:path';
+import dotenv from 'dotenv';
+
+dotenv.config({ path: path.join(__dirname, '..', 'src', '.env') });
+
+import '../src/config';
+import { NestFactory } from '@nestjs/core';
+import { AddressingTypeEnum, TriggerRequestCategoryEnum } from '@novu/shared';
+import { v4 as uuidv4 } from 'uuid';
+import { ParseEventRequestMulticastCommand } from '../src/app/events/usecases/parse-event-request/parse-event-request.command';
+import { ParseEventRequest } from '../src/app/events/usecases/parse-event-request/parse-event-request.usecase';
+import { AppModule } from '../src/app.module';
+
+interface SeedConfig {
+ workflow: string;
+ subscriber: string;
+ count: number;
+ organizationId: string;
+ environmentId: string;
+ userId: string;
+ delay: number;
+ payload: Record;
+ concurrent: number;
+}
+
+function parseCliArgs(): SeedConfig {
+ const args = process.argv.slice(2);
+ const config: Partial = {
+ delay: 0,
+ payload: {},
+ concurrent: 1,
+ count: 1,
+ };
+
+ for (let i = 0; i < args.length; i++) {
+ let arg = args[i];
+ let value = args[i + 1];
+
+ if (arg.includes('=')) {
+ const [key, val] = arg.split('=');
+ arg = key;
+ value = val;
+ }
+
+ switch (arg) {
+ case '--workflow':
+ case '-w':
+ config.workflow = value;
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--subscriber':
+ case '-s':
+ config.subscriber = value;
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--count':
+ case '-c':
+ config.count = parseInt(value, 10);
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--org-id':
+ config.organizationId = value;
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--env-id':
+ config.environmentId = value;
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--user-id':
+ config.userId = value;
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--delay':
+ case '-d':
+ config.delay = parseInt(value, 10);
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--payload':
+ case '-p':
+ try {
+ config.payload = JSON.parse(value);
+ } catch (error) {
+ console.error('Error: Invalid JSON payload');
+ process.exit(1);
+ }
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--concurrent':
+ config.concurrent = parseInt(value, 10);
+ if (!args[i].includes('=')) i++;
+ break;
+ case '--help':
+ case '-h':
+ printHelp();
+ process.exit(0);
+ }
+ }
+
+ const required: Array = ['workflow', 'subscriber', 'organizationId', 'environmentId', 'userId'];
+ const missing = required.filter((key) => !config[key]);
+
+ if (missing.length > 0) {
+ console.error(`Error: Missing required arguments: ${missing.join(', ')}`);
+ console.error('Run with --help for usage information');
+ process.exit(1);
+ }
+
+ if (!config.workflow || !config.subscriber || !config.organizationId || !config.environmentId || !config.userId) {
+ console.error('Error: Missing required arguments');
+ process.exit(1);
+ }
+
+ return {
+ workflow: config.workflow,
+ subscriber: config.subscriber,
+ count: config.count ?? 1,
+ organizationId: config.organizationId,
+ environmentId: config.environmentId,
+ userId: config.userId,
+ delay: config.delay ?? 0,
+ payload: config.payload ?? {},
+ concurrent: config.concurrent ?? 1,
+ };
+}
+
+function printHelp() {
+ console.log(`
+Natural Trigger Seed Script
+
+Usage: pnpm seed:triggers [options]
+
+Required Arguments:
+ -w, --workflow Workflow identifier (trigger name)
+ -s, --subscriber Subscriber ID to send to
+ --org-id Organization ID
+ --env-id Environment ID
+ --user-id User ID
+
+Optional Arguments:
+ -c, --count Number of triggers to execute (default: 1)
+ -d, --delay Delay between triggers in milliseconds (default: 0)
+ -p, --payload JSON payload to include (default: {})
+ --concurrent Number of concurrent triggers (default: 1)
+ -h, --help Show this help message
+
+Examples:
+ # Basic usage - trigger 100 times
+ pnpm seed:triggers \\
+ --workflow=my-workflow \\
+ --subscriber=subscriber-123 \\
+ --count=100 \\
+ --org-id=org-abc \\
+ --env-id=env-xyz \\
+ --user-id=user-456
+
+ # With custom payload and delay
+ pnpm seed:triggers \\
+ --workflow=order-confirmation \\
+ --subscriber=user@example.com \\
+ --count=50 \\
+ --org-id=org-abc \\
+ --env-id=env-xyz \\
+ --user-id=user-456 \\
+ --delay=100 \\
+ --payload='{"orderId":"12345","amount":99.99}'
+
+ # Concurrent triggers
+ pnpm seed:triggers \\
+ --workflow=newsletter \\
+ --subscriber=subscriber-123 \\
+ --count=1000 \\
+ --org-id=org-abc \\
+ --env-id=env-xyz \\
+ --user-id=user-456 \\
+ --concurrent=10
+ `);
+}
+
+function formatProgress(current: number, total: number): string {
+ const barLength = 30;
+ const percentage = (current / total) * 100;
+ const filledLength = Math.round((percentage / 100) * barLength);
+ const bar = '█'.repeat(filledLength) + '░'.repeat(barLength - filledLength);
+
+ return ` [${bar}] ${percentage.toFixed(1)}% (${current.toLocaleString()}/${total.toLocaleString()})`;
+}
+
+async function sleep(ms: number): Promise {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function triggerBatch(
+ parseEventRequest: ParseEventRequest,
+ config: SeedConfig,
+ batchSize: number,
+ successCount: { value: number },
+ errorCount: { value: number }
+): Promise {
+ const promises: Promise[] = [];
+
+ for (let i = 0; i < batchSize; i++) {
+ const promise = (async () => {
+ try {
+ await parseEventRequest.execute(
+ ParseEventRequestMulticastCommand.create({
+ userId: config.userId,
+ environmentId: config.environmentId,
+ organizationId: config.organizationId,
+ identifier: config.workflow,
+ payload: config.payload || {},
+ overrides: {},
+ to: [config.subscriber],
+ addressingType: AddressingTypeEnum.MULTICAST,
+ requestCategory: TriggerRequestCategoryEnum.SINGLE,
+ requestId: uuidv4(),
+ })
+ );
+ successCount.value++;
+ } catch (error) {
+ errorCount.value++;
+ console.error(`\nError triggering event: ${error.message}`);
+ }
+ })();
+
+ promises.push(promise);
+ }
+
+ await Promise.all(promises);
+}
+
+async function main() {
+ console.log('\n' + '='.repeat(60));
+ console.log('Natural Trigger Seed Script');
+ console.log('='.repeat(60) + '\n');
+
+ const config = parseCliArgs();
+
+ console.log('Configuration:');
+ console.log('-'.repeat(60));
+ console.log(` Workflow: ${config.workflow}`);
+ console.log(` Subscriber: ${config.subscriber}`);
+ console.log(` Count: ${config.count.toLocaleString()}`);
+ console.log(` Organization: ${config.organizationId}`);
+ console.log(` Environment: ${config.environmentId}`);
+ console.log(` User: ${config.userId}`);
+ console.log(` Delay: ${config.delay}ms`);
+ console.log(` Concurrent: ${config.concurrent}`);
+ console.log(` Payload: ${JSON.stringify(config.payload)}`);
+ console.log('');
+
+ console.log('Bootstrapping NestJS application...');
+ const app = await NestFactory.create(AppModule, {
+ logger: false,
+ });
+ console.log('✓ Application bootstrapped\n');
+
+ const parseEventRequest = app.get(ParseEventRequest);
+ console.log('✓ ParseEventRequest service retrieved\n');
+
+ console.log('Starting trigger execution:');
+ console.log('-'.repeat(60));
+
+ const startTime = Date.now();
+ const successCount = { value: 0 };
+ const errorCount = { value: 0 };
+ let processed = 0;
+
+ const totalBatches = Math.ceil(config.count / config.concurrent);
+
+ for (let batch = 0; batch < totalBatches; batch++) {
+ const batchSize = Math.min(config.concurrent, config.count - processed);
+
+ await triggerBatch(parseEventRequest, config, batchSize, successCount, errorCount);
+
+ processed += batchSize;
+ process.stdout.write('\r' + formatProgress(processed, config.count));
+
+ if (config.delay > 0 && processed < config.count) {
+ await sleep(config.delay);
+ }
+ }
+
+ console.log('\n');
+
+ const totalDuration = Date.now() - startTime;
+ const durationSeconds = totalDuration / 1000;
+
+ console.log('✓ Trigger execution complete');
+ console.log('-'.repeat(60));
+ console.log(` Total triggers: ${config.count.toLocaleString()}`);
+ console.log(` Successful: ${successCount.value.toLocaleString()}`);
+ console.log(` Failed: ${errorCount.value.toLocaleString()}`);
+ console.log(` Duration: ${durationSeconds.toFixed(2)}s`);
+ console.log(` Rate: ${(config.count / durationSeconds).toFixed(2)} triggers/second`);
+ console.log('');
+
+ console.log('✓ Seeding completed successfully!\n');
+
+ await app.close();
+}
+
+if (require.main === module) {
+ main()
+ .then(() => {
+ process.exit(0);
+ })
+ .catch((error) => {
+ console.error('Fatal error:', error);
+ process.exit(1);
+ });
+}
+
+export { main };
diff --git a/apps/api/src/app/activity/activity.module.ts b/apps/api/src/app/activity/activity.module.ts
index daa05173508..767934cef64 100644
--- a/apps/api/src/app/activity/activity.module.ts
+++ b/apps/api/src/app/activity/activity.module.ts
@@ -1,4 +1,5 @@
import { Module } from '@nestjs/common';
+import { WorkflowRunService } from '@novu/application-generic';
import { SharedModule } from '../shared/shared.module';
import { ActivityController } from './activity.controller';
import { BuildActiveSubscribersChart } from './usecases/build-active-subscribers-chart/build-active-subscribers-chart.usecase';
@@ -18,7 +19,6 @@ import { GetRequest } from './usecases/get-request/get-request.usecase';
import { GetRequests } from './usecases/get-requests/get-requests.usecase';
import { GetWorkflowRun } from './usecases/get-workflow-run/get-workflow-run.usecase';
import { GetWorkflowRuns } from './usecases/get-workflow-runs/get-workflow-runs.usecase';
-import { WorkflowRunService } from '@novu/application-generic';
const USE_CASES = [
GetRequests,
diff --git a/apps/api/src/app/activity/usecases/build-active-subscribers-chart/build-active-subscribers-chart.usecase.ts b/apps/api/src/app/activity/usecases/build-active-subscribers-chart/build-active-subscribers-chart.usecase.ts
index bd887e40478..e3383f1fcf9 100644
--- a/apps/api/src/app/activity/usecases/build-active-subscribers-chart/build-active-subscribers-chart.usecase.ts
+++ b/apps/api/src/app/activity/usecases/build-active-subscribers-chart/build-active-subscribers-chart.usecase.ts
@@ -1,12 +1,21 @@
import { Injectable } from '@nestjs/common';
-import { InstrumentUsecase, PinoLogger, WorkflowRunRepository } from '@novu/application-generic';
+import {
+ FeatureFlagsService,
+ InstrumentUsecase,
+ PinoLogger,
+ TraceRollupRepository,
+ WorkflowRunRepository,
+} from '@novu/application-generic';
+import { FeatureFlagsKeysEnum } from '@novu/shared';
import { ActiveSubscribersDataPointDto } from '../../dtos/get-charts.response.dto';
import { BuildActiveSubscribersChartCommand } from './build-active-subscribers-chart.command';
@Injectable()
export class BuildActiveSubscribersChart {
constructor(
+ private traceRollupRepository: TraceRollupRepository,
private workflowRunRepository: WorkflowRunRepository,
+ private featureFlagsService: FeatureFlagsService,
private logger: PinoLogger
) {
this.logger.setContext(BuildActiveSubscribersChart.name);
@@ -16,20 +25,49 @@ export class BuildActiveSubscribersChart {
async execute(command: BuildActiveSubscribersChartCommand): Promise {
const { environmentId, organizationId, startDate, endDate, workflowIds } = command;
- // Calculate previous period dates
const periodDuration = endDate.getTime() - startDate.getTime();
const previousEndDate = new Date(startDate.getTime() - 1);
const previousStartDate = new Date(previousEndDate.getTime() - periodDuration);
- const result = await this.workflowRunRepository.getActiveSubscribersData(
- environmentId,
- organizationId,
- startDate,
- endDate,
- previousStartDate,
- previousEndDate,
- workflowIds
- );
+ const featureFlagContext = {
+ organization: { _id: organizationId },
+ environment: { _id: environmentId },
+ };
+
+ const [isGlobalEnabled, isDedicatedEnabled] = await Promise.all([
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_LOGS_READ_GLOBAL_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_ACTIVE_SUBSCRIBERS_READ_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ ]);
+
+ const useNewQuery = isGlobalEnabled || isDedicatedEnabled;
+
+ const result = useNewQuery
+ ? await this.traceRollupRepository.getActiveSubscribersCount(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ previousStartDate,
+ previousEndDate,
+ workflowIds
+ )
+ : await this.workflowRunRepository.getActiveSubscribersData(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ previousStartDate,
+ previousEndDate,
+ workflowIds
+ );
return {
currentPeriod: result.currentPeriod,
diff --git a/apps/api/src/app/activity/usecases/build-active-subscribers-trend-chart/build-active-subscribers-trend-chart.usecase.ts b/apps/api/src/app/activity/usecases/build-active-subscribers-trend-chart/build-active-subscribers-trend-chart.usecase.ts
index 828e72334f3..8be04dfdd73 100644
--- a/apps/api/src/app/activity/usecases/build-active-subscribers-trend-chart/build-active-subscribers-trend-chart.usecase.ts
+++ b/apps/api/src/app/activity/usecases/build-active-subscribers-trend-chart/build-active-subscribers-trend-chart.usecase.ts
@@ -1,12 +1,21 @@
import { Injectable } from '@nestjs/common';
-import { InstrumentUsecase, PinoLogger, WorkflowRunRepository } from '@novu/application-generic';
+import {
+ FeatureFlagsService,
+ InstrumentUsecase,
+ PinoLogger,
+ TraceRollupRepository,
+ WorkflowRunRepository,
+} from '@novu/application-generic';
+import { FeatureFlagsKeysEnum } from '@novu/shared';
import { ActiveSubscribersTrendDataPointDto } from '../../dtos/get-charts.response.dto';
import { BuildActiveSubscribersTrendChartCommand } from './build-active-subscribers-trend-chart.command';
@Injectable()
export class BuildActiveSubscribersTrendChart {
constructor(
+ private traceRollupRepository: TraceRollupRepository,
private workflowRunRepository: WorkflowRunRepository,
+ private featureFlagsService: FeatureFlagsService,
private logger: PinoLogger
) {
this.logger.setContext(BuildActiveSubscribersTrendChart.name);
@@ -16,13 +25,41 @@ export class BuildActiveSubscribersTrendChart {
async execute(command: BuildActiveSubscribersTrendChartCommand): Promise {
const { environmentId, organizationId, startDate, endDate, workflowIds } = command;
- const activeSubscribers = await this.workflowRunRepository.getActiveSubscribersTrendData(
- environmentId,
- organizationId,
- startDate,
- endDate,
- workflowIds
- );
+ const featureFlagContext = {
+ organization: { _id: organizationId },
+ environment: { _id: environmentId },
+ };
+
+ const [isGlobalEnabled, isDedicatedEnabled] = await Promise.all([
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_LOGS_READ_GLOBAL_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_ACTIVE_SUBSCRIBER_TREND_READ_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ ]);
+
+ const useNewQuery = isGlobalEnabled || isDedicatedEnabled;
+
+ const activeSubscribers = useNewQuery
+ ? await this.traceRollupRepository.getActiveSubscribersTrendData(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ workflowIds
+ )
+ : await this.workflowRunRepository.getActiveSubscribersTrendData(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ workflowIds
+ );
const chartDataMap = new Map();
diff --git a/apps/api/src/app/activity/usecases/build-avg-messages-per-subscriber-chart/build-avg-messages-per-subscriber-chart.usecase.ts b/apps/api/src/app/activity/usecases/build-avg-messages-per-subscriber-chart/build-avg-messages-per-subscriber-chart.usecase.ts
index 4d7a1c16eef..610da0bde25 100644
--- a/apps/api/src/app/activity/usecases/build-avg-messages-per-subscriber-chart/build-avg-messages-per-subscriber-chart.usecase.ts
+++ b/apps/api/src/app/activity/usecases/build-avg-messages-per-subscriber-chart/build-avg-messages-per-subscriber-chart.usecase.ts
@@ -1,12 +1,21 @@
import { Injectable } from '@nestjs/common';
-import { InstrumentUsecase, PinoLogger, StepRunRepository } from '@novu/application-generic';
+import {
+ FeatureFlagsService,
+ InstrumentUsecase,
+ PinoLogger,
+ StepRunRepository,
+ TraceRollupRepository,
+} from '@novu/application-generic';
+import { FeatureFlagsKeysEnum } from '@novu/shared';
import { AvgMessagesPerSubscriberDataPointDto } from '../../dtos/get-charts.response.dto';
import { BuildAvgMessagesPerSubscriberChartCommand } from './build-avg-messages-per-subscriber-chart.command';
@Injectable()
export class BuildAvgMessagesPerSubscriberChart {
constructor(
+ private traceRollupRepository: TraceRollupRepository,
private stepRunRepository: StepRunRepository,
+ private featureFlagsService: FeatureFlagsService,
private logger: PinoLogger
) {
this.logger.setContext(BuildAvgMessagesPerSubscriberChart.name);
@@ -16,20 +25,49 @@ export class BuildAvgMessagesPerSubscriberChart {
async execute(command: BuildAvgMessagesPerSubscriberChartCommand): Promise {
const { environmentId, organizationId, startDate, endDate, workflowIds } = command;
- // Calculate previous period dates
const periodDuration = endDate.getTime() - startDate.getTime();
const previousEndDate = new Date(startDate.getTime() - 1);
const previousStartDate = new Date(previousEndDate.getTime() - periodDuration);
- const result = await this.stepRunRepository.getAvgMessagesPerSubscriberData(
- environmentId,
- organizationId,
- startDate,
- endDate,
- previousStartDate,
- previousEndDate,
- workflowIds
- );
+ const featureFlagContext = {
+ organization: { _id: organizationId },
+ environment: { _id: environmentId },
+ };
+
+ const [isGlobalEnabled, isDedicatedEnabled] = await Promise.all([
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_LOGS_READ_GLOBAL_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_AVG_MESSAGES_PER_SUBSCRIBER_READ_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ ]);
+
+ const useNewQuery = isGlobalEnabled || isDedicatedEnabled;
+
+ const result = useNewQuery
+ ? await this.traceRollupRepository.getAvgMessagesPerSubscriberData(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ previousStartDate,
+ previousEndDate,
+ workflowIds
+ )
+ : await this.stepRunRepository.getAvgMessagesPerSubscriberData(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ previousStartDate,
+ previousEndDate,
+ workflowIds
+ );
return {
currentPeriod: result.currentPeriod,
diff --git a/apps/api/src/app/activity/usecases/build-delivery-trend-chart/build-delivery-trend-chart.usecase.ts b/apps/api/src/app/activity/usecases/build-delivery-trend-chart/build-delivery-trend-chart.usecase.ts
index 9eaaef288f3..6f1e5dc16c4 100644
--- a/apps/api/src/app/activity/usecases/build-delivery-trend-chart/build-delivery-trend-chart.usecase.ts
+++ b/apps/api/src/app/activity/usecases/build-delivery-trend-chart/build-delivery-trend-chart.usecase.ts
@@ -1,12 +1,21 @@
import { Injectable } from '@nestjs/common';
-import { InstrumentUsecase, PinoLogger, StepRunRepository } from '@novu/application-generic';
+import {
+ DeliveryTrendCountsRepository,
+ FeatureFlagsService,
+ InstrumentUsecase,
+ PinoLogger,
+ StepRunRepository,
+} from '@novu/application-generic';
+import { FeatureFlagsKeysEnum } from '@novu/shared';
import { ChartDataPointDto } from '../../dtos/get-charts.response.dto';
import { BuildDeliveryTrendChartCommand } from './build-delivery-trend-chart.command';
@Injectable()
export class BuildDeliveryTrendChart {
constructor(
+ private deliveryTrendCountsRepository: DeliveryTrendCountsRepository,
private stepRunRepository: StepRunRepository,
+ private featureFlagsService: FeatureFlagsService,
private logger: PinoLogger
) {
this.logger.setContext(BuildDeliveryTrendChart.name);
@@ -16,13 +25,41 @@ export class BuildDeliveryTrendChart {
async execute(command: BuildDeliveryTrendChartCommand): Promise {
const { environmentId, organizationId, startDate, endDate, workflowIds } = command;
- const stepRuns = await this.stepRunRepository.getDeliveryTrendData(
- environmentId,
- organizationId,
- startDate,
- endDate,
- workflowIds
- );
+ const featureFlagContext = {
+ organization: { _id: organizationId },
+ environment: { _id: environmentId },
+ };
+
+ const [isGlobalEnabled, isDedicatedEnabled] = await Promise.all([
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_LOGS_READ_GLOBAL_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_DELIVERY_TREND_READ_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ ]);
+
+ const useNewQuery = isGlobalEnabled || isDedicatedEnabled;
+
+ const stepRuns = useNewQuery
+ ? await this.deliveryTrendCountsRepository.getDeliveryTrendData(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ workflowIds
+ )
+ : await this.stepRunRepository.getDeliveryTrendData(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ workflowIds
+ );
const chartDataMap = new Map>();
diff --git a/apps/api/src/app/activity/usecases/build-interaction-trend-chart/build-interaction-trend-chart.usecase.ts b/apps/api/src/app/activity/usecases/build-interaction-trend-chart/build-interaction-trend-chart.usecase.ts
index 449fbeddade..618cf64e740 100644
--- a/apps/api/src/app/activity/usecases/build-interaction-trend-chart/build-interaction-trend-chart.usecase.ts
+++ b/apps/api/src/app/activity/usecases/build-interaction-trend-chart/build-interaction-trend-chart.usecase.ts
@@ -1,12 +1,21 @@
import { Injectable } from '@nestjs/common';
-import { InstrumentUsecase, PinoLogger, TraceLogRepository } from '@novu/application-generic';
+import {
+ FeatureFlagsService,
+ InstrumentUsecase,
+ PinoLogger,
+ TraceLogRepository,
+ TraceRollupRepository,
+} from '@novu/application-generic';
+import { FeatureFlagsKeysEnum } from '@novu/shared';
import { InteractionTrendDataPointDto } from '../../dtos/get-charts.response.dto';
import { BuildInteractionTrendChartCommand } from './build-interaction-trend-chart.command';
@Injectable()
export class BuildInteractionTrendChart {
constructor(
+ private traceRollupRepository: TraceRollupRepository,
private traceLogRepository: TraceLogRepository,
+ private featureFlagsService: FeatureFlagsService,
private logger: PinoLogger
) {
this.logger.setContext(BuildInteractionTrendChart.name);
@@ -16,13 +25,41 @@ export class BuildInteractionTrendChart {
async execute(command: BuildInteractionTrendChartCommand): Promise {
const { environmentId, organizationId, startDate, endDate, workflowIds } = command;
- const traces = await this.traceLogRepository.getInteractionTrendData(
- environmentId,
- organizationId,
- startDate,
- endDate,
- workflowIds
- );
+ const featureFlagContext = {
+ organization: { _id: organizationId },
+ environment: { _id: environmentId },
+ };
+
+ const [isGlobalEnabled, isDedicatedEnabled] = await Promise.all([
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_LOGS_READ_GLOBAL_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_INTERACTION_TREND_READ_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ ]);
+
+ const useNewQuery = isGlobalEnabled || isDedicatedEnabled;
+
+ const traces = useNewQuery
+ ? await this.traceRollupRepository.getInteractionTrendData(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ workflowIds
+ )
+ : await this.traceLogRepository.getInteractionTrendData(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ workflowIds
+ );
const chartDataMap = new Map>();
diff --git a/apps/api/src/app/activity/usecases/build-messages-delivered-chart/build-messages-delivered-chart.usecase.ts b/apps/api/src/app/activity/usecases/build-messages-delivered-chart/build-messages-delivered-chart.usecase.ts
index bd1fe21bf49..84af32a29a9 100644
--- a/apps/api/src/app/activity/usecases/build-messages-delivered-chart/build-messages-delivered-chart.usecase.ts
+++ b/apps/api/src/app/activity/usecases/build-messages-delivered-chart/build-messages-delivered-chart.usecase.ts
@@ -1,12 +1,21 @@
import { Injectable } from '@nestjs/common';
-import { InstrumentUsecase, PinoLogger, StepRunRepository } from '@novu/application-generic';
+import {
+ FeatureFlagsService,
+ InstrumentUsecase,
+ PinoLogger,
+ StepRunRepository,
+ TraceRollupRepository,
+} from '@novu/application-generic';
+import { FeatureFlagsKeysEnum } from '@novu/shared';
import { MessagesDeliveredDataPointDto } from '../../dtos/get-charts.response.dto';
import { BuildMessagesDeliveredChartCommand } from './build-messages-delivered-chart.command';
@Injectable()
export class BuildMessagesDeliveredChart {
constructor(
+ private traceRollupRepository: TraceRollupRepository,
private stepRunRepository: StepRunRepository,
+ private featureFlagsService: FeatureFlagsService,
private logger: PinoLogger
) {
this.logger.setContext(BuildMessagesDeliveredChart.name);
@@ -16,20 +25,49 @@ export class BuildMessagesDeliveredChart {
async execute(command: BuildMessagesDeliveredChartCommand): Promise {
const { environmentId, organizationId, startDate, endDate, workflowIds } = command;
- // Calculate previous period dates
const periodDuration = endDate.getTime() - startDate.getTime();
- const previousEndDate = new Date(startDate.getTime() - 1); // Day before start date
+ const previousEndDate = new Date(startDate.getTime() - 1);
const previousStartDate = new Date(previousEndDate.getTime() - periodDuration);
- const result = await this.stepRunRepository.getMessagesDeliveredData(
- environmentId,
- organizationId,
- startDate,
- endDate,
- previousStartDate,
- previousEndDate,
- workflowIds
- );
+ const featureFlagContext = {
+ organization: { _id: organizationId },
+ environment: { _id: environmentId },
+ };
+
+ const [isGlobalEnabled, isDedicatedEnabled] = await Promise.all([
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_LOGS_READ_GLOBAL_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_MESSAGE_DELIVERY_READ_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ ]);
+
+ const useNewQuery = isGlobalEnabled || isDedicatedEnabled;
+
+ const result = useNewQuery
+ ? await this.traceRollupRepository.getMessageSendCount(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ previousStartDate,
+ previousEndDate,
+ workflowIds
+ )
+ : await this.stepRunRepository.getMessagesDeliveredData(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ previousStartDate,
+ previousEndDate,
+ workflowIds
+ );
return {
currentPeriod: result.currentPeriod,
diff --git a/apps/api/src/app/activity/usecases/build-provider-by-volume-chart/build-provider-by-volume-chart.usecase.ts b/apps/api/src/app/activity/usecases/build-provider-by-volume-chart/build-provider-by-volume-chart.usecase.ts
index 356c521000d..c0a0609c870 100644
--- a/apps/api/src/app/activity/usecases/build-provider-by-volume-chart/build-provider-by-volume-chart.usecase.ts
+++ b/apps/api/src/app/activity/usecases/build-provider-by-volume-chart/build-provider-by-volume-chart.usecase.ts
@@ -1,12 +1,21 @@
import { Injectable } from '@nestjs/common';
-import { InstrumentUsecase, PinoLogger, StepRunRepository } from '@novu/application-generic';
+import {
+ FeatureFlagsService,
+ InstrumentUsecase,
+ PinoLogger,
+ StepRunRepository,
+ TraceRollupRepository,
+} from '@novu/application-generic';
+import { FeatureFlagsKeysEnum } from '@novu/shared';
import { ProviderVolumeDataPointDto } from '../../dtos/get-charts.response.dto';
import { BuildProviderByVolumeChartCommand } from './build-provider-by-volume-chart.command';
@Injectable()
export class BuildProviderByVolumeChart {
constructor(
+ private traceRollupRepository: TraceRollupRepository,
private stepRunRepository: StepRunRepository,
+ private featureFlagsService: FeatureFlagsService,
private logger: PinoLogger
) {
this.logger.setContext(BuildProviderByVolumeChart.name);
@@ -16,13 +25,41 @@ export class BuildProviderByVolumeChart {
async execute(command: BuildProviderByVolumeChartCommand): Promise {
const { environmentId, organizationId, startDate, endDate, workflowIds } = command;
- const providerData = await this.stepRunRepository.getProviderVolumeData(
- environmentId,
- organizationId,
- startDate,
- endDate,
- workflowIds
- );
+ const featureFlagContext = {
+ organization: { _id: organizationId },
+ environment: { _id: environmentId },
+ };
+
+ const [isGlobalEnabled, isDedicatedEnabled] = await Promise.all([
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_LOGS_READ_GLOBAL_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_PROVIDER_VOLUME_READ_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ ]);
+
+ const useNewQuery = isGlobalEnabled || isDedicatedEnabled;
+
+ const providerData = useNewQuery
+ ? await this.traceRollupRepository.getProviderVolumeData(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ workflowIds
+ )
+ : await this.stepRunRepository.getProviderVolumeData(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ workflowIds
+ );
return providerData.map((dataPoint) => ({
providerId: dataPoint.provider_id,
diff --git a/apps/api/src/app/activity/usecases/build-total-interactions-chart/build-total-interactions-chart.usecase.ts b/apps/api/src/app/activity/usecases/build-total-interactions-chart/build-total-interactions-chart.usecase.ts
index ec7df2d271a..695aab9ceed 100644
--- a/apps/api/src/app/activity/usecases/build-total-interactions-chart/build-total-interactions-chart.usecase.ts
+++ b/apps/api/src/app/activity/usecases/build-total-interactions-chart/build-total-interactions-chart.usecase.ts
@@ -1,12 +1,21 @@
import { Injectable } from '@nestjs/common';
-import { InstrumentUsecase, PinoLogger, TraceLogRepository } from '@novu/application-generic';
+import {
+ FeatureFlagsService,
+ InstrumentUsecase,
+ PinoLogger,
+ TraceLogRepository,
+ TraceRollupRepository,
+} from '@novu/application-generic';
+import { FeatureFlagsKeysEnum } from '@novu/shared';
import { TotalInteractionsDataPointDto } from '../../dtos/get-charts.response.dto';
import { BuildTotalInteractionsChartCommand } from './build-total-interactions-chart.command';
@Injectable()
export class BuildTotalInteractionsChart {
constructor(
+ private traceRollupRepository: TraceRollupRepository,
private traceLogRepository: TraceLogRepository,
+ private featureFlagsService: FeatureFlagsService,
private logger: PinoLogger
) {
this.logger.setContext(BuildTotalInteractionsChart.name);
@@ -16,20 +25,49 @@ export class BuildTotalInteractionsChart {
async execute(command: BuildTotalInteractionsChartCommand): Promise {
const { environmentId, organizationId, startDate, endDate, workflowIds } = command;
- // Calculate previous period dates
const periodDuration = endDate.getTime() - startDate.getTime();
const previousEndDate = new Date(startDate.getTime() - 1);
const previousStartDate = new Date(previousEndDate.getTime() - periodDuration);
- const result = await this.traceLogRepository.getTotalInteractionsData(
- environmentId,
- organizationId,
- startDate,
- endDate,
- previousStartDate,
- previousEndDate,
- workflowIds
- );
+ const featureFlagContext = {
+ organization: { _id: organizationId },
+ environment: { _id: environmentId },
+ };
+
+ const [isGlobalEnabled, isDedicatedEnabled] = await Promise.all([
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_LOGS_READ_GLOBAL_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ this.featureFlagsService.getFlag({
+ key: FeatureFlagsKeysEnum.IS_ANALYTIC_V2_TOTAL_INTERACTIONS_READ_ENABLED,
+ defaultValue: false,
+ ...featureFlagContext,
+ }),
+ ]);
+
+ const useNewQuery = isGlobalEnabled || isDedicatedEnabled;
+
+ const result = useNewQuery
+ ? await this.traceRollupRepository.getTotalInteractionsCount(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ previousStartDate,
+ previousEndDate,
+ workflowIds
+ )
+ : await this.traceLogRepository.getTotalInteractionsData(
+ environmentId,
+ organizationId,
+ startDate,
+ endDate,
+ previousStartDate,
+ previousEndDate,
+ workflowIds
+ );
return {
currentPeriod: result.currentPeriod,
diff --git a/apps/api/src/app/activity/usecases/get-charts/get-charts.usecase.ts b/apps/api/src/app/activity/usecases/get-charts/get-charts.usecase.ts
index e789368dd6d..7eaf0f15273 100644
--- a/apps/api/src/app/activity/usecases/get-charts/get-charts.usecase.ts
+++ b/apps/api/src/app/activity/usecases/get-charts/get-charts.usecase.ts
@@ -351,7 +351,10 @@ export class GetCharts {
const buffer = 1 * 60 * 60 * 1000; // 1 hour
const bufferedEarliestAllowedDate = new Date(earliestAllowedDate.getTime() - buffer);
- if (startDate < bufferedEarliestAllowedDate || endDate < bufferedEarliestAllowedDate) {
+ if (
+ process.env.NODE_ENV !== 'local' &&
+ (startDate < bufferedEarliestAllowedDate || endDate < bufferedEarliestAllowedDate)
+ ) {
throw new HttpException(
`Requested date range exceeds your plan's retention period. ` +
`The earliest accessible date for your plan is ${earliestAllowedDate.toISOString().split('T')[0]}. ` +
diff --git a/apps/api/src/app/events/usecases/cancel-delayed/cancel-delayed.usecase.ts b/apps/api/src/app/events/usecases/cancel-delayed/cancel-delayed.usecase.ts
index b1d005e9b7a..bcb7ae60aeb 100644
--- a/apps/api/src/app/events/usecases/cancel-delayed/cancel-delayed.usecase.ts
+++ b/apps/api/src/app/events/usecases/cancel-delayed/cancel-delayed.usecase.ts
@@ -164,6 +164,7 @@ export class CancelDelayed {
workflow_run_identifier: job.identifier || '',
_notificationId: job._notificationId,
workflow_id: job._templateId,
+ provider_id: '',
}));
await this.messageInteractionService.trace(
diff --git a/apps/api/src/app/events/usecases/parse-event-request/parse-event-request.usecase.ts b/apps/api/src/app/events/usecases/parse-event-request/parse-event-request.usecase.ts
index 77974ab21d3..822976a8d98 100644
--- a/apps/api/src/app/events/usecases/parse-event-request/parse-event-request.usecase.ts
+++ b/apps/api/src/app/events/usecases/parse-event-request/parse-event-request.usecase.ts
@@ -275,6 +275,7 @@ export class ParseEventRequest {
entity_id: requestId,
workflow_run_identifier: command.identifier,
workflow_id: command.workflow?._id || '',
+ provider_id: '',
};
await this.traceLogRepository.createRequest([traceData]);
diff --git a/apps/api/src/app/inbox/usecases/delete-many-notifications/delete-many-notifications.usecase.ts b/apps/api/src/app/inbox/usecases/delete-many-notifications/delete-many-notifications.usecase.ts
index 1045d8187ba..39384dbf25d 100644
--- a/apps/api/src/app/inbox/usecases/delete-many-notifications/delete-many-notifications.usecase.ts
+++ b/apps/api/src/app/inbox/usecases/delete-many-notifications/delete-many-notifications.usecase.ts
@@ -1,6 +1,5 @@
import { BadRequestException, Injectable } from '@nestjs/common';
import {
- buildFeedKey,
buildMessageCountKey,
EventType,
InvalidateCacheService,
@@ -210,5 +209,6 @@ function createTraceLog({
workflow_run_identifier: '',
_notificationId: message._notificationId,
workflow_id: message._templateId,
+ provider_id: '',
};
}
diff --git a/apps/api/src/app/inbox/usecases/mark-many-notifications-as/mark-many-notifications-as.usecase.ts b/apps/api/src/app/inbox/usecases/mark-many-notifications-as/mark-many-notifications-as.usecase.ts
index 291a3684a36..52528fb7b4b 100644
--- a/apps/api/src/app/inbox/usecases/mark-many-notifications-as/mark-many-notifications-as.usecase.ts
+++ b/apps/api/src/app/inbox/usecases/mark-many-notifications-as/mark-many-notifications-as.usecase.ts
@@ -257,5 +257,6 @@ function createTraceLog({
workflow_run_identifier: '',
_notificationId: message._notificationId,
workflow_id: message._templateId,
+ provider_id: '',
};
}
diff --git a/apps/api/src/app/inbox/usecases/mark-notifications-as-seen/mark-notifications-as-seen.usecase.ts b/apps/api/src/app/inbox/usecases/mark-notifications-as-seen/mark-notifications-as-seen.usecase.ts
index 897f1083a2f..c471b16d995 100644
--- a/apps/api/src/app/inbox/usecases/mark-notifications-as-seen/mark-notifications-as-seen.usecase.ts
+++ b/apps/api/src/app/inbox/usecases/mark-notifications-as-seen/mark-notifications-as-seen.usecase.ts
@@ -1,7 +1,6 @@
import { BadRequestException, Injectable } from '@nestjs/common';
import {
AnalyticsService,
- buildFeedKey,
buildMessageCountKey,
InvalidateCacheService,
LogRepository,
@@ -270,6 +269,7 @@ export class MarkNotificationsAsSeen {
workflow_run_identifier: '',
_notificationId: message._notificationId,
workflow_id: message._templateId,
+ provider_id: '',
};
}
}
diff --git a/apps/api/src/app/notifications/usecases/get-activity-feed/get-activity-feed.usecase.ts b/apps/api/src/app/notifications/usecases/get-activity-feed/get-activity-feed.usecase.ts
index 76fbde4b6d3..26e0423ea8e 100644
--- a/apps/api/src/app/notifications/usecases/get-activity-feed/get-activity-feed.usecase.ts
+++ b/apps/api/src/app/notifications/usecases/get-activity-feed/get-activity-feed.usecase.ts
@@ -152,7 +152,10 @@ export class GetActivityFeed {
const buffer = 1 * 60 * 60 * 1000; // 1 hour
const bufferedEarliestAllowedDate = new Date(earliestAllowedDate.getTime() - buffer);
- if (afterDate < bufferedEarliestAllowedDate || beforeDate < bufferedEarliestAllowedDate) {
+ if (
+ process.env.NODE_ENV !== 'local' &&
+ (afterDate < bufferedEarliestAllowedDate || beforeDate < bufferedEarliestAllowedDate)
+ ) {
throw new HttpException(
`Requested date range exceeds your plan's retention period. ` +
`The earliest accessible date for your plan is ${earliestAllowedDate.toISOString().split('T')[0]}. ` +
diff --git a/apps/api/src/app/shared/shared.module.ts b/apps/api/src/app/shared/shared.module.ts
index 3dbc4dfbb6b..628e6ecc208 100644
--- a/apps/api/src/app/shared/shared.module.ts
+++ b/apps/api/src/app/shared/shared.module.ts
@@ -10,6 +10,7 @@ import {
clickHouseService,
createNestLoggingModuleOptions,
DalServiceHealthIndicator,
+ DeliveryTrendCountsRepository,
ExecuteBridgeRequest,
featureFlagsService,
GetDecryptedSecretKey,
@@ -20,6 +21,7 @@ import {
StepRunRepository,
storageService,
TraceLogRepository,
+ TraceRollupRepository,
WorkflowRunRepository,
} from '@novu/application-generic';
import {
@@ -120,6 +122,8 @@ const ANALYTICS_PROVIDERS = [
TraceLogRepository,
StepRunRepository,
WorkflowRunRepository,
+ TraceRollupRepository,
+ DeliveryTrendCountsRepository,
// Services
clickHouseService,
diff --git a/apps/api/src/app/widgets/usecases/mark-message-as/mark-message-as.usecase.ts b/apps/api/src/app/widgets/usecases/mark-message-as/mark-message-as.usecase.ts
index bf0b1278634..07ca862d247 100644
--- a/apps/api/src/app/widgets/usecases/mark-message-as/mark-message-as.usecase.ts
+++ b/apps/api/src/app/widgets/usecases/mark-message-as/mark-message-as.usecase.ts
@@ -1,7 +1,6 @@
import { Injectable, NotFoundException } from '@nestjs/common';
import {
AnalyticsService,
- buildFeedKey,
buildMessageCountKey,
buildSubscriberKey,
CachedResponse,
@@ -15,7 +14,6 @@ import {
PinoLogger,
SendWebhookMessage,
StepType,
- Trace,
WebSocketsQueueService,
} from '@novu/application-generic';
import { MessageEntity, MessageRepository, SubscriberEntity, SubscriberRepository } from '@novu/dal';
@@ -140,6 +138,7 @@ export class MarkMessageAs {
workflow_run_identifier: '',
_notificationId: message._notificationId,
workflow_id: message._templateId,
+ provider_id: '',
});
}
}
diff --git a/apps/api/src/app/workflows-v2/e2e/upsert-workflow.e2e.ts b/apps/api/src/app/workflows-v2/e2e/upsert-workflow.e2e.ts
index fe3e6e76eaf..be7ea70e882 100644
--- a/apps/api/src/app/workflows-v2/e2e/upsert-workflow.e2e.ts
+++ b/apps/api/src/app/workflows-v2/e2e/upsert-workflow.e2e.ts
@@ -506,8 +506,8 @@ describe('Upsert Workflow #novu-v2', () => {
expect(updatedEmailStep.controls.values.body).to.contain('
- test
- `);
+ test
+ `);
expect(updatedEmailStep.controls.values.body).to.contain('');
expect(updatedEmailStep.controls.values.body).to.contain('