Merge branch 'next' into nv-6862-js-subscriptions-module

Author: LetItRock

.cspell.json

@@ -770,7 +770,9 @@
     "BLKTM",
     "Novuand",
     "kobalte",
-    "jsonrepair"
+    "jsonrepair",
+    "xoxb",
+    "fanout"
   ],
   "flagWords": [],
   "patterns": [

.github/workflows/deploy.yml

@@ -28,11 +28,16 @@ on:
         default: staging
         options:
           - staging
-          - staging-apse1
+          - staging-sg
           - production-us
           - production-eu
-          - production-apse1
+          - production-sg
+          - production-au
+          - production-uk
+          - production-jp
+          - production-kr
           - production-us-and-eu
+          - production-sg-au-uk-jp-kr
 
       deploy_api:
         description: "Deploy API"
@@ -89,7 +94,7 @@ jobs:
           if [ "${{ github.event.inputs.environment }}" == "staging" ]; then
             envs+=("\"staging-eu\"")
           fi
-          if [ "${{ github.event.inputs.environment }}" == "staging-apse1" ]; then
+          if [ "${{ github.event.inputs.environment }}" == "staging-uk" ]; then
             envs+=("\"staging-apse1\"")
           fi
           if [ "${{ github.event.inputs.environment }}" == "production-us" ]; then
@@ -98,13 +103,32 @@ jobs:
           if [ "${{ github.event.inputs.environment }}" == "production-eu" ]; then
             envs+=("\"prod-eu\"")
           fi
-          if [ "${{ github.event.inputs.environment }}" == "production-apse1" ]; then
+          if [ "${{ github.event.inputs.environment }}" == "production-sg" ]; then
             envs+=("\"prod-apse1\"")
           fi
+          if [ "${{ github.event.inputs.environment }}" == "production-au" ]; then
+            envs+=("\"prod-apse2\"")
+          fi
+          if [ "${{ github.event.inputs.environment }}" == "production-uk" ]; then
+            envs+=("\"prod-ew2\"")
+          fi
+          if [ "${{ github.event.inputs.environment }}" == "production-jp" ]; then
+            envs+=("\"prod-apne1\"")
+          fi
+          if [ "${{ github.event.inputs.environment }}" == "production-kr" ]; then
+            envs+=("\"prod-apne2\"")
+          fi
           if [ "${{ github.event.inputs.environment }}" == "production-us-and-eu" ]; then
             envs+=("\"prod-us\"")
             envs+=("\"prod-eu\"")
           fi
+          if [ "${{ github.event.inputs.environment }}" == "production-sg-au-uk-jp-kr" ]; then
+            envs+=("\"prod-apse1\"")
+            envs+=("\"prod-apse2\"")
+            envs+=("\"prod-ew2\"")
+            envs+=("\"prod-apne1\"")
+            envs+=("\"prod-apne2\"")
+          fi
 
           # Collect selected services
           if [ "${{ github.event.inputs.deploy_api }}" == "true" ]; then

.github/workflows/prepare-self-hosted-release.yml

@@ -1,13 +1,60 @@
 name: Prepare Self-hosted Release
+run-name: >
+  Building self-hosted${{ github.event.inputs.nightly == 'true' && ' (nightly)' || '' }}:
+  ${{
+    github.event.inputs.build_api == 'true' && 'api, ' || ''
+  }}${{
+    github.event.inputs.build_worker == 'true' && 'worker, ' || ''
+  }}${{
+    github.event.inputs.build_dashboard == 'true' && 'dashboard, ' || ''
+  }}${{
+    github.event.inputs.build_webhook == 'true' && 'webhook, ' || ''
+  }}${{
+    github.event.inputs.build_ws == 'true' && 'ws' || ''
+  }}
 
 env:
   NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
 
 on:
   push:
     tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+'
+      - "v[0-9]+.[0-9]+.[0-9]+"
   workflow_dispatch:
+    inputs:
+      nightly:
+        description: "Build as nightly release"
+        required: false
+        default: "false"
+        type: choice
+        options:
+          - "true"
+          - "false"
+      build_api:
+        description: "Build API service"
+        required: false
+        type: boolean
+        default: true
+      build_worker:
+        description: "Build Worker service"
+        required: false
+        type: boolean
+        default: true
+      build_dashboard:
+        description: "Build Dashboard service"
+        required: false
+        type: boolean
+        default: true
+      build_webhook:
+        description: "Build Webhook service"
+        required: false
+        type: boolean
+        default: true
+      build_ws:
+        description: "Build WebSocket service"
+        required: false
+        type: boolean
+        default: true
 
 permissions:
   contents: write
@@ -16,13 +63,63 @@ permissions:
   id-token: write
 
 jobs:
+  setup_matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Validate Selected Services
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          if [ "${{ github.event.inputs.build_api }}" != "true" ] && \
+             [ "${{ github.event.inputs.build_worker }}" != "true" ] && \
+             [ "${{ github.event.inputs.build_dashboard }}" != "true" ] && \
+             [ "${{ github.event.inputs.build_webhook }}" != "true" ] && \
+             [ "${{ github.event.inputs.build_ws }}" != "true" ]; then
+            echo "Error: At least one service must be selected for building."
+            exit 1
+          fi
+
+      - name: Generate Build Matrix
+        id: set-matrix
+        run: |
+          # If triggered by tag push, build all services
+          if [ "${{ github.event_name }}" == "push" ]; then
+            matrix='["novu/api","novu/worker","novu/dashboard","novu/webhook","novu/ws"]'
+          else
+            # If triggered by workflow_dispatch, build only selected services
+            services=()
+            
+            if [ "${{ github.event.inputs.build_api }}" == "true" ]; then
+              services+=("\"novu/api\"")
+            fi
+            if [ "${{ github.event.inputs.build_worker }}" == "true" ]; then
+              services+=("\"novu/worker\"")
+            fi
+            if [ "${{ github.event.inputs.build_dashboard }}" == "true" ]; then
+              services+=("\"novu/dashboard\"")
+            fi
+            if [ "${{ github.event.inputs.build_webhook }}" == "true" ]; then
+              services+=("\"novu/webhook\"")
+            fi
+            if [ "${{ github.event.inputs.build_ws }}" == "true" ]; then
+              services+=("\"novu/ws\"")
+            fi
+            
+            matrix="[$(IFS=','; echo "${services[*]}")]"
+          fi
+
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+          echo "Building services: $matrix"
+
   build_docker:
+    needs: setup_matrix
     runs-on: ubuntu-latest
     timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:
-        name: ['novu/api', 'novu/worker', 'novu/dashboard', 'novu/webhook', 'novu/ws']
+        name: ${{ fromJson(needs.setup_matrix.outputs.matrix) }}
     steps:
       - name: Git Checkout
         uses: actions/checkout@v5
@@ -31,9 +128,21 @@ jobs:
         shell: bash
         run: |
           service=${{ matrix.name }}
-          LATEST_VERSION=$(jq -r '.version'  apps/api/package.json)
           SERVICE_NAME=$(basename "${service//-/-}")
           SERVICE_COMMON_NAME=$(echo "$SERVICE_NAME" | sed 's/-ee$//')
+
+          # Determine version based on nightly flag
+          IS_NIGHTLY=${{ github.event.inputs.nightly == 'true' }}
+          if [ "$IS_NIGHTLY" == "true" ]; then
+            DATE=$(date +'%Y%m%d')
+            SHORT_SHA=$(git rev-parse --short HEAD)
+            LATEST_VERSION="nightly-${DATE}-${SHORT_SHA}"
+            echo "IS_NIGHTLY=true" >> $GITHUB_ENV
+          else
+            LATEST_VERSION=$(jq -r '.version'  apps/api/package.json)
+            echo "IS_NIGHTLY=false" >> $GITHUB_ENV
+          fi
+
           echo "LATEST_VERSION=$LATEST_VERSION" >> $GITHUB_ENV
           echo "SERVICE_NAME=$SERVICE_NAME" >> $GITHUB_ENV
           echo "SERVICE_COMMON_NAME=$SERVICE_COMMON_NAME" >> $GITHUB_ENV
@@ -47,7 +156,7 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: 20.19.0
-          cache: 'pnpm'
+          cache: "pnpm"
 
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
@@ -71,7 +180,7 @@ jobs:
       - name: Set Up Docker Buildx
         uses: docker/setup-buildx-action@v3
         with:
-          driver-opts: 'image=moby/buildkit:v0.13.1'
+          driver-opts: "image=moby/buildkit:v0.13.1"
 
       - uses: ./.github/actions/free-space
         name: Extend space in Action Container
@@ -129,6 +238,12 @@ jobs:
         shell: bash
         run: |
           docker tag novu-$SERVICE_COMMON_NAME ghcr.io/$REGISTRY_OWNER/${{ matrix.name }}:${{ env.LATEST_VERSION }}
-          docker tag novu-$SERVICE_COMMON_NAME ghcr.io/$REGISTRY_OWNER/${{ matrix.name }}:latest
           docker push ghcr.io/$REGISTRY_OWNER/${{ matrix.name }}:${{ env.LATEST_VERSION }}
-          docker push ghcr.io/$REGISTRY_OWNER/${{ matrix.name }}:latest
+
+          if [ "${{ env.IS_NIGHTLY }}" == "true" ]; then
+            docker tag novu-$SERVICE_COMMON_NAME ghcr.io/$REGISTRY_OWNER/${{ matrix.name }}:nightly
+            docker push ghcr.io/$REGISTRY_OWNER/${{ matrix.name }}:nightly
+          else
+            docker tag novu-$SERVICE_COMMON_NAME ghcr.io/$REGISTRY_OWNER/${{ matrix.name }}:latest
+            docker push ghcr.io/$REGISTRY_OWNER/${{ matrix.name }}:latest
+          fi

.source

@@ -0,0 +1,10 @@
+-- Add workflow_id column to step_runs table
+-- This column stores the workflow template ID for each step execution
+ALTER TABLE step_runs
+ADD COLUMN IF NOT EXISTS workflow_id String DEFAULT '';
+
+
+-- Add workflow_id column to traces table
+-- This column stores the workflow template ID for each trace
+ALTER TABLE traces
+ADD COLUMN IF NOT EXISTS workflow_id String DEFAULT '';

apps/api/migrations/clickhouse-migrations/2_add_workflow_id_to_schema.sql

@@ -1,6 +1,6 @@
 {
   "name": "@novu/api-service",
-  "version": "3.9.0",
+  "version": "3.11.0",
   "description": "description",
   "author": "",
   "private": "true",
@@ -32,7 +32,7 @@
     "test:e2e:novu-v0": "cross-env TS_NODE_COMPILER_OPTIONS='{\"strictNullChecks\": false}' NODE_ENV=test mocha --timeout 15000 --retries 3 --grep '#novu-v0' --require ts-node/register --exit --file e2e/setup.ts src/**/*.e2e{,-ee}.ts",
     "test:e2e:novu-v2": "cross-env NODE_ENV=test CI_EE_TEST=true CLERK_ENABLED=true NODE_OPTIONS=--max_old_space_size=8192 mocha --timeout 30000  --retries 3 --grep '#novu-v2' --require ./swc-register.js --exit --file e2e/setup.ts 'src/**/*.e2e{,-ee}.ts' 'e2e/enterprise/**/*.e2e.ts'",
     "migration": "cross-env NODE_ENV=local MIGRATION=true ts-node --transpileOnly",
-    "clickhouse:migrate": "clickhouse-migrations migrate --host=http://localhost:8123 --user=default --password= --db=novu-local --migrations-home=./migrations/clickhouse-migrations",
+    "clickhouse:migrate:local": "clickhouse-migrations migrate --host=http://localhost:8123 --user=default --password= --db=novu-local --migrations-home=./migrations/clickhouse-migrations",
     "clickhouse:migrate:prod": "clickhouse-migrations migrate --migrations-home=./migrations/clickhouse-migrations",
     "link:submodules": "pnpm link ../../enterprise/packages/auth && pnpm link ../../enterprise/packages/translation && pnpm link ../../enterprise/packages/billing",
     "admin:remove-user-account": "cross-env NODE_ENV=local MIGRATION=true ts-node --transpileOnly ./admin/remove-user-account.ts",

apps/api/package.json

@@ -76,7 +76,10 @@ export class GetWorkflowRunResponseBaseDto {
   @IsBoolean()
   critical: boolean;
 
-  @ApiPropertyOptional({ description: 'Contexts (keys) in which the workflow run was executed', type: [String] })
+  @ApiPropertyOptional({
+    description: 'Context (single or multi) in which the workflow run was executed',
+    type: [String],
+  })
   @IsOptional()
   @IsArray()
   @IsString({ each: true })

apps/api/src/app/activity/dtos/shared.dto.ts

@@ -66,7 +66,17 @@ export class GetWorkflowRunsRequestDto {
   severity?: SeverityLevelEnum[];
 
   @IsOptional()
-  @Transform(({ value }) => (Array.isArray(value) ? value : [value]))
+  @Transform(({ value }) => {
+    // No parameter = no filter
+    if (value === undefined) return undefined;
+
+    // Empty string = filter for records with no (default) context
+    if (value === '') return [];
+
+    // Normalize to array and remove empty strings
+    const array = Array.isArray(value) ? value : [value];
+    return array.filter((v) => v !== '');
+  })
   @IsArray()
   @IsString({ each: true })
   contextKeys?: string[];

apps/api/src/app/activity/dtos/workflow-runs-request.dto.ts

@@ -1,5 +1,5 @@
 import { EnvironmentCommand } from '@novu/application-generic';
-import { IsDate, IsDefined } from 'class-validator';
+import { IsArray, IsDate, IsDefined, IsOptional, IsString } from 'class-validator';
 
 export class BuildActiveSubscribersChartCommand extends EnvironmentCommand {
   @IsDate()
@@ -9,4 +9,9 @@ export class BuildActiveSubscribersChartCommand extends EnvironmentCommand {
   @IsDate()
   @IsDefined()
   endDate: Date;
+
+  @IsOptional()
+  @IsArray()
+  @IsString({ each: true })
+  workflowIds?: string[];
 }

apps/api/src/app/activity/usecases/build-active-subscribers-chart/build-active-subscribers-chart.command.ts

@@ -14,7 +14,7 @@ export class BuildActiveSubscribersChart {
 
   @InstrumentUsecase()
   async execute(command: BuildActiveSubscribersChartCommand): Promise<ActiveSubscribersDataPointDto> {
-    const { environmentId, organizationId, startDate, endDate } = command;
+    const { environmentId, organizationId, startDate, endDate, workflowIds } = command;
 
     // Calculate previous period dates
     const periodDuration = endDate.getTime() - startDate.getTime();
@@ -27,7 +27,8 @@ export class BuildActiveSubscribersChart {
       startDate,
       endDate,
       previousStartDate,
-      previousEndDate
+      previousEndDate,
+      workflowIds
     );
 
     return {