Ram

backend/__tests__/githubAppWebhookIdempotency.test.js

@@ -0,0 +1,76 @@
+const request = require('supertest');
+const express = require('express');
+
+jest.mock('../middleware/verifyGithub', () => (_req, _res, next) => next());
+
+const mockProcessGithubWebhookJob = jest.fn();
+jest.mock('../services/githubWebhookWorker', () => ({
+  processGithubWebhookJob: (...args) => mockProcessGithubWebhookJob(...args),
+}));
+
+const {
+  waitForWebhookQueueIdle,
+  __resetWebhookQueueForTests,
+} = require('../services/webhookQueue');
+
+const buildPushPayload = () => ({
+  repository: {
+    id: 99,
+    name: 'repo-a',
+    full_name: 'owner-a/repo-a',
+  },
+  sender: {
+    login: 'octocat',
+  },
+  commits: [
+    {
+      id: 'sha-1',
+      message: 'feat: update architecture flow',
+      added: ['src/a.js'],
+      modified: ['src/b.js'],
+      removed: [],
+    },
+  ],
+});
+
+describe('GitHub webhook queue idempotency', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    __resetWebhookQueueForTests();
+    mockProcessGithubWebhookJob.mockResolvedValue({
+      projectId: 'project-1',
+      commitCount: 1,
+    });
+  });
+
+  afterEach(() => {
+    __resetWebhookQueueForTests();
+  });
+
+  test('same delivery ID submitted twice only queues/processes once', async () => {
+    const app = express();
+    app.use(express.json());
+    app.set('io', { emit: jest.fn() });
+    app.use('/api/github-app', require('../routes/githubAppWebhook'));
+
+    const first = await request(app)
+      .post('/api/github-app/webhook')
+      .set('x-github-event', 'push')
+      .set('x-github-delivery', 'delivery-dup-1')
+      .send(buildPushPayload());
+
+    const second = await request(app)
+      .post('/api/github-app/webhook')
+      .set('x-github-event', 'push')
+      .set('x-github-delivery', 'delivery-dup-1')
+      .send(buildPushPayload());
+
+    expect(first.status).toBe(202);
+    expect(first.body.duplicate).toBe(false);
+    expect(second.status).toBe(202);
+    expect(second.body.duplicate).toBe(true);
+
+    await waitForWebhookQueueIdle();
+    expect(mockProcessGithubWebhookJob).toHaveBeenCalledTimes(1);
+  });
+});

backend/__tests__/internalMetricsAndLoadShedding.test.js

@@ -0,0 +1,85 @@
+const request = require('supertest');
+const express = require('express');
+
+const ORIGINAL_MEMORY_USAGE = process.memoryUsage;
+
+describe('internal metrics route', () => {
+  test('returns memory and webhook queue metrics', async () => {
+    const app = express();
+    app.use('/internal', require('../routes/internalMetrics'));
+
+    const res = await request(app).get('/internal/metrics');
+    expect(res.status).toBe(200);
+    expect(res.body).toEqual(
+      expect.objectContaining({
+        memoryMb: expect.objectContaining({
+          rss: expect.any(Number),
+          heapUsed: expect.any(Number),
+          heapTotal: expect.any(Number),
+        }),
+        webhookQueue: expect.objectContaining({
+          depth: expect.any(Number),
+          lagMs: expect.any(Number),
+          processing: expect.any(Boolean),
+          trackedJobs: expect.any(Number),
+        }),
+        timestamp: expect.any(String),
+      })
+    );
+  });
+});
+
+describe('load shedding middleware', () => {
+  const loadSheddingPath = '../middleware/loadShedding';
+
+  afterEach(() => {
+    jest.resetModules();
+    process.memoryUsage = ORIGINAL_MEMORY_USAGE;
+    delete process.env.LOAD_SHED_HEAP_LIMIT_MB;
+    delete process.env.LOAD_SHED_RETRY_AFTER_SECONDS;
+    delete process.env.LOAD_SHED_HEAVY_PATHS;
+  });
+
+  test('returns 503 for heavy route when heap limit is crossed', async () => {
+    process.env.LOAD_SHED_HEAP_LIMIT_MB = '50';
+
+    const { loadSheddingMiddleware } = require(loadSheddingPath);
+    const app = express();
+    app.use('/api', loadSheddingMiddleware);
+    app.get('/api/generate-project', (_req, res) => res.status(200).json({ ok: true }));
+
+    process.memoryUsage = jest.fn(() => ({
+      rss: 800 * 1024 * 1024,
+      heapTotal: 600 * 1024 * 1024,
+      heapUsed: 550 * 1024 * 1024,
+      external: 0,
+      arrayBuffers: 0,
+    }));
+
+    const res = await request(app).get('/api/generate-project');
+    expect(res.status).toBe(503);
+    expect(res.headers['retry-after']).toBeDefined();
+    expect(res.body.reason).toBe('load_shedding');
+  });
+
+  test('does not block allowlisted sessions path even above limit', async () => {
+    process.env.LOAD_SHED_HEAP_LIMIT_MB = '50';
+
+    const { loadSheddingMiddleware } = require(loadSheddingPath);
+    const app = express();
+    app.use('/api', loadSheddingMiddleware);
+    app.get('/api/sessions', (_req, res) => res.status(200).json({ ok: true }));
+
+    process.memoryUsage = jest.fn(() => ({
+      rss: 800 * 1024 * 1024,
+      heapTotal: 600 * 1024 * 1024,
+      heapUsed: 550 * 1024 * 1024,
+      external: 0,
+      arrayBuffers: 0,
+    }));
+
+    const res = await request(app).get('/api/sessions');
+    expect(res.status).toBe(200);
+    expect(res.body.ok).toBe(true);
+  });
+});

backend/index.js

@@ -8,6 +8,7 @@ const helmet = require('helmet');
 const { Server } = require("socket.io");
 const rateLimit = require('express-rate-limit');
 require('dotenv').config();
+const { loadSheddingMiddleware } = require('./middleware/loadShedding');
 
 const app = express();
 
@@ -81,6 +82,7 @@ const chatRoutes = require('./routes/chatRoutes');
 const taskRoutes = require('./routes/taskRoutes');
 const calendarRoutes = require('./routes/calendarRoutes');
 const supportRoutes = require('./routes/supportRoutes');
+const internalMetricsRoutes = require('./routes/internalMetrics');
 
 
 
@@ -120,6 +122,7 @@ const limiter = rateLimit({
 
 // Apply rate limiting to all requests
 app.use('/api/', limiter);
+app.use('/api/', loadSheddingMiddleware);
 
 // Keep raw body only for webhook signature verification routes.
 const webhookJsonParser = express.json({
@@ -158,6 +161,7 @@ app.use('/api/google', require('./routes/googleRoutes'));
 app.use('/api/calendar', calendarRoutes);
 app.use('/api/support', supportRoutes);
 app.use('/api/cache/sample', require('./routes/redisCacheSampleRoutes'));
+app.use('/internal', internalMetricsRoutes);
 
 
 

backend/middleware/loadShedding.js

@@ -0,0 +1,59 @@
+const { getSafeEnvInt } = require('../utils/safeEnv');
+
+const HEAP_LIMIT_MB = getSafeEnvInt('LOAD_SHED_HEAP_LIMIT_MB', 128, 4096, 400);
+const RETRY_AFTER_SECONDS = getSafeEnvInt('LOAD_SHED_RETRY_AFTER_SECONDS', 1, 300, 15);
+
+const ALLOWLIST_PATH_PREFIXES = ['/api/auth', '/api/sessions', '/api/chat'];
+const DEFAULT_HEAVY_PATH_PREFIXES = [
+  '/api/github-app/webhook',
+  '/api/webhooks/github',
+  '/api/generate-project',
+  '/api/design',
+  '/api/inspiration',
+];
+
+const parsePathListEnv = (rawValue) =>
+  String(rawValue || '')
+    .split(',')
+    .map((value) => value.trim())
+    .filter(Boolean);
+
+const HEAVY_PATH_PREFIXES = (() => {
+  const custom = parsePathListEnv(process.env.LOAD_SHED_HEAVY_PATHS);
+  return custom.length > 0 ? custom : DEFAULT_HEAVY_PATH_PREFIXES;
+})();
+
+const isPathMatchedByPrefixes = (path, prefixes) => prefixes.some((prefix) => path.startsWith(prefix));
+
+const loadSheddingMiddleware = (req, res, next) => {
+  const requestPath = String((req.originalUrl || req.path || '').split('?')[0]);
+
+  if (isPathMatchedByPrefixes(requestPath, ALLOWLIST_PATH_PREFIXES)) {
+    return next();
+  }
+
+  if (!isPathMatchedByPrefixes(requestPath, HEAVY_PATH_PREFIXES)) {
+    return next();
+  }
+
+  const heapUsedMb = process.memoryUsage().heapUsed / (1024 * 1024);
+  if (heapUsedMb < HEAP_LIMIT_MB) {
+    return next();
+  }
+
+  res.set('Retry-After', String(RETRY_AFTER_SECONDS));
+  return res.status(503).json({
+    message: 'Service under memory pressure, please retry shortly.',
+    reason: 'load_shedding',
+    heapUsedMb: Number(heapUsedMb.toFixed(2)),
+    heapLimitMb: HEAP_LIMIT_MB,
+  });
+};
+
+module.exports = {
+  loadSheddingMiddleware,
+  ALLOWLIST_PATH_PREFIXES,
+  HEAVY_PATH_PREFIXES,
+  HEAP_LIMIT_MB,
+  RETRY_AFTER_SECONDS,
+};

backend/models/Project.js

@@ -26,6 +26,17 @@ const projectSchema = new mongoose.Schema(
 
     // Tracking
     isTrackingActive: { type: Boolean, default: false },
+
+    // Webhook aggregation snapshot (used to reduce per-commit fanout/write load)
+    lastWebhookEventAt: { type: Date, default: null },
+    lastWebhookCommitCount: { type: Number, default: 0 },
+    lastWebhookCommitShas: { type: [String], default: [] },
+    lastWebhookChangedFiles: { type: [String], default: [] },
+    lastWebhookPusher: { type: String, default: null },
+    lastWebhookDeliveryId: { type: String, default: null },
+    lastWebhookAiSummary: { type: String, default: null },
+    lastWebhookAiTaskMentions: { type: Number, default: 0 },
+    lastWebhookAiAnalyzedCommits: { type: Number, default: 0 },
   },
   {
     timestamps: true,