diff --git a/attachment-summarizer/.env.example b/attachment-summarizer/.env.example new file mode 100644 index 00000000..e9287e51 --- /dev/null +++ b/attachment-summarizer/.env.example @@ -0,0 +1,6 @@ +AWS_REGION=us-east-1 +SQS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/123456789012/attachments +GCS_PROJECT_ID=your-project-id +OLLAMA_URL=http://localhost:11434 +LLM_MODEL=llama3.2 +LOG_LEVEL=info diff --git a/attachment-summarizer/Dockerfile b/attachment-summarizer/Dockerfile new file mode 100644 index 00000000..9b0a4334 --- /dev/null +++ b/attachment-summarizer/Dockerfile @@ -0,0 +1,7 @@ +FROM node:20-slim +WORKDIR /app +COPY package*.json ./ +RUN npm ci --only=production +COPY dist ./dist +EXPOSE 3000 +CMD ["node", "dist/index.js"] diff --git a/attachment-summarizer/README.md b/attachment-summarizer/README.md new file mode 100644 index 00000000..362a2ee9 --- /dev/null +++ b/attachment-summarizer/README.md @@ -0,0 +1,24 @@ +# Attachment Summarizer Service + +$960 warpSpeed bounty - Node.js/TypeScript service that summarizes email attachments. + +## Setup (3 steps) + +1. `npm install` +2. `cp .env.example .env` - configure AWS, GCS, Ollama +3. `npm start` + +## Docker + +```bash +docker build -t attachment-summarizer . +docker run --env-file .env attachment-summarizer +``` + +## Supported Formats + +- PDF, Word, Excel, HTML, Plain Text + +## Architecture + +SQS -> Consumer -> GCS -> Extractor -> Ollama LLM -> Summary diff --git a/attachment-summarizer/package.json b/attachment-summarizer/package.json new file mode 100644 index 00000000..463b6dff --- /dev/null +++ b/attachment-summarizer/package.json @@ -0,0 +1,32 @@ +{ + "name": "attachment-summarizer", + "version": "1.0.0", + "description": "Email attachment summarizer - SQS consumer, GCS downloader, content extraction, LLM summarization", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "start": "node dist/index.js", + "dev": "ts-node src/index.ts", + "test": "jest --passWithNoTests", + "lint": "eslint src/**/*.ts" + }, + "dependencies": { + "@aws-sdk/client-sqs": "^3.0.0", + "@google-cloud/storage": "^7.0.0", + "pdf-parse": "^1.1.1", + "mammoth": "^1.6.0", + "xlsx": "^0.18.0", + "cheerio": "^1.0.0", + "node-fetch": "^3.3.0", + "winston": "^3.11.0", + "dotenv": "^16.3.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "@types/jest": "^29.5.0", + "jest": "^29.7.0", + "ts-jest": "^29.1.0", + "ts-node": "^10.9.0", + "typescript": "^5.2.0" + } +} \ No newline at end of file diff --git a/attachment-summarizer/src/index.ts b/attachment-summarizer/src/index.ts new file mode 100644 index 00000000..8a3e62af --- /dev/null +++ b/attachment-summarizer/src/index.ts @@ -0,0 +1,28 @@ +import dotenv from 'dotenv'; +import { logger } from './utils/logger'; +import { createSqsConsumer } from './services/sqs-consumer'; +import { createAttachmentProcessor } from './services/attachment-processor'; + +dotenv.config(); + +async function main(): Promise { + logger.info('Attachment Summarizer Service starting...'); + + const processor = createAttachmentProcessor(); + const consumer = createSqsConsumer(processor); + + await consumer.start(); + + logger.info('Service started. Listening for messages on SQS...'); + + process.on('SIGTERM', async () => { + logger.info('SIGTERM received, shutting down...'); + await consumer.stop(); + process.exit(0); + }); +} + +main().catch((error) => { + logger.error('Fatal error:', error); + process.exit(1); +}); diff --git a/attachment-summarizer/src/services/attachment-processor.ts b/attachment-summarizer/src/services/attachment-processor.ts new file mode 100644 index 00000000..0bb293de --- /dev/null +++ b/attachment-summarizer/src/services/attachment-processor.ts @@ -0,0 +1,40 @@ +import { logger } from '../utils/logger'; +import { AttachmentEvent, SummaryResult } from '../types'; +import { downloadFromGCS } from './gcs-downloader'; +import { extractContent } from './content-extractor'; +import { summarize } from './llm-summarizer'; + +export interface AttachmentProcessor { + process: (event: AttachmentEvent) => Promise; +} + +export function createAttachmentProcessor(): AttachmentProcessor { + return { + async process(event: AttachmentEvent): Promise { + logger.info(`Processing ${event.fileName} (${event.contentType})`); + + try { + const buffer = await downloadFromGCS(event.bucketName, event.objectPath); + const extraction = await extractContent(buffer, event.contentType, event.fileName); + + if (!extraction.text.trim()) { + return { + attachmentId: event.attachmentId, + summary: 'No extractable text content.', + metadata: { ...extraction.metadata, empty: true }, + }; + } + + const summary = await summarize(extraction.text, event.attachmentId); + return summary; + } catch (error) { + logger.error(`Failed to process ${event.attachmentId}:`, error); + return { + attachmentId: event.attachmentId, + summary: `Processing failed: ${String(error)}`, + metadata: { error: String(error) }, + }; + } + }, + }; +} \ No newline at end of file diff --git a/attachment-summarizer/src/services/content-extractor.ts b/attachment-summarizer/src/services/content-extractor.ts new file mode 100644 index 00000000..441a793e --- /dev/null +++ b/attachment-summarizer/src/services/content-extractor.ts @@ -0,0 +1,68 @@ +import pdfParse from 'pdf-parse'; +import mammoth from 'mammoth'; +import * as XLSX from 'xlsx'; +import * as cheerio from 'cheerio'; +import { ExtractionResult } from '../types'; +import { logger } from '../utils/logger'; + +export async function extractContent( + buffer: Buffer, + contentType: string, + fileName: string +): Promise { + logger.info(`Extracting content from ${fileName} (${contentType})`); + + try { + switch (contentType) { + case 'application/pdf': + return await extractPdf(buffer); + case 'application/msword': + case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': + return await extractWord(buffer); + case 'application/vnd.ms-excel': + case 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': + return await extractExcel(buffer); + case 'text/html': + return await extractHtml(buffer); + case 'text/plain': + return await extractPlainText(buffer); + default: + logger.warn(`Unsupported content type: ${contentType}`); + return { text: '', metadata: { error: 'Unsupported file type' } }; + } + } catch (error) { + logger.error(`Extraction failed for ${fileName}:`, error); + return { text: '', metadata: { error: String(error) } }; + } +} + +async function extractPdf(buffer: Buffer): Promise { + const data = await pdfParse(buffer); + return { text: data.text, metadata: { pages: data.numpages } }; +} + +async function extractWord(buffer: Buffer): Promise { + const result = await mammoth.extractRawText({ buffer }); + return { text: result.value, metadata: { messages: result.messages } }; +} + +async function extractExcel(buffer: Buffer): Promise { + const workbook = XLSX.read(buffer, { type: 'buffer' }); + const allText = workbook.SheetNames.map((name) => { + const sheet = workbook.Sheets[name]; + return XLSX.utils.sheet_to_csv(sheet); + }).join('\n\n'); + return { text: allText, metadata: { sheets: workbook.SheetNames } }; +} + +async function extractHtml(buffer: Buffer): Promise { + const html = buffer.toString('utf-8'); + const $ = cheerio.load(html); + const text = $('body').text().replace(/\s+/g, ' ').trim(); + return { text, metadata: { title: $('title').text() } }; +} + +async function extractPlainText(buffer: Buffer): Promise { + const text = buffer.toString('utf-8'); + return { text, metadata: { length: text.length } }; +} diff --git a/attachment-summarizer/src/services/gcs-downloader.ts b/attachment-summarizer/src/services/gcs-downloader.ts new file mode 100644 index 00000000..cb4e764b --- /dev/null +++ b/attachment-summarizer/src/services/gcs-downloader.ts @@ -0,0 +1,20 @@ +import { Storage } from '@google-cloud/storage'; +import { logger } from '../utils/logger'; +import { ExtractionResult } from '../types'; + +const storage = new Storage(); + +export async function downloadFromGCS( + bucketName: string, + objectPath: string +): Promise { + logger.info(`Downloading from GCS: ${bucketName}/${objectPath}`); + + const bucket = storage.bucket(bucketName); + const file = bucket.file(objectPath); + + const [buffer] = await file.download(); + + logger.info(`Downloaded ${buffer.length} bytes`); + return buffer; +} diff --git a/attachment-summarizer/src/types/index.ts b/attachment-summarizer/src/types/index.ts new file mode 100644 index 00000000..9a91696e --- /dev/null +++ b/attachment-summarizer/src/types/index.ts @@ -0,0 +1,21 @@ +export interface AttachmentEvent { + messageId: string; + emailId: string; + attachmentId: string; + bucketName: string; + objectPath: string; + fileName: string; + contentType: string; + timestamp: string; +} + +export interface ExtractionResult { + text: string; + metadata: Record; +} + +export interface SummaryResult { + attachmentId: string; + summary: string; + metadata: Record; +} diff --git a/attachment-summarizer/src/utils/logger.ts b/attachment-summarizer/src/utils/logger.ts new file mode 100644 index 00000000..30d59f81 --- /dev/null +++ b/attachment-summarizer/src/utils/logger.ts @@ -0,0 +1,14 @@ +import winston from 'winston'; + +export const logger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.json() + ), + transports: [ + new winston.transports.Console({ + format: winston.format.simple(), + }), + ], +});