diff --git a/ACCESS_PLAN.md b/ACCESS_PLAN.md new file mode 100644 index 00000000..d125c095 --- /dev/null +++ b/ACCESS_PLAN.md @@ -0,0 +1,556 @@ +# DeepTutor Application Access Plan + +This document provides a comprehensive step-by-step guide to access and run the DeepTutor application. + +## Table of Contents +1. [Prerequisites](#prerequisites) +2. [Installation Methods](#installation-methods) +3. [Configuration](#configuration) +4. [Launching the Application](#launching-the-application) +5. [Accessing the Application](#accessing-the-application) +6. [Troubleshooting](#troubleshooting) + +--- + +## Prerequisites + +### Required Software + +#### Option A: Docker (Recommended for Easy Setup) +- **Docker** (version 20.10+) +- **Docker Compose** (version 2.0+) +- **No Python/Node.js required** when using Docker + +**Install Docker:** +- macOS: Download from [Docker Desktop](https://docs.docker.com/get-docker/) +- Linux: Follow [Docker installation guide](https://docs.docker.com/engine/install/) +- Windows: Download from [Docker Desktop](https://docs.docker.com/get-docker/) + +#### Option B: Manual Installation +- **Python** 3.10 or higher +- **Node.js** 18 or higher +- **npm** (comes with Node.js) + +**Verify installations:** +```bash +python --version # Should show 3.10.x or higher +node --version # Should show v18.x.x or higher +npm --version # Should show version number +``` + +### API Keys Required + +You'll need API keys for: +1. **LLM (Large Language Model)** - Choose one: + - OpenAI (GPT-4, GPT-3.5) + - Azure OpenAI + - Anthropic Claude + - Other compatible providers + +2. **Embedding Model** - For knowledge base: + - OpenAI embeddings + - Azure OpenAI embeddings + - Other compatible providers + +3. **Search Provider** (Optional but recommended): + - Perplexity (default) + - Tavily + - Serper + - Jina + - Exa + - Baidu + +--- + +## Installation Methods + +### Method 1: Docker Deployment (Easiest) + +#### Step 1: Navigate to Project Directory +```bash +cd /Users/anup.singh/Documents/Deeptutor/DeepTutor +``` + +#### Step 2: Configure Environment Variables +```bash +# Copy the example environment file +cp .env.example .env + +# Edit .env file with your API keys +# Use your preferred text editor (nano, vim, VS Code, etc.) +nano .env +# or +code .env +``` + +#### Step 3: Start with Docker Compose +```bash +# Build and start (first time takes ~11 minutes) +docker compose up + +# Or use pre-built image (faster) +docker run -d --name deeptutor \ + -p 8001:8001 -p 3782:3782 \ + --env-file .env \ + -v $(pwd)/data:/app/data \ + -v $(pwd)/config:/app/config:ro \ + ghcr.io/hkuds/deeptutor:latest +``` + +#### Step 4: Access the Application +- **Frontend**: http://localhost:3782 +- **Backend API Docs**: http://localhost:8001/docs + +**Common Docker Commands:** +```bash +docker compose up -d # Start in background +docker compose down # Stop services +docker compose logs -f # View logs +docker compose up --build # Rebuild after changes +``` + +--- + +### Method 2: Manual Installation + +#### Step 1: Navigate to Project Directory +```bash +cd /Users/anup.singh/Documents/Deeptutor/DeepTutor +``` + +#### Step 2: Set Up Python Environment + +**Option A: Using Conda (Recommended)** +```bash +conda create -n deeptutor python=3.10 +conda activate deeptutor +``` + +**Option B: Using venv** +```bash +python -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate +``` + +#### Step 3: Install Dependencies + +**One-Click Installation (Recommended):** +```bash +python scripts/install_all.py +``` + +**Or Manual Installation:** +```bash +# Install backend dependencies +pip install -r requirements.txt + +# Install frontend dependencies +cd web +npm install +cd .. +``` + +#### Step 4: Configure Environment Variables +```bash +# Copy the example environment file +cp .env.example .env + +# Edit .env file with your API keys +nano .env +# or +code .env +``` + +#### Step 5: Launch the Application +```bash +# Start both frontend and backend +python scripts/start_web.py + +# Or start separately: +# Backend only: +python src/api/run_server.py +# Frontend only (in another terminal): +cd web && npm run dev -- -p 3782 +``` + +--- + +## Configuration + +### Environment Variables (.env file) + +Create a `.env` file in the project root with the following variables: + +#### Required Variables + +```bash +# LLM Configuration +LLM_MODEL=gpt-4o # Model name (e.g., gpt-4o, gpt-3.5-turbo) +LLM_API_KEY=your-api-key-here # Your LLM API key +LLM_HOST=https://api.openai.com/v1 # API endpoint URL + +# Embedding Configuration +EMBEDDING_MODEL=text-embedding-3-large # Embedding model name +EMBEDDING_API_KEY=your-api-key-here # Embedding API key +EMBEDDING_HOST=https://api.openai.com/v1 # Embedding API endpoint +``` + +#### Optional Variables + +```bash +# Port Configuration (defaults shown) +BACKEND_PORT=8001 # Backend API port +FRONTEND_PORT=3782 # Frontend web port + +# Frontend API URL (for remote access) +NEXT_PUBLIC_API_BASE=http://localhost:8001 +# For remote/LAN access, use your server IP: +# NEXT_PUBLIC_API_BASE=http://192.168.1.100:8001 + +# Search Provider (optional) +SEARCH_PROVIDER=perplexity # Options: perplexity, tavily, serper, jina, exa, baidu +SEARCH_API_KEY=your-search-api-key + +# Text-to-Speech (optional) +TTS_API_KEY=your-tts-api-key +``` + +### Azure OpenAI Configuration + +If using Azure OpenAI, add: +```bash +LLM_API_VERSION=2024-02-15-preview +EMBEDDING_API_VERSION=2024-02-15-preview +``` + +### Remote Access Configuration + +If accessing from another device on your network: + +1. Find your computer's IP address: + ```bash + # macOS/Linux + ifconfig | grep "inet " | grep -v 127.0.0.1 + + # Windows + ipconfig + ``` + +2. Update `.env` file: + ```bash + NEXT_PUBLIC_API_BASE=http://YOUR_IP_ADDRESS:8001 + ``` + +3. Ensure firewall allows connections on ports 8001 and 3782 + +--- + +## Launching the Application + +### Quick Start (Recommended) + +```bash +# Navigate to project directory +cd /Users/anup.singh/Documents/Deeptutor/DeepTutor + +# Ensure .env file is configured +# Then start: +python scripts/start_web.py +``` + +### What Happens When You Start + +1. **Backend starts** (FastAPI server on port 8001) +2. **Frontend starts** (Next.js server on port 3782) +3. **Services are ready** when you see: + ``` + ✅ Services are running! + - Backend: http://localhost:8001/docs + - Frontend: http://localhost:3782 + ``` + +### Starting Services Separately + +**Backend Only:** +```bash +python src/api/run_server.py +# Or: +uvicorn src.api.main:app --host 0.0.0.0 --port 8001 --reload +``` + +**Frontend Only:** +```bash +cd web +npm run dev -- -p 3782 +``` + +**Note:** If starting frontend separately, create `web/.env.local`: +```bash +NEXT_PUBLIC_API_BASE=http://localhost:8001 +``` + +--- + +## Accessing the Application + +### Web Interface + +1. **Open your browser** and navigate to: + ``` + http://localhost:3782 + ``` + +2. **Main Features Available:** + - 📚 **Knowledge Base Management** - Upload and manage documents + - 🧠 **Smart Solver** - AI-powered problem solving + - 📝 **Question Generator** - Generate practice questions + - 🔬 **Deep Research** - Comprehensive research reports + - 💡 **Idea Generation** - Research idea generation + - 🎓 **Guided Learning** - Interactive learning paths + - 📓 **Notebook** - Personal learning records + +### API Documentation + +Access interactive API documentation at: +``` +http://localhost:8001/docs +``` + +This provides: +- Complete API endpoint documentation +- Interactive API testing interface +- Request/response schemas + +### First Steps After Launch + +1. **Create a Knowledge Base:** + - Go to http://localhost:3782/knowledge + - Click "New Knowledge Base" + - Upload PDF/TXT/MD documents + - Wait for processing to complete + +2. **Try Problem Solving:** + - Go to http://localhost:3782/solver + - Select a knowledge base + - Enter a question + - Click "Solve" + +3. **Explore Other Modules:** + - Question Generator: http://localhost:3782/question + - Deep Research: http://localhost:3782/research + - Guided Learning: http://localhost:3782/guide + +--- + +## Troubleshooting + +### Backend Fails to Start + +**Symptoms:** +- Port already in use error +- Import errors +- Configuration errors + +**Solutions:** + +1. **Check if port is in use:** + ```bash + # macOS/Linux + lsof -i :8001 + kill -9 + + # Windows + netstat -ano | findstr :8001 + taskkill /PID /F + ``` + +2. **Verify Python version:** + ```bash + python --version # Should be 3.10+ + ``` + +3. **Reinstall dependencies:** + ```bash + pip install -r requirements.txt + ``` + +4. **Check .env file:** + - Ensure all required variables are set + - Verify API keys are correct + +### Frontend Cannot Connect to Backend + +**Symptoms:** +- "Failed to fetch" errors +- Connection refused errors + +**Solutions:** + +1. **Verify backend is running:** + ```bash + curl http://localhost:8001/docs + # Should return HTML + ``` + +2. **Create `web/.env.local`:** + ```bash + NEXT_PUBLIC_API_BASE=http://localhost:8001 + ``` + +3. **Check firewall settings** + +4. **For remote access**, ensure: + - `NEXT_PUBLIC_API_BASE` is set to your server IP + - Firewall allows connections on port 8001 + +### npm: command not found + +**Symptoms:** +- `npm: command not found` error +- Exit status 127 + +**Solutions:** + +1. **Install Node.js:** + ```bash + # Using conda (recommended) + conda install -c conda-forge nodejs + + # Using Homebrew (macOS) + brew install node + + # Or download from https://nodejs.org/ + ``` + +2. **Verify installation:** + ```bash + node --version + npm --version + ``` + +### Docker Issues + +**Problem: Frontend cannot connect in cloud deployment** + +**Solution:** +Set `NEXT_PUBLIC_API_BASE_EXTERNAL` in `.env`: +```bash +NEXT_PUBLIC_API_BASE_EXTERNAL=https://your-server.com:8001 +``` + +**Problem: Custom ports not working** + +**Solution:** +Set both environment variables AND port mappings: +```bash +docker run -d --name deeptutor \ + -p 9001:9001 -p 4000:4000 \ + -e BACKEND_PORT=9001 \ + -e FRONTEND_PORT=4000 \ + -e NEXT_PUBLIC_API_BASE_EXTERNAL=http://localhost:9001 \ + --env-file .env \ + ghcr.io/hkuds/deeptutor:latest +``` + +### Knowledge Base Issues + +**Problem: Numbered items extraction failed** + +**Solution:** +```bash +# Use the shell script +./scripts/extract_numbered_items.sh + +# Or direct Python command +python src/knowledge/extract_numbered_items.py --kb --base-dir ./data/knowledge_bases +``` + +### Windows-Specific Issues + +**Problem: Long path names error** + +**Solution:** +Enable long path support (run as Administrator): +```cmd +reg add "HKLM\SYSTEM\CurrentControlSet\Control\FileSystem" /v LongPathsEnabled /t REG_DWORD /d 1 /f +``` +Restart terminal after running this command. + +--- + +## Quick Reference + +### Default Ports +- **Backend**: 8001 +- **Frontend**: 3782 + +### Key URLs +- **Frontend**: http://localhost:3782 +- **Backend API**: http://localhost:8001/docs +- **API Base**: http://localhost:8001 + +### Important Directories +- **Configuration**: `config/` +- **Data Storage**: `data/` +- **Knowledge Bases**: `data/knowledge_bases/` +- **User Data**: `data/user/` + +### Common Commands + +```bash +# Start application +python scripts/start_web.py + +# Install dependencies +python scripts/install_all.py + +# Start with Docker +docker compose up + +# Stop Docker +docker compose down + +# View logs (Docker) +docker compose logs -f + +# Check backend health +curl http://localhost:8001/health +``` + +--- + +## Next Steps + +1. ✅ **Complete installation** using one of the methods above +2. ✅ **Configure API keys** in `.env` file +3. ✅ **Launch the application** using `python scripts/start_web.py` +4. ✅ **Access the web interface** at http://localhost:3782 +5. ✅ **Create your first knowledge base** +6. ✅ **Start using DeepTutor features** + +--- + +## Additional Resources + +- **Official Documentation**: https://hkuds.github.io/DeepTutor/ +- **GitHub Repository**: https://github.com/HKUDS/DeepTutor +- **Issues & Support**: https://github.com/HKUDS/DeepTutor/issues +- **Discord Community**: https://discord.gg/eRsjPgMU4t + +--- + +## Support + +If you encounter issues: + +1. Check the [FAQ section](https://github.com/HKUDS/DeepTutor#-faq) in the README +2. Search existing [GitHub Issues](https://github.com/HKUDS/DeepTutor/issues) +3. Create a new issue with: + - Error messages + - Steps to reproduce + - System information (OS, Python version, etc.) +4. Join the [Discord community](https://discord.gg/eRsjPgMU4t) for help + +--- + +**Last Updated**: Based on DeepTutor v0.6.0 diff --git a/OPTIMIZE_PERFORMANCE.md b/OPTIMIZE_PERFORMANCE.md new file mode 100644 index 00000000..4effbcaa --- /dev/null +++ b/OPTIMIZE_PERFORMANCE.md @@ -0,0 +1,116 @@ +# Performance Optimization Guide + +## Current Issue: High CPU Usage (1696% = ~17 cores) + +The Docker container is using excessive CPU resources, causing slowdowns. + +## Root Causes + +1. **Docling Parser is CPU-Intensive** + - PDF parsing, OCR, layout analysis + - No concurrency limits + - Processes multiple files simultaneously + +2. **No Docker Resource Limits** + - Container can use all available CPUs + - No memory limits + - Causes system slowdown + +3. **Multiple Processes Running** + - 243 PIDs in container + - Background tasks competing for resources + +## Solutions Applied + +### 1. Docker Resource Limits (docker-compose.yml) + +Added CPU and memory limits: +```yaml +deploy: + resources: + limits: + cpus: '8' # Max 8 CPUs + memory: 12G # Max 12GB RAM + reservations: + cpus: '2' # Min 2 CPUs + memory: 4G # Min 4GB RAM +``` + +**To apply:** +```bash +docker compose down +docker compose up -d +``` + +### 2. Process Files Sequentially + +Files are already processed one at a time (see `add_documents.py:531`), but Docling parser itself uses multiple threads internally. + +### 3. Monitor Resource Usage + +```bash +# Watch Docker stats +docker stats deeptutor + +# Check container limits +docker inspect deeptutor | grep -A 10 "Resources" +``` + +## Additional Optimizations + +### Option 1: Reduce CPU Limit (If Still Too High) + +Edit `docker-compose.yml`: +```yaml +deploy: + resources: + limits: + cpus: '4' # Reduce to 4 CPUs + memory: 8G +``` + +### Option 2: Process Fewer Files at Once + +Currently processes 1 file at a time. If you want to batch process, you can modify the code, but sequential is recommended for stability. + +### Option 3: Use MinerU Instead of Docling (Faster) + +MinerU parser is generally faster than Docling: +- Switch KB metadata to `"rag_provider": "raganything"` +- Trade-off: Docling has better table/layout parsing + +### Option 4: Increase Docker Desktop Resources + +In Docker Desktop → Settings → Resources: +- CPUs: Set to match your system (e.g., 8-12) +- Memory: Set to 16GB+ if available +- Apply & Restart + +## Expected Results + +After applying resource limits: +- **CPU Usage**: Should drop to ~400-800% (4-8 cores) +- **Performance**: Slightly slower but more stable +- **System**: Won't freeze or slow down your Mac + +## Monitoring + +```bash +# Real-time monitoring +watch -n 2 'docker stats --no-stream deeptutor' + +# Check if limits are applied +docker inspect deeptutor | grep -A 15 "Resources" +``` + +## Troubleshooting + +If container is too slow after limits: +1. Increase CPU limit to 6-8 CPUs +2. Ensure Docker Desktop has enough resources allocated +3. Close other resource-intensive applications + +If container crashes: +1. Increase memory limit +2. Check logs: `docker compose logs deeptutor` +3. Reduce concurrent operations diff --git a/TROUBLESHOOT_EMBEDDING.md b/TROUBLESHOOT_EMBEDDING.md new file mode 100644 index 00000000..d1ff9f7f --- /dev/null +++ b/TROUBLESHOOT_EMBEDDING.md @@ -0,0 +1,252 @@ +# Knowledge Base Embedding Process - Troubleshooting Options + +## Current Situation +- **Process Status**: Running but appears stuck +- **Knowledge Base**: Mathmatics +- **Progress**: 0/27 documents (0%) +- **Last Update**: Yesterday (stale progress file) + +--- + +## Available Options + +### Option 1: Monitor the Process (Recommended First Step) +Check if the process is actually making progress: + +```bash +# Watch Docker logs in real-time +cd /Users/anup.singh/Documents/Deeptutor/DeepTutor +docker compose logs -f deeptutor + +# Or check recent logs for errors +docker compose logs --tail=100 deeptutor | grep -i "error\|fail\|Mathmatics" +``` + +**What to look for:** +- Embedding API calls +- Document processing messages +- Error messages +- Timeout warnings + +--- + +### Option 2: Kill the Stuck Process +If the process is truly stuck, kill it: + +```bash +# Method 1: Kill by process ID +kill 57432 57430 57418 + +# Method 2: Kill by pattern (more thorough) +pkill -f "docker compose exec.*DocumentAdder" +pkill -f "docker compose exec.*add_documents" + +# Method 3: Use the provided script +chmod +x kill_stuck_process.sh +./kill_stuck_process.sh +``` + +**After killing:** +- Clear the progress file (see Option 3) +- Restart the process (see Option 4) + +--- + +### Option 3: Clear Progress File +Clear the stale progress file to reset the status: + +**Via API:** +```bash +curl -X POST http://localhost:8001/api/v1/knowledge/Mathmatics/progress/clear +``` + +**Via File System:** +```bash +rm /Users/anup.singh/Documents/Deeptutor/DeepTutor/data/knowledge_bases/Mathmatics/.progress.json +``` + +**Via Docker:** +```bash +docker compose exec deeptutor rm /app/data/knowledge_bases/Mathmatics/.progress.json +``` + +--- + +### Option 4: Restart the Processing +After clearing progress, restart the embedding process: + +**Via Web UI:** +1. Go to `http://localhost:3782/knowledge` +2. Find "Mathmatics" knowledge base +3. Click "Process" or "Initialize" button + +**Via Command Line:** +```bash +docker compose exec deeptutor python3 -c " +import sys +sys.path.insert(0, '/app') +from pathlib import Path +import asyncio +from src.knowledge.add_documents import DocumentAdder +from src.services.llm import get_llm_config + +# Get missing files +raw_dir = Path('/app/data/knowledge_bases/Mathmatics/raw') +content_list_dir = Path('/app/data/knowledge_bases/Mathmatics/content_list') + +processed = set(f.stem for f in content_list_dir.glob('*.json') if f.is_file()) +all_pdfs = [f for f in raw_dir.glob('*.pdf') if f.is_file()] +missing = [f for f in all_pdfs if f.stem not in processed] + +if missing: + llm_config = get_llm_config() + adder = DocumentAdder( + kb_name='Mathmatics', + base_dir='/app/data/knowledge_bases', + api_key=llm_config.api_key, + base_url=llm_config.base_url, + rag_provider='raganything_docling' + ) + + file_paths = [str(f) for f in missing] + print(f'Processing {len(file_paths)} files...') + + async def process(): + staged = adder.add_documents(file_paths, allow_duplicates=False) + if staged: + processed_files = await adder.process_new_documents(staged) + if processed_files: + adder.extract_numbered_items_for_new_docs(processed_files) + adder.update_metadata(len(processed_files)) + print(f'✅ Successfully processed {len(processed_files)} files') + else: + print('⚠️ No files were processed') + else: + print('⚠️ No files to stage') + + asyncio.run(process()) +else: + print('✅ All files already processed') +" +``` + +--- + +### Option 5: Check System Status +Verify that embedding service is working: + +**Check Embedding API Status:** +```bash +# Test embedding connection +curl -X POST http://localhost:8001/api/v1/system/test/embeddings + +# Check system status +curl http://localhost:8001/api/v1/system/status +``` + +**Check Docker Container:** +```bash +docker compose ps +docker compose exec deeptutor python3 -c " +from src.services.embedding import get_embedding_client, get_embedding_config +import asyncio + +async def test(): + try: + config = get_embedding_config() + client = get_embedding_client() + print(f'Embedding Model: {config.model}') + print(f'Provider: {config.binding}') + result = await client.embed(['test']) + print(f'✅ Embedding service working! Vector size: {len(result[0])}') + except Exception as e: + print(f'❌ Error: {e}') + +asyncio.run(test()) +" +``` + +--- + +### Option 6: Check Resource Usage +Verify system resources aren't exhausted: + +```bash +# Check Docker container resources +docker stats deeptutor + +# Check disk space +df -h +docker system df + +# Check memory +docker compose exec deeptutor free -h 2>/dev/null || docker compose exec deeptutor cat /proc/meminfo | head -5 +``` + +--- + +### Option 7: Wait and Monitor +If the process is just slow (not stuck), you can wait: + +**Monitor progress via WebSocket:** +- Open browser console on knowledge page +- Progress updates come via WebSocket every few seconds + +**Check progress file periodically:** +```bash +watch -n 5 'cat /Users/anup.singh/Documents/Deeptutor/DeepTutor/data/knowledge_bases/Mathmatics/.progress.json' +``` + +--- + +## Recommended Action Plan + +1. **First**: Monitor logs (Option 1) for 2-3 minutes to see if there's any activity +2. **If stuck**: Kill process (Option 2) → Clear progress (Option 3) → Restart (Option 4) +3. **If slow**: Check system status (Option 5) and resources (Option 6) +4. **If working**: Just wait and monitor (Option 7) + +--- + +## Quick Fix Script + +Run this to kill stuck process, clear progress, and check status: + +```bash +#!/bin/bash +cd /Users/anup.singh/Documents/Deeptutor/DeepTutor + +echo "1. Killing stuck processes..." +pkill -f "docker compose exec.*DocumentAdder" 2>/dev/null +sleep 2 + +echo "2. Clearing progress file..." +docker compose exec -T deeptutor rm -f /app/data/knowledge_bases/Mathmatics/.progress.json 2>/dev/null + +echo "3. Checking embedding service..." +docker compose exec -T deeptutor python3 -c " +from src.services.embedding import get_embedding_client, get_embedding_config +import asyncio + +async def test(): + try: + config = get_embedding_config() + client = get_embedding_client() + result = await client.embed(['test']) + print(f'✅ Embedding OK: {config.model} ({len(result[0])}D)') + except Exception as e: + print(f'❌ Embedding Error: {e}') + +asyncio.run(test()) +" + +echo "4. Done! You can now restart processing via web UI or command line." +``` + +--- + +## Need More Help? + +- Check logs: `docker compose logs deeptutor` +- Check API docs: `http://localhost:8001/docs` +- Check web UI: `http://localhost:3782/knowledge` diff --git a/content_files.txt b/content_files.txt new file mode 100644 index 00000000..0001fdda --- /dev/null +++ b/content_files.txt @@ -0,0 +1,25 @@ +1. Number System +10. Percentage +11. Speed, Time, Distance +12. Fund concepts Algebra +13. Simple Equations +14. Fund Geom Concepts +15. Angles +16. Triangles +17. Quadrilaterals +19. Polygons +2. Operations on Whole Numbers +20. 3D Shapes Solids +21. Linear Symmetry +22. Geom Constructions +23. Permiter and Area of Plane fig +24. Data Handling +25. Graphical rep of Data +26. Mean and Median +3. Integers +4. Fractions +5. Decimal Fractions +6. Playing with Numbers +7. Sets +8. Ratio and Proportion +9. Unitary Method diff --git a/docker-compose.yml b/docker-compose.yml index b7a3c0bb..dc16219f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,11 +78,21 @@ services: volumes: # Mount local config directory (read-only) - ./config:/app/config:ro + # Mount local src directory for development (read-only) + - ./src:/app/src:ro # Mount local data directory for persistence (read-write) # This ensures all outputs go to your local ./data directory - ./data/user:/app/data/user - ./data/knowledge_bases:/app/data/knowledge_bases + # Resource limits to prevent CPU/memory overload + # Note: For Docker Compose v2+, use 'deploy' section (requires swarm mode) + # For standalone, use 'cpus' and 'mem_limit' (deprecated but works) + # Alternative: Use 'docker update --cpus=8 --memory=12g deeptutor' after starting + cpus: '8' # Limit to 8 CPUs (adjust based on your system) + mem_limit: 12g # Limit to 12GB RAM + mem_reservation: 4g # Reserve 4GB RAM minimum + healthcheck: test: ["CMD", "curl", "-f", "http://localhost:${BACKEND_PORT:-8001}/"] interval: 30s diff --git a/kill_stuck_process.sh b/kill_stuck_process.sh new file mode 100755 index 00000000..12a538fc --- /dev/null +++ b/kill_stuck_process.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Kill stuck knowledge base processing processes + +echo "Killing stuck processes..." +pkill -f "docker compose exec.*DocumentAdder" +pkill -f "docker compose exec.*add_documents" +pkill -f "python.*DocumentAdder" + +echo "Checking if processes are killed..." +ps aux | grep -E "docker compose exec|DocumentAdder" | grep -v grep + +echo "Done!" diff --git a/quick_fix_embedding.sh b/quick_fix_embedding.sh new file mode 100755 index 00000000..dab299ed --- /dev/null +++ b/quick_fix_embedding.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Quick fix for stuck knowledge base embedding process + +cd /Users/anup.singh/Documents/Deeptutor/DeepTutor + +echo "==========================================" +echo "Knowledge Base Embedding - Quick Fix" +echo "==========================================" +echo "" + +echo "Step 1: Checking for stuck processes..." +PROCESSES=$(ps aux | grep -E "docker compose exec.*DocumentAdder|docker compose exec.*add_documents" | grep -v grep | wc -l) +if [ "$PROCESSES" -gt 0 ]; then + echo " Found $PROCESSES stuck process(es). Killing..." + pkill -f "docker compose exec.*DocumentAdder" 2>/dev/null + pkill -f "docker compose exec.*add_documents" 2>/dev/null + sleep 2 + echo " ✅ Processes killed" +else + echo " ✅ No stuck processes found" +fi + +echo "" +echo "Step 2: Clearing stale progress file..." +docker compose exec -T deeptutor rm -f /app/data/knowledge_bases/Mathmatics/.progress.json 2>/dev/null +if [ $? -eq 0 ]; then + echo " ✅ Progress file cleared" +else + echo " ⚠️ Could not clear progress file (may not exist)" +fi + +echo "" +echo "Step 3: Testing embedding service..." +docker compose exec -T deeptutor python3 -c " +from src.services.embedding import get_embedding_client, get_embedding_config +import asyncio + +async def test(): + try: + config = get_embedding_config() + client = get_embedding_client() + result = await client.embed(['test']) + print(f' ✅ Embedding service OK') + print(f' Model: {config.model}') + print(f' Provider: {config.binding}') + print(f' Vector dimension: {len(result[0])}') + except Exception as e: + print(f' ❌ Embedding service ERROR: {e}') + +asyncio.run(test()) +" 2>&1 | sed 's/^/ /' + +echo "" +echo "Step 4: Checking Docker container status..." +docker compose ps deeptutor + +echo "" +echo "==========================================" +echo "Next Steps:" +echo "==========================================" +echo "1. Monitor logs: docker compose logs -f deeptutor" +echo "2. Restart processing via web UI: http://localhost:3782/knowledge" +echo "3. Or use the command line script (see TROUBLESHOOT_EMBEDDING.md)" +echo "" +echo "Done!" diff --git a/raw_files.txt b/raw_files.txt new file mode 100644 index 00000000..bd10e629 --- /dev/null +++ b/raw_files.txt @@ -0,0 +1,27 @@ +1. Number System +10. Percentage +11. Speed, Time, Distance +12. Fund concepts Algebra +13. Simple Equations +14. Fund Geom Concepts +15. Angles +16. Triangles +17. Quadrilaterals +18. Circles +19. Polygons +2. Operations on Whole Numbers +20. 3D Shapes Solids +21. Linear Symmetry +22. Geom Constructions +23. Permiter and Area of Plane fig +24. Data Handling +25. Graphical rep of Data +26. Mean and Median +27. Answers +3. Integers +4. Fractions +5. Decimal Fractions +6. Playing with Numbers +7. Sets +8. Ratio and Proportion +9. Unitary Method diff --git a/src/agents/guide/agents/locate_agent.py b/src/agents/guide/agents/locate_agent.py index da48515d..585ca40d 100644 --- a/src/agents/guide/agents/locate_agent.py +++ b/src/agents/guide/agents/locate_agent.py @@ -106,30 +106,77 @@ async def process( try: result = json.loads(response) + # Debug: Log the parsed result structure + self.logger.debug(f"Parsed JSON result type: {type(result).__name__}") + if isinstance(result, dict): + self.logger.debug(f"Result keys: {list(result.keys())}") + if isinstance(result, list): knowledge_points = result elif isinstance(result, dict): - knowledge_points = ( - result.get("knowledge_points") - or result.get("points") - or result.get("data") - or [] - ) + # Check if dict IS a single knowledge point (has knowledge_title key) + if "knowledge_title" in result or "title" in result: + self.logger.debug("Result is a single knowledge point, wrapping in list") + knowledge_points = [result] + else: + # Try multiple possible keys for array of points + knowledge_points = ( + result.get("knowledge_points") + or result.get("points") + or result.get("data") + or result.get("items") + or result.get("learning_points") + or [] + ) + # If still empty but dict has content, try to extract from first list-like value + if not knowledge_points: + for key, value in result.items(): + if isinstance(value, list) and len(value) > 0: + self.logger.debug(f"Found list in key '{key}' with {len(value)} items") + knowledge_points = value + break else: knowledge_points = [] + self.logger.debug(f"Extracted {len(knowledge_points)} knowledge points") + validated_points = [] - for point in knowledge_points: + for i, point in enumerate(knowledge_points): if isinstance(point, dict): + # Debug: Log point keys + if i == 0: + self.logger.debug(f"First point keys: {list(point.keys())}") + + # Try multiple key variations + title = ( + point.get("knowledge_title") + or point.get("title") + or point.get("name") + or "Unnamed knowledge point" + ) + summary = ( + point.get("knowledge_summary") + or point.get("summary") + or point.get("description") + or point.get("content") + or "" + ) + difficulty = ( + point.get("user_difficulty") + or point.get("difficulty") + or point.get("challenges") + or "" + ) + validated_points.append( { - "knowledge_title": point.get( - "knowledge_title", "Unnamed knowledge point" - ), - "knowledge_summary": point.get("knowledge_summary", ""), - "user_difficulty": point.get("user_difficulty", ""), + "knowledge_title": title, + "knowledge_summary": summary, + "user_difficulty": difficulty, } ) + else: + self.logger.debug(f"Point {i} is not a dict: {type(point).__name__}") return { "success": True, diff --git a/src/agents/research/agents/research_agent.py b/src/agents/research/agents/research_agent.py index 7e7a29b9..7db791c8 100644 --- a/src/agents/research/agents/research_agent.py +++ b/src/agents/research/agents/research_agent.py @@ -616,7 +616,7 @@ def send_progress(event_type: str, **data): # Step 3: Call tool raw_answer = await call_tool_callback(tool_type, query) - # Send progress after tool call + # Send progress after tool call (summary will be added after NoteAgent processes) send_progress( "tool_completed", iteration=iteration, @@ -657,6 +657,17 @@ def send_progress(event_type: str, **data): ) topic_block.add_tool_trace(trace) + # Send note_generated event with the summary for UI display + send_progress( + "note_generated", + iteration=iteration, + max_iterations=self.max_iterations, + tool_type=tool_type, + query=query, + summary=trace.summary, + citation_id=citation_id, + ) + # Step 6: Add citation information to citation manager # Support both sync and async citation_manager if hasattr(citation_manager, "add_citation") and callable( diff --git a/src/knowledge/add_documents.py b/src/knowledge/add_documents.py index 7d845cbc..3b7bdd97 100644 --- a/src/knowledge/add_documents.py +++ b/src/knowledge/add_documents.py @@ -25,6 +25,7 @@ import tempfile from typing import Any, Dict, List +import numpy as np from dotenv import load_dotenv from src.knowledge.extract_numbered_items import process_content_list @@ -331,7 +332,9 @@ async def _process_lightrag(self, new_files: List[Path]) -> List[Path]: embedding_client = get_embedding_client() async def unified_embed_func(texts): - return await embedding_client.embed(texts) + embeddings = await embedding_client.embed(texts) + # Convert list of lists to numpy array for LightRAG compatibility + return np.array(embeddings) embedding_func = EmbeddingFunc( embedding_dim=embedding_cfg.dim, @@ -485,7 +488,9 @@ async def _process_raganything( embedding_client = get_embedding_client() async def unified_embed_func(texts): - return await embedding_client.embed(texts) + embeddings = await embedding_client.embed(texts) + # Convert list of lists to numpy array for LightRAG compatibility + return np.array(embeddings) embedding_func = EmbeddingFunc( embedding_dim=embedding_cfg.dim, diff --git a/web/components/research/ActiveTaskDetail.tsx b/web/components/research/ActiveTaskDetail.tsx index c36767a8..6f08781f 100644 --- a/web/components/research/ActiveTaskDetail.tsx +++ b/web/components/research/ActiveTaskDetail.tsx @@ -120,7 +120,9 @@ export const ActiveTaskDetail: React.FC = ({ task }) => { ? "bg-violet-50 dark:bg-violet-900/40 border-violet-200 dark:border-violet-800" : thought.type === "tool_call" ? "bg-amber-50 dark:bg-amber-900/40 border-amber-200 dark:border-amber-800" - : "bg-slate-50 dark:bg-slate-700 border-slate-200 dark:border-slate-600" + : thought.type === "note" + ? "bg-emerald-50 dark:bg-emerald-900/40 border-emerald-200 dark:border-emerald-800" + : "bg-slate-50 dark:bg-slate-700 border-slate-200 dark:border-slate-600" }`} > {getThoughtIcon(thought.type)} @@ -141,17 +143,38 @@ export const ActiveTaskDetail: React.FC = ({ task }) => { ? "text-violet-600 dark:text-violet-400" : thought.type === "tool_call" ? "text-amber-600 dark:text-amber-400" - : "text-slate-600 dark:text-slate-400" + : thought.type === "note" + ? "text-emerald-600 dark:text-emerald-400" + : "text-slate-600 dark:text-slate-400" }`} > - {thought.type.replace("_", " ")} + {thought.type === "note" + ? "Research Finding" + : thought.type.replace("_", " ")} {formatTimestamp(thought.timestamp)} -
+
+ {/* Show metadata header for notes */} + {thought.type === "note" && thought.metadata && ( +
+ + [{thought.metadata.citation_id}] + + + via {thought.metadata.tool_type} + +
+ )} ( diff --git a/web/hooks/useResearchReducer.ts b/web/hooks/useResearchReducer.ts index 348c53be..9a757d71 100644 --- a/web/hooks/useResearchReducer.ts +++ b/web/hooks/useResearchReducer.ts @@ -400,6 +400,30 @@ export const researchReducer = ( }; } + case "note_generated": { + const blockId = event.block_id; + if (!blockId || !state.tasks[blockId]) return state; + + const task = state.tasks[blockId]; + // Add the research note/summary as a thought + return { + ...state, + tasks: { + ...state.tasks, + [blockId]: addThought(task, { + type: "note", + content: event.summary || "No summary generated", + metadata: { + tool_type: event.tool_type, + query: event.query, + citation_id: event.citation_id, + }, + timestamp: Date.now(), + }), + }, + }; + } + case "processing_notes": { const blockId = event.block_id; if (!blockId || !state.tasks[blockId]) return state; diff --git a/web/types/research.ts b/web/types/research.ts index f5e6bfd4..731e82b8 100644 --- a/web/types/research.ts +++ b/web/types/research.ts @@ -134,6 +134,7 @@ export type ResearchEventType = | "generating_query" | "tool_calling" | "tool_completed" + | "note_generated" | "processing_notes" | "iteration_completed" | "new_topic_added"