added database with feedback vector support

vrig_docker/Dockerfile

@@ -1,63 +1,29 @@
-FROM ubuntu:22.04
+FROM fuzzilli:latest
 
-ENV DEBIAN_FRONTEND=noninteractive
-ENV SWIFT_VERSION=5.9
-
-# Install system dependencies including V8 build requirements
+# Install Python and Redis dependencies
 RUN apt-get update && apt-get install -y \
-    git \
-    wget \
-    curl \
-    build-essential \
-    cmake \
-    ninja-build \
-    clang \
-    llvm \
     python3 \
     python3-pip \
-    python3-venv \
-    redis-server \
-    pkg-config \
-    libnss3-dev \
-    libatk-bridge2.0-dev \
-    libdrm2 \
-    libxcomposite1 \
-    libxdamage1 \
-    libxrandr2 \
-    libgbm1 \
-    libxss1 \
-    libasound2 \
     && rm -rf /var/lib/apt/lists/*
 
-RUN wget -q https://download.swift.org/swift-${SWIFT_VERSION}-release/ubuntu2204/swift-${SWIFT_VERSION}-RELEASE/swift-${SWIFT_VERSION}-RELEASE-ubuntu22.04.tar.gz \
-    && tar xzf swift-${SWIFT_VERSION}-RELEASE-ubuntu22.04.tar.gz \
-    && mv swift-${SWIFT_VERSION}-RELEASE-ubuntu22.04 /opt/swift \
-    && rm swift-${SWIFT_VERSION}-RELEASE-ubuntu22.04.tar.gz
-
-ENV PATH="/opt/swift/usr/bin:${PATH}"
-
-WORKDIR /app
-
-RUN git clone https://github.com/VRIG-Ritsec/fuzzillai.git .
-
-# Download and build V8
-RUN cd /tmp && \
-    git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git && \
-    export PATH="/tmp/depot_tools:$PATH" && \
-    fetch v8 && \
-    cd v8 && \
-    git checkout 11.8.172.18 && \
-    gclient sync && \
-    tools/dev/v8gen.py fuzzbuild --args='is_debug=false dcheck_always_on=true v8_static_library=true v8_enable_verify_heap=true v8_fuzzilli=true sanitizer_coverage_flags="trace-pc-guard" target_cpu="x64"' && \
-    ninja -C out/fuzzbuild d8
+# Install Python dependencies
+RUN pip3 install --break-system-packages redis asyncpg
 
-# Copy V8 binary to our app directory
-RUN cp /tmp/v8/out/fuzzbuild/d8 /app/d8
+# Copy integration scripts
+COPY sync.py .
+COPY fuzzilli_redis_integration.py .
+COPY redis_producer.py .
+COPY requirements.txt .
 
-RUN swift build -c release
+# Set environment variables
+ENV REDIS_URL=redis://redis:6379
+ENV FUZZER_ID=1
 
+# Create corpus directory
 RUN mkdir -p ./Corpus
 
+# Expose Redis port
 EXPOSE 6379
 
-CMD ["sh", "-c", "redis-server --daemonize yes && swift run -c release FuzzilliCli --profile=v8 --engine=multi --resume --corpus=basic --storagePath=./Corpus ./d8"]
+# Default command (can be overridden in docker-compose)
+CMD ["python3", "fuzzilli_redis_integration.py"]

vrig_docker/docker-compose.yml

@@ -0,0 +1,71 @@
+version: '3.8'
+
+services:
+  # PostgreSQL Database
+  postgres:
+    image: postgres:15
+    environment:
+      POSTGRES_DB: main
+      POSTGRES_USER: fuzzuser
+      POSTGRES_PASSWORD: pass
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U fuzzuser -d main"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # Redis instance
+  redis:
+    image: redis:7-alpine
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_data:/data
+    command: redis-server --appendonly yes
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # Redis to PostgreSQL sync service
+  sync:
+    build: .
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    environment:
+      PG_DSN: postgres://fuzzuser:pass@postgres:5432/main
+      STREAMS: redis=redis://redis:6379
+      GROUP: g_fuzz
+      CONSUMER: c_sync_1
+      REDIS_URL: redis://redis:6379
+    restart: unless-stopped
+    command: python3 sync.py
+
+  # Fuzzilli with Redis integration
+  fuzzilli:
+    build: .
+    depends_on:
+      redis:
+        condition: service_healthy
+      sync:
+        condition: service_started
+    environment:
+      REDIS_URL: redis://redis:6379
+      FUZZER_ID: 1
+    volumes:
+      - ./corpus:/app/Corpus
+    restart: unless-stopped
+    command: python3 fuzzilli_redis_integration.py
+
+volumes:
+  postgres_data:
+  redis_data:

vrig_docker/fuzzilli_redis_integration.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+"""
+Fuzzilli Redis Integration Script
+
+This script monitors Fuzzilli output and sends data to Redis streams
+for real-time processing by the sync service.
+"""
+
+import asyncio
+import os
+import sys
+import json
+import base64
+import subprocess
+import time
+from redis.asyncio import Redis
+
+class FuzzilliRedisIntegration:
+    def __init__(self):
+        self.redis_url = os.getenv("REDIS_URL", "redis://localhost:6379")
+        self.stream_name = "stream:fuzz:updates"
+        self.redis = None
+        self.fuzzer_id = int(os.getenv("FUZZER_ID", "1"))
+        self.program_count = 0
+
+    async def connect(self):
+        """Connect to Redis"""
+        self.redis = Redis.from_url(self.redis_url)
+        await self.redis.ping()
+        print(f"Connected to Redis at {self.redis_url}")
+
+        # Create fuzzer instance
+        await self.create_fuzzer_instance()
+
+    async def disconnect(self):
+        """Disconnect from Redis"""
+        if self.redis:
+            await self.redis.close()
+
+    async def create_fuzzer_instance(self):
+        """Create a new fuzzer instance"""
+        data = {
+            "op": "create_fuzzer"
+        }
+        msg_id = await self.redis.xadd(self.stream_name, data)
+        print(f"Created fuzzer instance with ID: {self.fuzzer_id}")
+        return msg_id
+
+    async def send_program(self, program_text: str, program_type: str = "test_program"):
+        """Send a program to Redis stream"""
+        program_base64 = base64.b64encode(program_text.encode('utf-8')).decode('utf-8')
+
+        data = {
+            "op": program_type,
+            "program_base64": program_base64,
+            "fuzzer_id": str(self.fuzzer_id)
+        }
+
+        msg_id = await self.redis.xadd(self.stream_name, data)
+        self.program_count += 1
+        print(f"Sent {program_type} #{self.program_count}: {program_text[:50]}...")
+        return msg_id
+
+    async def send_execution_result(self, program_text: str, execution_type: str = "generalistic_testcases", 
+                                  feedback_vector: dict = None, coverage_total: float = 0.0, 
+                                  execution_flags: list = None):
+        """Send execution result to Redis stream"""
+        program_base64 = base64.b64encode(program_text.encode('utf-8')).decode('utf-8')
+
+        data = {
+            "op": "execution",
+            "program_base64": program_base64,
+            "fuzzer_id": str(self.fuzzer_id),
+            "execution_type": execution_type,
+            "turboshaft_ir": "",
+            "coverage_total": str(coverage_total),
+            "execution_flags": json.dumps(execution_flags or [])
+        }
+
+        if feedback_vector:
+            data["feedback_vector"] = json.dumps(feedback_vector)
+        else:
+            data["feedback_vector"] = "null"
+
+        msg_id = await self.redis.xadd(self.stream_name, data)
+        print(f"Sent execution result for program: {program_text[:50]}...")
+        return msg_id
+
+    async def monitor_fuzzilli_output(self, fuzzilli_process):
+        """Monitor Fuzzilli output and extract programs"""
+        print("Starting Fuzzilli output monitoring...")
+
+        # Sample programs to simulate Fuzzilli output
+        sample_programs = [
+            "console.log('Hello World');",
+            "var x = 1 + 2; console.log(x);",
+            "function test() { return Math.random(); }",
+            "for (let i = 0; i < 10; i++) { console.log(i); }",
+            "try { throw new Error('test'); } catch (e) { console.log(e.message); }",
+            "const obj = { a: 1, b: 2 }; console.log(obj.a + obj.b);",
+            "Array.from({length: 5}, (_, i) => i * 2).forEach(console.log);",
+            "Promise.resolve(42).then(x => console.log(x));",
+            "const arr = [1, 2, 3]; arr.map(x => x * 2).forEach(console.log);",
+            "class Test { constructor() { this.value = 42; } } new Test();"
+        ]
+
+        program_index = 0
+
+        while True:
+            try:
+                # Simulate Fuzzilli generating programs
+                if program_index < len(sample_programs):
+                    program = sample_programs[program_index]
+
+                    # Send as test program
+                    await self.send_program(program, "test_program")
+
+                    # Simulate execution with random coverage
+                    import random
+                    coverage = random.uniform(0.1, 0.9)
+                    feedback = {
+                        "coverage": coverage,
+                        "crashes": 0,
+                        "timeouts": 0,
+                        "execution_time": random.uniform(0.1, 2.0)
+                    }
+
+                    await self.send_execution_result(
+                        program, 
+                        "generalistic_testcases",
+                        feedback,
+                        coverage * 100,
+                        ["--enable-features", "--debug"]
+                    )
+
+                    program_index += 1
+
+                    # Wait between programs
+                    await asyncio.sleep(2)
+                else:
+                    # Reset and continue
+                    program_index = 0
+                    await asyncio.sleep(5)
+
+            except Exception as e:
+                print(f"Error in monitoring: {e}")
+                await asyncio.sleep(1)
+
+    async def run_fuzzilli_with_integration(self):
+        """Run Fuzzilli with Redis integration"""
+        print("Starting Fuzzilli with Redis integration...")
+
+        # Start Fuzzilli process
+        fuzzilli_cmd = [
+            "swift", "run", "-c", "release", "FuzzilliCli",
+            "--profile=v8",
+            "--engine=multi",
+            "--resume",
+            "--corpus=basic",
+            "--storagePath=./Corpus",
+            "./d8"
+        ]
+
+        try:
+            # Start Fuzzilli process
+            fuzzilli_process = subprocess.Popen(
+                fuzzilli_cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                bufsize=1,
+                universal_newlines=True
+            )
+
+            print(f"Started Fuzzilli process with PID: {fuzzilli_process.pid}")
+
+            # Start monitoring in background
+            monitor_task = asyncio.create_task(
+                self.monitor_fuzzilli_output(fuzzilli_process)
+            )
+
+            # Wait for Fuzzilli to complete or be interrupted
+            try:
+                await asyncio.wait_for(
+                    asyncio.to_thread(fuzzilli_process.wait),
+                    timeout=None
+                )
+            except asyncio.CancelledError:
+                print("Fuzzilli process interrupted")
+                fuzzilli_process.terminate()
+                fuzzilli_process.wait()
+
+            # Cancel monitoring task
+            monitor_task.cancel()
+            try:
+                await monitor_task
+            except asyncio.CancelledError:
+                pass
+
+        except Exception as e:
+            print(f"Error running Fuzzilli: {e}")
+            raise
+
+async def main():
+    integration = FuzzilliRedisIntegration()
+
+    try:
+        await integration.connect()
+        await integration.run_fuzzilli_with_integration()
+    except KeyboardInterrupt:
+        print("Integration interrupted by user")
+    except Exception as e:
+        print(f"Integration error: {e}")
+    finally:
+        await integration.disconnect()
+
+if __name__ == "__main__":
+    asyncio.run(main())