diff --git a/hindsight-api/hindsight_api/alembic/env.py b/hindsight-api/hindsight_api/alembic/env.py index 80dd804c..6eea9b7a 100644 --- a/hindsight-api/hindsight_api/alembic/env.py +++ b/hindsight-api/hindsight_api/alembic/env.py @@ -7,10 +7,11 @@ import os from pathlib import Path -from alembic import context from dotenv import load_dotenv from sqlalchemy import engine_from_config, pool +from alembic import context + # Import your models here from hindsight_api.models import Base diff --git a/hindsight-api/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py b/hindsight-api/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py index 866e37dd..b35c743c 100644 --- a/hindsight-api/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py +++ b/hindsight-api/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py @@ -9,10 +9,11 @@ from collections.abc import Sequence import sqlalchemy as sa -from alembic import op from pgvector.sqlalchemy import Vector from sqlalchemy.dialects import postgresql +from alembic import op + # revision identifiers, used by Alembic. revision: str = "5a366d414dce" down_revision: str | Sequence[str] | None = None diff --git a/hindsight-api/hindsight_api/alembic/versions/b7c4d8e9f1a2_add_chunks_table.py b/hindsight-api/hindsight_api/alembic/versions/b7c4d8e9f1a2_add_chunks_table.py index d0dae396..4f111a57 100644 --- a/hindsight-api/hindsight_api/alembic/versions/b7c4d8e9f1a2_add_chunks_table.py +++ b/hindsight-api/hindsight_api/alembic/versions/b7c4d8e9f1a2_add_chunks_table.py @@ -9,9 +9,10 @@ from collections.abc import Sequence import sqlalchemy as sa -from alembic import op from sqlalchemy.dialects import postgresql +from alembic import op + # revision identifiers, used by Alembic. revision: str = "b7c4d8e9f1a2" down_revision: str | Sequence[str] | None = "5a366d414dce" diff --git a/hindsight-api/hindsight_api/alembic/versions/c8e5f2a3b4d1_add_retain_params_to_documents.py b/hindsight-api/hindsight_api/alembic/versions/c8e5f2a3b4d1_add_retain_params_to_documents.py index edbcc13e..17770560 100644 --- a/hindsight-api/hindsight_api/alembic/versions/c8e5f2a3b4d1_add_retain_params_to_documents.py +++ b/hindsight-api/hindsight_api/alembic/versions/c8e5f2a3b4d1_add_retain_params_to_documents.py @@ -9,9 +9,10 @@ from collections.abc import Sequence import sqlalchemy as sa -from alembic import op from sqlalchemy.dialects import postgresql +from alembic import op + # revision identifiers, used by Alembic. revision: str = "c8e5f2a3b4d1" down_revision: str | Sequence[str] | None = "b7c4d8e9f1a2" diff --git a/hindsight-api/hindsight_api/alembic/versions/e0a1b2c3d4e5_disposition_to_3_traits.py b/hindsight-api/hindsight_api/alembic/versions/e0a1b2c3d4e5_disposition_to_3_traits.py index 8db391ab..1853fbc3 100644 --- a/hindsight-api/hindsight_api/alembic/versions/e0a1b2c3d4e5_disposition_to_3_traits.py +++ b/hindsight-api/hindsight_api/alembic/versions/e0a1b2c3d4e5_disposition_to_3_traits.py @@ -12,6 +12,7 @@ from collections.abc import Sequence import sqlalchemy as sa + from alembic import context, op # revision identifiers, used by Alembic. diff --git a/hindsight-api/hindsight_api/alembic/versions/rename_personality_to_disposition.py b/hindsight-api/hindsight_api/alembic/versions/rename_personality_to_disposition.py index 6a711458..2c1e9dc9 100644 --- a/hindsight-api/hindsight_api/alembic/versions/rename_personality_to_disposition.py +++ b/hindsight-api/hindsight_api/alembic/versions/rename_personality_to_disposition.py @@ -9,9 +9,10 @@ from collections.abc import Sequence import sqlalchemy as sa -from alembic import context, op from sqlalchemy.dialects import postgresql +from alembic import context, op + # revision identifiers, used by Alembic. revision: str = "rename_personality" down_revision: str | Sequence[str] | None = "d9f6a3b4c5e2" diff --git a/hindsight-api/hindsight_api/api/__init__.py b/hindsight-api/hindsight_api/api/__init__.py index 1351847a..c33b29f7 100644 --- a/hindsight-api/hindsight_api/api/__init__.py +++ b/hindsight-api/hindsight_api/api/__init__.py @@ -52,6 +52,7 @@ def create_app( if mcp_api_enabled: try: from .mcp import create_mcp_app + mcp_app = create_mcp_app(memory=memory) except ImportError as e: logger.error(f"MCP server requested but dependencies not available: {e}") diff --git a/hindsight-api/hindsight_api/config.py b/hindsight-api/hindsight_api/config.py index 9f20b14a..34a68f0e 100644 --- a/hindsight-api/hindsight_api/config.py +++ b/hindsight-api/hindsight_api/config.py @@ -166,7 +166,9 @@ def from_env(cls) -> "HindsightConfig": lazy_reranker=os.getenv(ENV_LAZY_RERANKER, "false").lower() == "true", # Observation thresholds observation_min_facts=int(os.getenv(ENV_OBSERVATION_MIN_FACTS, str(DEFAULT_OBSERVATION_MIN_FACTS))), - observation_top_entities=int(os.getenv(ENV_OBSERVATION_TOP_ENTITIES, str(DEFAULT_OBSERVATION_TOP_ENTITIES))), + observation_top_entities=int( + os.getenv(ENV_OBSERVATION_TOP_ENTITIES, str(DEFAULT_OBSERVATION_TOP_ENTITIES)) + ), ) def get_llm_base_url(self) -> str: diff --git a/hindsight-api/hindsight_api/engine/llm_wrapper.py b/hindsight-api/hindsight_api/engine/llm_wrapper.py index d3e11f9b..8b9a32c7 100644 --- a/hindsight-api/hindsight_api/engine/llm_wrapper.py +++ b/hindsight-api/hindsight_api/engine/llm_wrapper.py @@ -120,11 +120,7 @@ def __init__( elif self.provider in ("ollama", "lmstudio"): # Use dummy key if not provided for local api_key = self.api_key or "local" - client_kwargs = { - "api_key": api_key, - "base_url": self.base_url, - "max_retries": 0 - } + client_kwargs = {"api_key": api_key, "base_url": self.base_url, "max_retries": 0} if self.timeout: client_kwargs["timeout"] = self.timeout self._client = AsyncOpenAI(**client_kwargs) @@ -207,7 +203,14 @@ async def call( # Handle Anthropic provider separately if self.provider == "anthropic": return await self._call_anthropic( - messages, response_format, max_completion_tokens, max_retries, initial_backoff, max_backoff, skip_validation, start_time + messages, + response_format, + max_completion_tokens, + max_retries, + initial_backoff, + max_backoff, + skip_validation, + start_time, ) # Handle Ollama with native API for structured output (better schema enforcement) @@ -297,8 +300,8 @@ async def call( schema_msg + "\n\n" + call_params["messages"][0]["content"] ) if self.provider not in ("lmstudio", "ollama"): - call_params["response_format"] = {"type": "json_object"} - + call_params["response_format"] = {"type": "json_object"} + logger.debug(f"Sending request to {self.provider}/{self.model} (timeout={self.timeout})") response = await self._client.chat.completions.create(**call_params) logger.debug(f"Received response from {self.provider}/{self.model}") diff --git a/hindsight-api/hindsight_api/engine/retain/fact_extraction.py b/hindsight-api/hindsight_api/engine/retain/fact_extraction.py index 07227dba..f8c35738 100644 --- a/hindsight-api/hindsight_api/engine/retain/fact_extraction.py +++ b/hindsight-api/hindsight_api/engine/retain/fact_extraction.py @@ -17,6 +17,44 @@ from ..llm_wrapper import LLMConfig, OutputTooLongError +def _infer_temporal_date(fact_text: str, event_date: datetime) -> str | None: + """ + Infer a temporal date from fact text when LLM didn't provide occurred_start. + + This is a fallback for when the LLM fails to extract temporal information + from relative time expressions like "last night", "yesterday", etc. + """ + import re + + fact_lower = fact_text.lower() + + # Map relative time expressions to day offsets + temporal_patterns = { + r"\blast night\b": -1, + r"\byesterday\b": -1, + r"\btoday\b": 0, + r"\bthis morning\b": 0, + r"\bthis afternoon\b": 0, + r"\bthis evening\b": 0, + r"\btonigh?t\b": 0, + r"\btomorrow\b": 1, + r"\blast week\b": -7, + r"\bthis week\b": 0, + r"\bnext week\b": 7, + r"\blast month\b": -30, + r"\bthis month\b": 0, + r"\bnext month\b": 30, + } + + for pattern, offset_days in temporal_patterns.items(): + if re.search(pattern, fact_lower): + target_date = event_date + timedelta(days=offset_days) + return target_date.replace(hour=0, minute=0, second=0, microsecond=0).isoformat() + + # If no relative time expression found, return None + return None + + def _sanitize_text(text: str) -> str: """ Sanitize text by removing invalid Unicode surrogate characters. @@ -676,13 +714,18 @@ def get_value(field_name): if fact_kind == "event": occurred_start = get_value("occurred_start") occurred_end = get_value("occurred_end") - if occurred_start: + + # If LLM didn't set temporal fields, try to extract them from the fact text + if not occurred_start: + fact_data["occurred_start"] = _infer_temporal_date(combined_text, event_date) + else: fact_data["occurred_start"] = occurred_start - # For point events: if occurred_end not set, default to occurred_start - if occurred_end: - fact_data["occurred_end"] = occurred_end - else: - fact_data["occurred_end"] = occurred_start + + # For point events: if occurred_end not set, default to occurred_start + if occurred_end: + fact_data["occurred_end"] = occurred_end + elif fact_data.get("occurred_start"): + fact_data["occurred_end"] = fact_data["occurred_start"] # Add entities if present (validate as Entity objects) # LLM sometimes returns strings instead of {"text": "..."} format diff --git a/hindsight-api/hindsight_api/main.py b/hindsight-api/hindsight_api/main.py index 04a2296a..98841baf 100644 --- a/hindsight-api/hindsight_api/main.py +++ b/hindsight-api/hindsight_api/main.py @@ -169,6 +169,8 @@ def release_lock(): llm_api_key=config.llm_api_key, llm_model=config.llm_model, llm_base_url=config.llm_base_url, + llm_max_concurrent=config.llm_max_concurrent, + llm_timeout=config.llm_timeout, embeddings_provider=config.embeddings_provider, embeddings_local_model=config.embeddings_local_model, embeddings_tei_url=config.embeddings_tei_url, @@ -180,6 +182,8 @@ def release_lock(): log_level=args.log_level, mcp_enabled=config.mcp_enabled, graph_retriever=config.graph_retriever, + observation_min_facts=config.observation_min_facts, + observation_top_entities=config.observation_top_entities, skip_llm_verification=config.skip_llm_verification, lazy_reranker=config.lazy_reranker, ) @@ -196,6 +200,7 @@ def release_lock(): operation_validator = load_extension("OPERATION_VALIDATOR", OperationValidatorExtension) if operation_validator: import logging + logging.info(f"Loaded operation validator: {operation_validator.__class__.__name__}") # Create MemoryEngine (reads configuration from environment) diff --git a/hindsight-api/hindsight_api/migrations.py b/hindsight-api/hindsight_api/migrations.py index 91022980..1502c795 100644 --- a/hindsight-api/hindsight_api/migrations.py +++ b/hindsight-api/hindsight_api/migrations.py @@ -20,10 +20,11 @@ import os from pathlib import Path -from alembic import command from alembic.config import Config from sqlalchemy import create_engine, text +from alembic import command + logger = logging.getLogger(__name__) # Advisory lock ID for migrations (arbitrary unique number) diff --git a/hindsight-api/tests/test_fact_extraction_quality.py b/hindsight-api/tests/test_fact_extraction_quality.py index 7d67900e..eebdc533 100644 --- a/hindsight-api/tests/test_fact_extraction_quality.py +++ b/hindsight-api/tests/test_fact_extraction_quality.py @@ -402,6 +402,8 @@ async def test_date_field_calculation_last_night(self): Ideally: If conversation is on August 14, 2023 and text says "last night", the date field should be August 13. We accept 13 or 14 as LLM may vary. + + Retries up to 3 times to account for LLM inconsistencies. """ text = """ Melanie: Hey Caroline! Last night was amazing! We celebrated my daughter's birthday @@ -410,41 +412,69 @@ async def test_date_field_calculation_last_night(self): context = "Conversation between Melanie and Caroline" llm_config = LLMConfig.for_memory() - event_date = datetime(2023, 8, 14, 14, 24) - facts, _ = await extract_facts_from_text( - text=text, - event_date=event_date, - context=context, - llm_config=llm_config, - agent_name="Melanie" - ) - - assert len(facts) > 0, "Should extract at least one fact" - - birthday_fact = None - for fact in facts: - if "birthday" in fact.fact.lower() or "concert" in fact.fact.lower(): - birthday_fact = fact - break - - assert birthday_fact is not None, "Should extract fact about birthday celebration" - - fact_date_str = birthday_fact.occurred_start - assert fact_date_str is not None, "occurred_start should not be None for temporal events" - - if 'T' in fact_date_str: - fact_date = datetime.fromisoformat(fact_date_str.replace('Z', '+00:00')) - else: - fact_date = datetime.fromisoformat(fact_date_str) - - assert fact_date.year == 2023, "Year should be 2023" - assert fact_date.month == 8, "Month should be August" - # Accept day 13 (ideal: last night) or 14 (conversation date) as valid - assert fact_date.day in (13, 14), ( - f"Day should be 13 or 14 (around Aug 14 event), but got {fact_date.day}." - ) + last_error = None + max_retries = 3 + + for attempt in range(max_retries): + try: + facts, _ = await extract_facts_from_text( + text=text, + event_date=event_date, + context=context, + llm_config=llm_config, + agent_name="Melanie" + ) + + assert len(facts) > 0, "Should extract at least one fact" + + birthday_fact = None + for fact in facts: + if "birthday" in fact.fact.lower() or "concert" in fact.fact.lower(): + birthday_fact = fact + break + + assert birthday_fact is not None, "Should extract fact about birthday celebration" + + fact_date_str = birthday_fact.occurred_start + assert fact_date_str is not None, "occurred_start should not be None for temporal events" + + if 'T' in fact_date_str: + fact_date = datetime.fromisoformat(fact_date_str.replace('Z', '+00:00')) + else: + fact_date = datetime.fromisoformat(fact_date_str) + + assert fact_date.year == 2023, "Year should be 2023" + assert fact_date.month == 8, "Month should be August" + # Accept day 13 (ideal: last night) or 14 (conversation date) as valid + assert fact_date.day in (13, 14), ( + f"Day should be 13 or 14 (around Aug 14 event), but got {fact_date.day}." + ) + + # If we reach here, test passed + return + + except AssertionError as e: + last_error = e + if attempt < max_retries - 1: + print(f"Test attempt {attempt + 1} failed: {e}. Retrying...") + continue + else: + # Last attempt failed, re-raise the error + raise e + except Exception as e: + last_error = e + if attempt < max_retries - 1: + print(f"Test attempt {attempt + 1} failed with exception: {e}. Retrying...") + continue + else: + # Last attempt failed, re-raise the error + raise e + + # Should not reach here, but just in case + if last_error: + raise last_error @pytest.mark.asyncio async def test_date_field_calculation_yesterday(self): diff --git a/hindsight-control-plane/package.json b/hindsight-control-plane/package.json index 90aabd0e..c157c8e6 100644 --- a/hindsight-control-plane/package.json +++ b/hindsight-control-plane/package.json @@ -47,6 +47,7 @@ "clsx": "^2.1.1", "cmdk": "^1.1.1", "cytoscape": "^3.33.1", + "cytoscape-fcose": "^2.2.0", "eslint": "^9.39.1", "eslint-config-next": "^16.0.1", "lucide-react": "^0.553.0", diff --git a/hindsight-control-plane/src/app/api/banks/route.ts b/hindsight-control-plane/src/app/api/banks/route.ts index 41309ae2..38d904d5 100644 --- a/hindsight-control-plane/src/app/api/banks/route.ts +++ b/hindsight-control-plane/src/app/api/banks/route.ts @@ -4,6 +4,13 @@ import { sdk, lowLevelClient } from "@/lib/hindsight-client"; export async function GET() { try { const response = await sdk.listBanks({ client: lowLevelClient }); + + // Check if the response has an error or no data + if (response.error || !response.data) { + console.error("API error:", response.error); + return NextResponse.json({ error: "Failed to fetch banks from API" }, { status: 500 }); + } + return NextResponse.json(response.data, { status: 200 }); } catch (error) { console.error("Error fetching banks:", error); @@ -26,7 +33,8 @@ export async function POST(request: Request) { body: {}, }); - return NextResponse.json(response.data, { status: 201 }); + const serializedData = JSON.parse(JSON.stringify(response.data)); + return NextResponse.json(serializedData, { status: 201 }); } catch (error) { console.error("Error creating bank:", error); return NextResponse.json({ error: "Failed to create bank" }, { status: 500 }); diff --git a/hindsight-control-plane/src/components/data-view.tsx b/hindsight-control-plane/src/components/data-view.tsx index 6baaf3b6..5cdfb3a5 100644 --- a/hindsight-control-plane/src/components/data-view.tsx +++ b/hindsight-control-plane/src/components/data-view.tsx @@ -54,7 +54,7 @@ export function DataView({ factType }: DataViewProps) { // Graph controls state const [showLabels, setShowLabels] = useState(true); - const [maxNodes, setMaxNodes] = useState(50); + const [maxNodes, setMaxNodes] = useState(undefined); const [showControlPanel, setShowControlPanel] = useState(true); const [visibleLinkTypes, setVisibleLinkTypes] = useState>( new Set(["semantic", "temporal", "entity", "causal"]) @@ -228,6 +228,19 @@ export function DataView({ factType }: DataViewProps) { } }, [factType, currentBank]); + // Enforce 50 node limit to prevent UI instability, default to 20 or max whichever is smaller + useEffect(() => { + if (data && maxNodes === undefined) { + if (graph2DData.nodes.length > 50) { + // Always set maxNodes to 20 when we have >50 nodes (never leave as undefined) + setMaxNodes(20); + } else if (graph2DData.nodes.length > 20) { + setMaxNodes(20); + } + // If ≤20 nodes, leave maxNodes undefined to show all + } + }, [data, graph2DData.nodes.length, maxNodes]); + return (
{loading ? ( @@ -467,22 +480,41 @@ export function DataView({ factType }: DataViewProps) {
- {maxNodes ?? "All"} / {graph2DData.nodes.length} + {graph2DData.nodes.length > 50 + ? `${maxNodes ?? 50} / ${graph2DData.nodes.length}` + : `${maxNodes ?? "All"} / ${graph2DData.nodes.length}`}
50 + ? maxNodes || 20 + : maxNodes || Math.min(graph2DData.nodes.length, 20), + ]} min={10} - max={Math.max(graph2DData.nodes.length, 10)} + max={Math.min(Math.max(graph2DData.nodes.length, 10), 50)} step={10} - onValueChange={([v]) => - setMaxNodes(v >= graph2DData.nodes.length ? undefined : v) - } + onValueChange={([v]) => { + const effectiveMax = Math.min(graph2DData.nodes.length, 50); + // If we have >50 nodes, never allow "All" (undefined), cap at 50 + if (graph2DData.nodes.length > 50) { + setMaxNodes(v); + } else { + // Original behavior for ≤50 nodes: allow "All" when slider reaches max + setMaxNodes(v >= effectiveMax ? undefined : v); + } + }} className="w-full" />

All links between visible nodes are shown. + {graph2DData.nodes.length > 50 && ( + + ⚠️ Limited to 50 nodes for performance. Total:{" "} + {graph2DData.nodes.length} + + )}

diff --git a/hindsight-control-plane/src/components/graph-2d.tsx b/hindsight-control-plane/src/components/graph-2d.tsx index 98f7ecea..93bab0d5 100644 --- a/hindsight-control-plane/src/components/graph-2d.tsx +++ b/hindsight-control-plane/src/components/graph-2d.tsx @@ -1,7 +1,12 @@ "use client"; import { useRef, useEffect, useState, useMemo } from "react"; -import cytoscape, { Core, NodeSingular } from "cytoscape"; +import cytoscape from "cytoscape"; + +import fcose from "cytoscape-fcose"; + +// Register the fcose extension +cytoscape.use(fcose); // Hook to detect dark mode function useIsDarkMode() { @@ -72,14 +77,10 @@ export interface Graph2DProps { // Brand colors const BRAND_PRIMARY = "#0074d9"; -const BRAND_TEAL = "#009296"; const LINK_SEMANTIC = "#0074d9"; // Primary blue for semantic -const LINK_TEMPORAL = "#009296"; // Teal for temporal -const LINK_ENTITY = "#f59e0b"; // Amber for entity const DEFAULT_NODE_COLOR = BRAND_PRIMARY; const DEFAULT_LINK_COLOR = LINK_SEMANTIC; -const DEFAULT_NODE_SIZE = 20; const DEFAULT_LINK_WIDTH = 1; // ============================================================================ @@ -98,12 +99,16 @@ export function Graph2D({ linkWidthFn, maxNodes, }: Graph2DProps) { - const containerRef = useRef(null); - const cyRef = useRef(null); - const [hoveredNode, setHoveredNode] = useState(null); + const [containerDiv, setContainerDiv] = useState(null); + const cyRef = useRef(null); + const isInitializingRef = useRef(false); + const lastDataSignatureRef = useRef(""); + const [_hoveredNode, setHoveredNode] = useState(null); const [hoveredLink, setHoveredLink] = useState(null); const [linkTooltipPos, setLinkTooltipPos] = useState<{ x: number; y: number } | null>(null); const [isLoading, setIsLoading] = useState(true); + const [isMounted, setIsMounted] = useState(false); + const [isFocusMode, setIsFocusMode] = useState(false); const isDarkMode = useIsDarkMode(); // Use refs to store callbacks and data to prevent re-renders from resetting the graph @@ -112,11 +117,13 @@ export function Graph2D({ const fullDataRef = useRef(data); const nodeColorFnRef = useRef(nodeColorFn); const linkColorFnRef = useRef(linkColorFn); + const isFocusModeRef = useRef(isFocusMode); onNodeClickRef.current = onNodeClick; onNodeHoverRef.current = onNodeHover; fullDataRef.current = data; nodeColorFnRef.current = nodeColorFn; linkColorFnRef.current = linkColorFn; + isFocusModeRef.current = isFocusMode; // Transform and limit data - only limit nodes, show ALL links between visible nodes const graphData = useMemo(() => { @@ -134,17 +141,38 @@ export function Graph2D({ return { nodes, links }; }, [data, maxNodes]); + // Track mounting state + useEffect(() => { + setIsMounted(true); + return () => setIsMounted(false); + }, []); + // Convert to Cytoscape format const cyElements = useMemo(() => { - const nodes = graphData.nodes.map((node) => ({ - data: { - id: node.id, - label: node.label || node.id.substring(0, 8), - color: nodeColorFn ? nodeColorFn(node) : node.color || DEFAULT_NODE_COLOR, - size: nodeSizeFn ? nodeSizeFn(node) : node.size || DEFAULT_NODE_SIZE, - originalNode: node, - }, - })); + // Calculate node importance based on connections + const nodeConnections = new Map(); + graphData.links.forEach((link) => { + nodeConnections.set(link.source, (nodeConnections.get(link.source) || 0) + 1); + nodeConnections.set(link.target, (nodeConnections.get(link.target) || 0) + 1); + }); + + const nodes = graphData.nodes.map((node) => { + const connections = nodeConnections.get(node.id) || 0; + const dynamicSize = nodeSizeFn + ? nodeSizeFn(node) + : Math.max(16, Math.min(40, 16 + connections * 4)); // Smaller, more subtle sizing + + return { + data: { + id: node.id, + label: node.label || node.id.substring(0, 8), + color: nodeColorFn ? nodeColorFn(node) : node.color || DEFAULT_NODE_COLOR, + size: node.size || dynamicSize, + originalNode: node, + connections: connections, + }, + }; + }); const edges = graphData.links.map((link, idx) => ({ data: { @@ -163,328 +191,392 @@ export function Graph2D({ return [...nodes, ...edges]; }, [graphData, nodeColorFn, nodeSizeFn, linkColorFn, linkWidthFn]); + // Create data signature to prevent double initialization + const dataSignature = useMemo(() => { + return JSON.stringify({ + nodeCount: graphData.nodes.length, + linkCount: graphData.links.length, + nodeIds: graphData.nodes + .map((n) => n.id) + .sort() + .join(","), + showLabels, + isDarkMode, + maxNodes, + }); + }, [graphData.nodes, graphData.links, showLabels, isDarkMode, maxNodes]); + // Initialize Cytoscape useEffect(() => { - if (!containerRef.current) return; + let isCancelled = false; - // Handle empty data case - if (cyElements.length === 0) { - setIsLoading(false); - return; - } + // Small delay to ensure container is mounted + const timeout = setTimeout(() => { + if (isCancelled || !isMounted || !containerDiv || isInitializingRef.current) return; - setIsLoading(true); - - // Theme-aware colors - const textColor = isDarkMode ? "#ffffff" : "#1f2937"; - const textBgColor = isDarkMode ? "rgba(0,0,0,0.8)" : "rgba(255,255,255,0.9)"; - const borderColor = isDarkMode ? "#ffffff" : "#374151"; - - const cy = cytoscape({ - container: containerRef.current, - elements: cyElements, - style: [ - { - selector: "node", - style: { - "background-fill": "radial-gradient", - "background-gradient-stop-colors": ["#0074d9", "#005bb5"], - "background-gradient-stop-positions": ["0%", "100%"], - width: "data(size)", - height: "data(size)", - label: showLabels ? "data(label)" : "", - color: textColor, - "text-valign": "bottom", - "text-halign": "center", - "font-size": "8px", - "font-weight": 500, - "text-margin-y": 3, - "text-wrap": "wrap", - "text-max-width": "80px", - "text-background-color": textBgColor, - "text-background-opacity": 0.9, - "text-background-padding": "2px", - "text-background-shape": "roundrectangle", - "border-width": 0, - "z-index": 0, - }, - }, - { - selector: "node:selected", - style: { - "border-width": 3, - "border-color": "#0074d9", - "border-opacity": 1, - }, - }, - { - selector: "node:active", - style: { - "overlay-opacity": 0, - }, - }, - { - selector: "edge", - style: { - width: "data(width)", - "line-color": "data(color)", - "target-arrow-color": "data(color)", - "curve-style": "bezier", - opacity: isDarkMode ? 0.5 : 0.6, - "z-index": 1, - }, - }, - { - selector: "edge:selected", - style: { - opacity: 1, - width: 3, - }, - }, - // Dimmed state for non-selected elements - { - selector: ".dimmed", - style: { - opacity: 0.15, - }, - }, - // Highlighted state for selected node and neighbors - { - selector: "node.highlighted", - style: { - opacity: 1, - "border-width": 3, - "border-color": "#0074d9", - "border-opacity": 1, - }, - }, - { - selector: "edge.highlighted", - style: { - opacity: 0.9, - width: 2, - }, - }, - ], - layout: { - name: "cose", - animate: false, - randomize: true, - nodeRepulsion: () => 100000, - idealEdgeLength: () => 300, - edgeElasticity: () => 20, - nestingFactor: 0.1, - gravity: 0.01, - numIter: 2500, - coolingFactor: 0.95, - minTemp: 1.0, - nodeOverlap: 20, - nodeDimensionsIncludeLabels: true, - padding: 50, - } as any, - minZoom: 0.1, - maxZoom: 5, - wheelSensitivity: 0.3, - }); + // Check if data has actually changed to prevent double initialization + if (lastDataSignatureRef.current === dataSignature) { + console.log("Data signature unchanged, skipping graph initialization"); + return; + } - cyRef.current = cy; + // Additional validation - check if element has dimensions + const rect = containerDiv.getBoundingClientRect(); + if (rect.width === 0 || rect.height === 0) { + console.warn("Container has no dimensions, skipping cytoscape initialization"); + setIsLoading(false); + return; + } - // Event handlers - cy.on("tap", "node", (evt) => { - const node = evt.target as NodeSingular; - const originalNode = node.data("originalNode") as GraphNode; - if (onNodeClickRef.current && originalNode) { - onNodeClickRef.current(originalNode); + // Handle empty data case + if (cyElements.length === 0) { + setIsLoading(false); + return; } - // Find ALL connected nodes from full data (not just visible ones) - const fullData = fullDataRef.current; - const clickedNodeId = originalNode.id; - - // Find all links connected to this node from full data - const connectedLinks = fullData.links.filter( - (l) => l.source === clickedNodeId || l.target === clickedNodeId - ); - - // Find all connected node IDs - const connectedNodeIds = new Set(); - connectedLinks.forEach((l) => { - connectedNodeIds.add(l.source); - connectedNodeIds.add(l.target); - }); - - // Add any missing nodes to the graph - const existingNodeIds = new Set(cy.nodes().map((n) => n.id())); - const nodesToAdd: any[] = []; - const edgesToAdd: any[] = []; - - connectedNodeIds.forEach((nodeId) => { - if (!existingNodeIds.has(nodeId)) { - const nodeData = fullData.nodes.find((n) => n.id === nodeId); - if (nodeData) { - nodesToAdd.push({ - group: "nodes", - data: { - id: nodeData.id, - label: nodeData.label || nodeData.id.substring(0, 8), - color: nodeColorFnRef.current - ? nodeColorFnRef.current(nodeData) - : nodeData.color || DEFAULT_NODE_COLOR, - size: nodeData.size || DEFAULT_NODE_SIZE, - originalNode: nodeData, - isTemporary: true, // Mark as temporarily added - }, - }); - } + // Check if we already have a graph with the same data + if (cyRef.current && !cyRef.current.destroyed()) { + const currentNodes = cyRef.current.nodes().length; + const currentEdges = cyRef.current.edges().length; + const newNodes = cyElements.filter((el) => !(el.data as any).source).length; + const newEdges = cyElements.filter((el) => (el.data as any).source).length; + + // If the element counts are the same, just update styles and skip reinitialization + if (currentNodes === newNodes && currentEdges === newEdges) { + console.log("Graph already initialized with same data, skipping reinitialization"); + setIsLoading(false); + return; } - }); - - // Add missing edges - const existingEdgeIds = new Set( - cy.edges().map((e) => `${e.data("source")}-${e.data("target")}`) - ); - connectedLinks.forEach((link, idx) => { - const edgeKey = `${link.source}-${link.target}`; - const reverseKey = `${link.target}-${link.source}`; - if (!existingEdgeIds.has(edgeKey) && !existingEdgeIds.has(reverseKey)) { - edgesToAdd.push({ - group: "edges", - data: { - id: `temp-edge-${idx}-${Date.now()}`, - source: link.source, - target: link.target, - color: linkColorFnRef.current - ? linkColorFnRef.current(link) - : link.color || DEFAULT_LINK_COLOR, - width: link.width || DEFAULT_LINK_WIDTH, - type: link.type, - isTemporary: true, + + // Clean up existing graph before creating new one + console.log("Data changed, destroying existing graph"); + cyRef.current.destroy(); + cyRef.current = null; + } + + setIsLoading(true); + isInitializingRef.current = true; + + // Theme-aware colors + const textColor = isDarkMode ? "#ffffff" : "#1f2937"; + const textBgColor = isDarkMode ? "rgba(0,0,0,0.8)" : "rgba(255,255,255,0.9)"; + + try { + console.log("Initializing cytoscape with container:", containerDiv); + console.log("Elements count:", cyElements.length); + console.log("Sample elements:", cyElements.slice(0, 2)); + + // Try minimal initialization first + const cy = cytoscape({ + container: containerDiv, + elements: [], + // Disable edge selection to prevent gray border on click + selectionType: "single", + userZoomingEnabled: true, + userPanningEnabled: true, + boxSelectionEnabled: false, + // Disable automatic layout on initialization + layout: { name: "preset" }, + style: [ + { + selector: "node", + style: { + "background-color": "data(color)", + width: "data(size)", + height: "data(size)", + label: showLabels ? "data(label)" : "", + color: textColor, + "text-valign": "bottom", + "text-halign": "center", + "font-size": "8px", + "font-weight": 500, + "text-margin-y": 3, + "text-wrap": "wrap", + "text-max-width": "80px", + "text-background-color": textBgColor, + "text-background-opacity": 0.9, + "text-background-padding": "2px", + "text-background-shape": "roundrectangle", + "border-width": 1, + "border-color": isDarkMode ? "#ffffff20" : "#00000020", + "border-opacity": 0.3, + }, + }, + { + selector: "node:selected", + style: { + "border-width": 3, + "border-color": "#0074d9", + "border-opacity": 1, + }, + }, + { + selector: "edge", + style: { + width: "data(width)", + "line-color": "data(color)", + "target-arrow-color": "data(color)", + "target-arrow-shape": "triangle", + "target-arrow-size": 6, + "curve-style": "bezier", + opacity: isDarkMode ? 0.6 : 0.7, + }, + }, + // Focus mode styles + { + selector: ".dimmed", + style: { + opacity: 0.2, + }, }, - }); + { + selector: ".focused", + style: { + "border-width": 4, + "border-color": "#ff6b35", + "border-opacity": 1, + "z-index": 999, + }, + }, + { + selector: ".connected", + style: { + "border-width": 2, + "border-color": "#0074d9", + "border-opacity": 0.8, + opacity: 1, + }, + }, + { + selector: "edge.connection", + style: { + width: 2, + opacity: 1, + "z-index": 100, + }, + }, + { + selector: "edge.connection:hover", + style: { + width: 3, + opacity: 1, + "z-index": 200, + }, + }, + // Disable edge selection styling + { + selector: "edge:selected", + style: { + "overlay-opacity": 0, + "overlay-color": "transparent", + "overlay-padding": 0, + }, + }, + ], + }); + + cyRef.current = cy; + + console.log("Cytoscape initialized successfully"); + + // Add elements after initialization + if (cyElements.length > 0) { + console.log("Adding elements to cytoscape"); + cy.add(cyElements); + cy.layout({ + name: "fcose", + quality: "default", + randomize: false, + animate: true, + animationDuration: 1500, + // Separation settings - increase to spread nodes more + nodeSeparation: 200, + idealEdgeLength: () => 250, + edgeElasticity: () => 0.05, + nestingFactor: 0.05, + gravity: 0.05, // Reduced gravity spreads nodes more + numIter: 2500, + // Overlap prevention + nodeOverlap: 30, + avoidOverlap: true, + nodeDimensionsIncludeLabels: true, + // Layout bounds - reduce padding to use more space + padding: 20, + boundingBox: undefined, + // Tiling - increase spacing between disconnected components + tile: true, + tilingPaddingVertical: 30, + tilingPaddingHorizontal: 30, + // Force more spread + uniformNodeDimensions: false, + packComponents: false, // Don't pack components tightly + }).run(); + + // Fit to viewport + cy.fit(); } - }); - - // Add new elements to graph - if (nodesToAdd.length > 0 || edgesToAdd.length > 0) { - cy.add([...nodesToAdd, ...edgesToAdd]); - - // Position new nodes near the clicked node - const clickedPos = node.position(); - cy.nodes("[?isTemporary]").forEach((n, i) => { - const angle = (2 * Math.PI * i) / nodesToAdd.length; - const radius = 150; - n.position({ - x: clickedPos.x + radius * Math.cos(angle), - y: clickedPos.y + radius * Math.sin(angle), - }); + + // Add basic interactions + cy.on("tap", "node", (evt: any) => { + const node = evt.target as cytoscape.NodeSingular; + const originalNode = node.data("originalNode") as GraphNode; + if (onNodeClickRef.current && originalNode) { + onNodeClickRef.current(originalNode); + } }); - } - // Get all connected elements (including newly added) - const neighborhood = node.neighborhood().add(node); + cy.on("mouseover", "node", (evt: any) => { + const node = evt.target as cytoscape.NodeSingular; + const originalNode = node.data("originalNode") as GraphNode; + setHoveredNode(originalNode); + if (onNodeHoverRef.current && originalNode) { + onNodeHoverRef.current(originalNode); + } + if (containerDiv) containerDiv.style.cursor = "pointer"; + }); - // Dim all elements first - cy.elements().addClass("dimmed"); + cy.on("mouseout", "node", () => { + setHoveredNode(null); + if (onNodeHoverRef.current) { + onNodeHoverRef.current(null); + } + if (containerDiv) containerDiv.style.cursor = "default"; + }); - // Highlight the neighborhood - neighborhood.removeClass("dimmed"); - neighborhood.addClass("highlighted"); + // Edge hover handlers - only work in focus mode and on highlighted edges + cy.on("mouseover", "edge", (evt: any) => { + const edge = evt.target; - // Center on the neighborhood without changing positions - cy.animate( - { - fit: { eles: neighborhood, padding: 50 }, - }, - { duration: 400 } - ); - }); + // Only allow interaction if we're in focus mode and edge is highlighted + if (!isFocusModeRef.current || !edge.hasClass("connection")) { + return; + } - // Click on background to reset - cy.on("tap", (evt) => { - if (evt.target === cy) { - // Remove temporary nodes and edges - cy.elements("[?isTemporary]").remove(); - - cy.elements().removeClass("dimmed highlighted"); - cy.animate( - { - fit: { eles: cy.elements(), padding: 50 }, - }, - { duration: 400 } - ); - } - }); + const originalLink = edge.data("originalLink") as GraphLink; + if (originalLink) { + setHoveredLink(originalLink); + // Get position for tooltip + const renderedPos = edge.renderedMidpoint(); + setLinkTooltipPos({ x: renderedPos.x, y: renderedPos.y }); + } + }); - cy.on("mouseover", "node", (evt) => { - const node = evt.target as NodeSingular; - const originalNode = node.data("originalNode") as GraphNode; - setHoveredNode(originalNode); - if (onNodeHoverRef.current && originalNode) { - onNodeHoverRef.current(originalNode); - } - containerRef.current!.style.cursor = "pointer"; - }); + cy.on("mouseout", "edge", (evt: any) => { + const edge = evt.target; - cy.on("mouseout", "node", () => { - setHoveredNode(null); - if (onNodeHoverRef.current) { - onNodeHoverRef.current(null); - } - containerRef.current!.style.cursor = "default"; - }); + // Only clear hover state if we were actually hovering a highlighted edge + if (!isFocusModeRef.current || !edge.hasClass("connection")) { + return; + } - // Edge hover handlers - cy.on("mouseover", "edge", (evt) => { - const edge = evt.target; - const originalLink = edge.data("originalLink") as GraphLink; - if (originalLink) { - setHoveredLink(originalLink); - // Get position for tooltip - const renderedPos = edge.renderedMidpoint(); - setLinkTooltipPos({ x: renderedPos.x, y: renderedPos.y }); - } - containerRef.current!.style.cursor = "pointer"; - }); + setHoveredLink(null); + setLinkTooltipPos(null); + }); - cy.on("mouseout", "edge", () => { - setHoveredLink(null); - setLinkTooltipPos(null); - containerRef.current!.style.cursor = "default"; - }); + // Prevent edge selection to avoid gray border on click + cy.on("select", "edge", (evt: any) => { + evt.target.unselect(); + }); + + // Double-click to focus on node and its connections + cy.on("dblclick", "node", (evt: any) => { + const focusedNode = evt.target as cytoscape.NodeSingular; + const focusedNodeId = focusedNode.id(); - // Run layout - cy.layout({ - name: "cose", - animate: false, - randomize: true, - nodeRepulsion: () => 100000, - idealEdgeLength: () => 300, - edgeElasticity: () => 20, - nestingFactor: 0.1, - gravity: 0.01, - numIter: 2500, - coolingFactor: 0.95, - minTemp: 1.0, - nodeOverlap: 20, - nodeDimensionsIncludeLabels: true, - padding: 50, - } as any).run(); - - // Fit to viewport - cy.fit(undefined, 50); - setIsLoading(false); + console.log("Double-clicked node:", focusedNodeId); + + // Enter focus mode + setIsFocusMode(true); + + // Clear any existing focus classes + cy.elements().removeClass("dimmed focused connected connection"); + + // Get all connected nodes and edges + const connectedElements = focusedNode.neighborhood(); + const connectedNodes = connectedElements.nodes(); + const connectedEdges = connectedElements.edges(); + + // Apply styling classes + cy.elements().addClass("dimmed"); // Dim everything first + focusedNode.removeClass("dimmed").addClass("focused"); // Highlight the focused node + connectedNodes.removeClass("dimmed").addClass("connected"); // Highlight connected nodes + connectedEdges.removeClass("dimmed").addClass("connection"); // Highlight connecting edges + + // Create a collection of all relevant elements for positioning + const relevantElements = focusedNode.union(connectedElements); + + // Reorient the graph to focus on this subgraph + cy.animate( + { + fit: { + eles: relevantElements, + padding: 100, + }, + center: { + eles: focusedNode, + }, + }, + { + duration: 800, + easing: "ease-out-cubic", + } + ); + }); + + // Click on background to reset focus + cy.on("tap", (evt: any) => { + if (evt.target === cy) { + console.log("Clicked background - resetting focus"); + + // Exit focus mode + setIsFocusMode(false); + + // Remove all focus classes + cy.elements().removeClass("dimmed focused connected connection"); + + // Zoom out to show all elements + cy.animate( + { + fit: { + eles: cy.elements(), + padding: 50, + }, + }, + { + duration: 600, + easing: "ease-out", + } + ); + } + }); + + setIsLoading(false); + isInitializingRef.current = false; + lastDataSignatureRef.current = dataSignature; + } catch (error) { + console.error("Error initializing cytoscape:", error); + setIsLoading(false); + isInitializingRef.current = false; + } + }, 100); // 100ms delay return () => { - cy.destroy(); + isCancelled = true; + clearTimeout(timeout); + isInitializingRef.current = false; + if (cyRef.current) { + cyRef.current.destroy(); + cyRef.current = null; + } }; - }, [cyElements, showLabels, isDarkMode]); + }, [dataSignature, isMounted, containerDiv]); // Handle resize useEffect(() => { const handleResize = () => { if (cyRef.current) { cyRef.current.resize(); - cyRef.current.fit(undefined, 50); + cyRef.current.fit(undefined, 80); } }; @@ -508,17 +600,19 @@ export function Graph2D({ )} {/* Cytoscape container */} -
+ {isMounted && ( +
+ )} {/* Empty state */} {!isLoading && graphData.nodes.length === 0 && ( @@ -571,7 +665,7 @@ export function Graph2D({ {/* Controls hint */}
- Drag to pan • Scroll to zoom • Click node to focus + Drag to pan • Scroll to zoom • Double-click node to focus • Click background to reset
); diff --git a/hindsight-control-plane/src/components/ui/slider.tsx b/hindsight-control-plane/src/components/ui/slider.tsx index 1b070709..4d10b3e1 100644 --- a/hindsight-control-plane/src/components/ui/slider.tsx +++ b/hindsight-control-plane/src/components/ui/slider.tsx @@ -14,7 +14,7 @@ const Slider = React.forwardRef< className={cn("relative flex w-full touch-none select-none items-center", className)} {...props} > - + diff --git a/hindsight-control-plane/tsconfig.json b/hindsight-control-plane/tsconfig.json index 877b650f..13a445fa 100644 --- a/hindsight-control-plane/tsconfig.json +++ b/hindsight-control-plane/tsconfig.json @@ -33,7 +33,8 @@ "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", - ".next/dev/types/**/*.ts" + ".next/dev/types/**/*.ts", + "types/**/*.d.ts" ], "exclude": [ "node_modules" diff --git a/hindsight-control-plane/types/cytoscape-fcose.d.ts b/hindsight-control-plane/types/cytoscape-fcose.d.ts new file mode 100644 index 00000000..c59f2687 --- /dev/null +++ b/hindsight-control-plane/types/cytoscape-fcose.d.ts @@ -0,0 +1,126 @@ +declare module 'cytoscape-fcose' { + import { Ext } from 'cytoscape'; + + interface FcoseLayoutOptions { + name: 'fcose'; + quality?: 'default' | 'draft' | 'proof'; + randomize?: boolean; + animate?: boolean; + animationDuration?: number; + animationEasing?: string; + fit?: boolean; + padding?: number; + nodeDimensionsIncludeLabels?: boolean; + uniformNodeDimensions?: boolean; + packComponents?: boolean; + step?: 'transformed' | 'untransformed' | 'all'; + samplingType?: boolean; + sampleSize?: number; + nodeSeparation?: number; + piTol?: number; + nodeRepulsion?: number; + idealEdgeLength?: number; + edgeElasticity?: number; + nestingFactor?: number; + gravity?: number; + numIter?: number; + initialTemp?: number; + coolingFactor?: number; + minTemp?: number; + fixedNodeConstraint?: any[]; + alignmentConstraint?: any[]; + relativePlacementConstraint?: any[]; + } + + const fcose: Ext; + export = fcose; +} + +// Extend cytoscape module declarations to include missing CSS properties and methods +declare module 'cytoscape' { + // Add the main cytoscape function + interface CytoscapeOptions { + container?: HTMLElement; + elements?: any[]; + style?: any[]; + layout?: any; + selectionType?: string; + userZoomingEnabled?: boolean; + userPanningEnabled?: boolean; + boxSelectionEnabled?: boolean; + [key: string]: any; + } + + // Add Core interface + interface Core { + add(elements: any): void; + layout(options: any): any; + on(event: string, selector: string, handler: Function): void; + on(event: string, handler: Function): void; + off(event: string, handler?: Function): void; + removeListener(event: string, handler?: Function): void; + destroy(): void; + nodes(): any; + edges(): any; + elements(): any; + getElementById(id: string): any; + fit(): void; + zoom(): number; + zoom(level: number): void; + pan(): { x: number; y: number }; + pan(position: { x: number; y: number }): void; + resize(): void; + animate(options: any, timing?: any): any; + } + + // Define cytoscape as both callable function and object with properties + interface CytoscapeStatic { + (options: CytoscapeOptions): Core; + use(extension: any): void; + } + + // Make cytoscape the default export + const cytoscape: CytoscapeStatic; + export = cytoscape; + + namespace Css { + interface Node { + 'target-arrow-color'?: string; + 'target-arrow-shape'?: string; + 'target-arrow-size'?: number; + 'curve-style'?: string; + 'text-valign'?: string; + 'text-halign'?: string; + 'font-size'?: string; + 'font-weight'?: string | number; + 'text-margin-y'?: number; + 'text-wrap'?: string; + 'text-max-width'?: string; + 'text-background-color'?: string; + 'text-background-opacity'?: number; + 'text-background-padding'?: string; + 'text-background-shape'?: string; + 'border-width'?: number; + 'border-color'?: string; + 'border-opacity'?: number; + 'background-color'?: string; + 'line-color'?: string; + 'overlay-opacity'?: number; + 'overlay-color'?: string; + 'overlay-padding'?: number; + 'z-index'?: number; + } + + interface Edge { + 'target-arrow-color'?: string; + 'target-arrow-shape'?: string; + 'target-arrow-size'?: number; + 'curve-style'?: string; + 'line-color'?: string; + 'overlay-opacity'?: number; + 'overlay-color'?: string; + 'overlay-padding'?: number; + 'z-index'?: number; + } + } +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index c1e93547..98d234ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ }, "hindsight-clients/typescript": { "name": "@vectorize-io/hindsight-client", - "version": "0.1.14", + "version": "0.1.16", "license": "MIT", "devDependencies": { "@hey-api/openapi-ts": "^0.88.0", @@ -26,7 +26,7 @@ }, "hindsight-control-plane": { "name": "@vectorize-io/hindsight-control-plane", - "version": "0.1.14", + "version": "0.1.16", "license": "ISC", "dependencies": { "@radix-ui/react-alert-dialog": "^1.1.15", @@ -49,6 +49,7 @@ "clsx": "^2.1.1", "cmdk": "^1.1.1", "cytoscape": "^3.33.1", + "cytoscape-fcose": "^2.2.0", "eslint": "^9.39.1", "eslint-config-next": "^16.0.1", "lucide-react": "^0.553.0", @@ -9917,6 +9918,8 @@ }, "node_modules/cytoscape-fcose": { "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cytoscape-fcose/-/cytoscape-fcose-2.2.0.tgz", + "integrity": "sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==", "license": "MIT", "dependencies": { "cose-base": "^2.2.0"