fix some visualize problems in detail page and add show ghost nodes feature

scripts/static/js/graph.js

@@ -56,6 +56,7 @@ export function updateGraphNodeSelection() {
 }
 
 export function getNodeColor(d) {
+    if (d.island === -1) return "#ffffff";
     if (d.island !== undefined) return d3.schemeCategory10[d.island % 10];
     return getComputedStyle(document.documentElement)
         .getPropertyValue('--node-default').trim() || "#fff";
@@ -110,6 +111,44 @@ export function selectProgram(programId) {
     updateGraphEdgeSelection(); // update edge highlight on selection
 }
 
+// Add toggle for showing negative island programs in branching view
+(function() {
+    window.addEventListener('DOMContentLoaded', function() {
+        const branchingDiv = document.getElementById('view-branching');
+        if (!branchingDiv) return;
+
+        // Check if toggle already exists
+        let toggleDiv = document.getElementById('branching-negative-island-toggle');
+        if (!toggleDiv) {
+            toggleDiv = document.createElement('div');
+            toggleDiv.id = 'branching-negative-island-toggle';
+            toggleDiv.style = 'display:flex;align-items:center;gap:0.7em;margin-left:3em;';
+            toggleDiv.innerHTML = `
+            <label class="toggle-switch">
+                <input type="checkbox" id="show-negative-island-toggle-graph">
+                <span class="toggle-slider"></span>
+            </label>
+            <span style="font-weight:500;font-size:1.08em;color:#fff;background:#0008;padding:0.2em 0.5em;border-radius:4px;">Show Ghost Nodes</span>
+            `;
+            branchingDiv.insertBefore(toggleDiv, branchingDiv.firstChild);
+
+            // Add event listener for the toggle
+            const toggle = document.getElementById('show-negative-island-toggle-graph');
+            toggle.addEventListener('change', function() {
+                // Re-render the graph when toggle changes
+                let edges = [];
+                if (typeof lastDataStr === 'string') {
+                    try {
+                        const parsed = JSON.parse(lastDataStr);
+                        edges = parsed.edges || [];
+                    } catch {}
+                }
+                renderGraph({ nodes: allNodeData, edges: edges });
+            });
+        }
+    });
+})();
+
 let svg = null;
 let g = null;
 let simulation = null; // Keep simulation alive
@@ -185,6 +224,18 @@ function applyDragHandlersToAllNodes() {
 }
 
 function renderGraph(data, options = {}) {
+    // Filter nodes based on show negative island toggle
+    const showNegativeIsland = document.getElementById('show-negative-island-toggle-graph')?.checked;
+    let filteredData = { nodes: data.nodes, edges: data.edges };
+    if (!showNegativeIsland) {
+        filteredData.nodes = data.nodes.filter(n => n.island !== -1);
+        // Also filter edges to only include edges between filtered nodes
+        const filteredNodeIds = new Set(filteredData.nodes.map(n => n.id));
+        filteredData.edges = data.edges.filter(e => 
+            filteredNodeIds.has(e.source.id || e.source) && filteredNodeIds.has(e.target.id || e.target)
+        );
+    }
+
     const { svg: svgEl, g: gEl } = ensureGraphSvg();
     svg = svgEl;
     g = gEl;
@@ -206,34 +257,34 @@ function renderGraph(data, options = {}) {
 
     // Keep simulation alive and update nodes/links
     if (!simulation) {
-        simulation = d3.forceSimulation(data.nodes)
-            .force("link", d3.forceLink(data.edges).id(d => d.id).distance(80))
+        simulation = d3.forceSimulation(filteredData.nodes)
+            .force("link", d3.forceLink(filteredData.edges).id(d => d.id).distance(80))
             .force("charge", d3.forceManyBody().strength(-200))
             .force("center", d3.forceCenter(width / 2, height / 2));
     } else {
-        simulation.nodes(data.nodes);
-        simulation.force("link").links(data.edges);
+        simulation.nodes(filteredData.nodes);
+        simulation.force("link").links(filteredData.edges);
         simulation.alpha(0.7).restart();
     }
 
     const link = g.append("g")
         .attr("stroke", "#999")
         .attr("stroke-opacity", 0.6)
         .selectAll("line")
-        .data(data.edges)
+        .data(filteredData.edges)
         .enter().append("line")
         .attr("stroke-width", 2);
 
     const metric = getSelectedMetric();
     const highlightFilter = document.getElementById('highlight-select').value;
-    const highlightNodes = getHighlightNodes(data.nodes, highlightFilter, metric);
+    const highlightNodes = getHighlightNodes(filteredData.nodes, highlightFilter, metric);
     const highlightIds = new Set(highlightNodes.map(n => n.id));
 
     const node = g.append("g")
         .attr("stroke", getComputedStyle(document.documentElement).getPropertyValue('--node-stroke').trim() || "#fff")
         .attr("stroke-width", 1.5)
         .selectAll("circle")
-        .data(data.nodes)
+        .data(filteredData.nodes)
         .enter().append("circle")
         .attr("r", d => getNodeRadius(d))
         .attr("fill", d => getNodeColor(d))
@@ -399,8 +450,17 @@ export function animateGraphNodeAttributes() {
     if (!g) return;
     const metric = getSelectedMetric();
     const filter = document.getElementById('highlight-select').value;
-    const highlightNodes = getHighlightNodes(allNodeData, filter, metric);
+
+    // Filter nodes based on show negative island toggle
+    const showNegativeIsland = document.getElementById('show-negative-island-toggle-graph')?.checked;
+    let dataToUse = allNodeData;
+    if (!showNegativeIsland) {
+        dataToUse = allNodeData.filter(n => n.island !== -1);
+    }
+
+    const highlightNodes = getHighlightNodes(dataToUse, filter, metric);
     const highlightIds = new Set(highlightNodes.map(n => n.id));
+
     g.selectAll('circle')
         .transition().duration(400)
         .attr('r', d => getNodeRadius(d))

scripts/static/js/performance.js

@@ -21,6 +21,22 @@ import { selectListNodeById } from './list.js';
             `;
             perfDiv.insertBefore(toggleDiv, perfDiv.firstChild);
         }
+
+        // Add toggle for showing island -1 programs
+        let negativeIslandToggleDiv = document.getElementById('perf-negative-island-toggle');
+        if (!negativeIslandToggleDiv) {
+            negativeIslandToggleDiv = document.createElement('div');
+            negativeIslandToggleDiv.id = 'perf-negative-island-toggle';
+            negativeIslandToggleDiv.style = 'display:flex;align-items:center;gap:0.7em;margin-left:3em;';
+            negativeIslandToggleDiv.innerHTML = `
+            <label class="toggle-switch">
+                <input type="checkbox" id="show-negative-island-toggle">
+                <span class="toggle-slider"></span>
+            </label>
+            <span style="font-weight:500;font-size:1.08em;">Show Ghost Nodes</span>
+            `;
+            perfDiv.insertBefore(negativeIslandToggleDiv, perfDiv.firstChild);
+        }
         function animatePerformanceGraphAttributes() {
             const svg = d3.select('#performance-graph');
             if (svg.empty()) return;
@@ -29,7 +45,11 @@ import { selectListNodeById } from './list.js';
             const metric = getSelectedMetric();
             const highlightFilter = document.getElementById('highlight-select').value;
             const showIslands = document.getElementById('show-islands-toggle')?.checked;
-            const nodes = allNodeData;
+            const showNegativeIsland = document.getElementById('show-negative-island-toggle')?.checked;
+            let nodes = allNodeData;
+            if (!showNegativeIsland) {
+                nodes = allNodeData.filter(n => n.island !== -1);
+            }
             const validNodes = nodes.filter(n => n.metrics && typeof n.metrics[metric] === 'number');
             const undefinedNodes = nodes.filter(n => !n.metrics || n.metrics[metric] == null || isNaN(n.metrics[metric]));
             let islands = [];
@@ -156,6 +176,11 @@ import { selectListNodeById } from './list.js';
         document.getElementById('show-islands-toggle').addEventListener('change', function() {
             updatePerformanceGraph(allNodeData);
         });
+
+        // Show negative island (-1) toggle event
+        document.getElementById('show-negative-island-toggle').addEventListener('change', function() {
+            updatePerformanceGraph(allNodeData);
+        });
         // Responsive resize
         window.addEventListener('resize', function() {
             if (typeof allNodeData !== 'undefined' && allNodeData.length && perfDiv.style.display !== 'none') {
@@ -301,6 +326,13 @@ function autoZoomPerformanceGraph(nodes, x, yScales, islands, graphHeight, margi
 }
 
 function updatePerformanceGraph(nodes, options = {}) {
+    // Filter nodes based on show negative island toggle
+    const showNegativeIsland = document.getElementById('show-negative-island-toggle')?.checked;
+    let filteredNodes = nodes;
+    if (!showNegativeIsland) {
+        filteredNodes = nodes.filter(n => n.island !== -1);
+    }
+
     // Get or create SVG
     if (!svg) {
         svg = d3.select('#performance-graph');
@@ -371,7 +403,7 @@ function updatePerformanceGraph(nodes, options = {}) {
                 .attr('stroke', function(d) {
                     // Use highlight color if highlighted, else default
                     const highlightFilter = document.getElementById('highlight-select').value;
-                    const highlightNodes = getHighlightNodes(nodes, highlightFilter, getSelectedMetric());
+                    const highlightNodes = getHighlightNodes(filteredNodes, highlightFilter, getSelectedMetric());
                     const highlightIds = new Set(highlightNodes.map(n => n.id));
                     return highlightIds.has(d.id) ? '#2196f3' : '#333';
                 })
@@ -389,16 +421,16 @@ function updatePerformanceGraph(nodes, options = {}) {
     const sidebarWidth = sidebarEl.offsetWidth || 400;
     const width = Math.max(windowWidth - sidebarWidth - padding, 400);
     const metric = getSelectedMetric();
-    const validNodes = nodes.filter(n => n.metrics && typeof n.metrics[metric] === 'number');
-    const undefinedNodes = nodes.filter(n => !n.metrics || n.metrics[metric] == null || isNaN(n.metrics[metric]));
+    const validNodes = filteredNodes.filter(n => n.metrics && typeof n.metrics[metric] === 'number');
+    const undefinedNodes = filteredNodes.filter(n => !n.metrics || n.metrics[metric] == null || isNaN(n.metrics[metric]));
     const showIslands = document.getElementById('show-islands-toggle')?.checked;
     let islands = [];
     if (showIslands) {
-        islands = Array.from(new Set(nodes.map(n => n.island))).sort((a,b)=>a-b);
+        islands = Array.from(new Set(filteredNodes.map(n => n.island))).sort((a,b)=>a-b);
     } else {
         islands = [null];
     }
-    const yExtent = d3.extent(nodes, d => d.generation);
+    const yExtent = d3.extent(filteredNodes, d => d.generation);
     const minGen = 0;
     const maxGen = yExtent[1];
     const margin = {top: 60, right: 40, bottom: 40, left: 60};
@@ -523,8 +555,8 @@ function updatePerformanceGraph(nodes, options = {}) {
         });
     }
     // Data join for edges
-    const nodeById = Object.fromEntries(nodes.map(n => [n.id, n]));
-    const edges = nodes.filter(n => n.parent_id && nodeById[n.parent_id]).map(n => ({ source: nodeById[n.parent_id], target: n }));
+    const nodeById = Object.fromEntries(filteredNodes.map(n => [n.id, n]));
+    const edges = filteredNodes.filter(n => n.parent_id && nodeById[n.parent_id]).map(n => ({ source: nodeById[n.parent_id], target: n }));
     // Remove all old edges before re-adding (fixes missing/incorrect edges after metric change)
     g.selectAll('line.performance-edge').remove();
     // Helper to get x/y for a node (handles NaN and valid nodes)

scripts/templates/program_page.html

@@ -61,12 +61,18 @@ <h2>Prompts:</h2>
         {% endmacro %}
         {#-- loop over every prompts --#}
         <div class="prompts">
-        {% for key, value in program_data.prompts.items() %}
+        {% if program_data.prompts is defined and program_data.prompts is mapping %}
+            {% for key, value in program_data.prompts.items() %}
+                <section>
+                <h3>{{ key|title }}</h3>
+                {{ display(value) }}
+                </section>
+            {% endfor %}
+        {% else %}
             <section>
-            <h3>{{ key|title }}</h3>
-            {{ display(value) }}
+            <p>No prompts available</p>
             </section>
-        {% endfor %}
+        {% endif %}
         </div>
     </ul>
     {% if artifacts_json %}

scripts/visualizer.py

@@ -38,34 +38,55 @@ def load_evolution_data(checkpoint_folder):
     nodes = []
     id_to_program = {}
     pids = set()
+
+    # Build a map of program_id -> island_idx from islands data
+    pid_to_island = {}
     for island_idx, id_list in enumerate(meta.get("islands", [])):
         for pid in id_list:
-            prog_path = os.path.join(programs_dir, f"{pid}.json")
-
-            # Keep track of PIDs and if one is double, append "-copyN" to the PID
-            if pid in pids:
-                base_pid = pid
-
-                # If base_pid already has a "-copyN" suffix, strip it
-                if "-copy" in base_pid:
-                    base_pid = base_pid.rsplit("-copy", 1)[0]
-
-                # Find the next available copy number
-                copy_num = 1
-                while f"{base_pid}-copy{copy_num}" in pids:
-                    copy_num += 1
-                pid = f"{base_pid}-copy{copy_num}"
-            pids.add(pid)
-
-            if os.path.exists(prog_path):
-                with open(prog_path) as pf:
-                    prog = json.load(pf)
-                prog["id"] = pid
-                prog["island"] = island_idx
-                nodes.append(prog)
-                id_to_program[pid] = prog
-            else:
-                logger.debug(f"Program file not found: {prog_path}")
+            pid_to_island[pid] = island_idx
+
+    # Start with programs from archive
+    archive = set(meta.get("archive", []))
+    to_load = list(archive)
+    loaded = set()
+
+    # Recursively load parent nodes even if they're not in archive
+    while to_load:
+        pid = to_load.pop(0)
+        if pid in loaded:
+            continue
+        loaded.add(pid)
+
+        prog_path = os.path.join(programs_dir, f"{pid}.json")
+
+        # Keep track of PIDs and if one is double, append "-copyN" to the PID
+        effective_pid = pid
+        if pid in pids:
+            base_pid = pid
+            if "-copy" in base_pid:
+                base_pid = base_pid.rsplit("-copy", 1)[0]
+            copy_num = 1
+            while f"{base_pid}-copy{copy_num}" in pids:
+                copy_num += 1
+            effective_pid = f"{base_pid}-copy{copy_num}"
+        pids.add(effective_pid)
+
+        if os.path.exists(prog_path):
+            with open(prog_path) as pf:
+                prog = json.load(pf)
+            prog["id"] = effective_pid
+            # Assign island index if the program is in an active island, otherwise -1
+            # Programs not in archive get island = -1 (historical/removed)
+            prog["island"] = pid_to_island.get(pid, -1) if pid in archive else -1
+            nodes.append(prog)
+            id_to_program[effective_pid] = prog
+
+            # Add parent to loading queue if it exists and hasn't been processed
+            parent_id = prog.get("parent_id")
+            if parent_id and parent_id not in loaded:
+                to_load.append(parent_id)
+        else:
+            logger.debug(f"Program file not found: {prog_path}")
 
     edges = []
     for prog in nodes:
@@ -113,9 +134,36 @@ def program_page(program_id):
 
     data = load_evolution_data(checkpoint_dir)
     program_data = next((p for p in data["nodes"] if p["id"] == program_id), None)
-    program_data = {"code": "", "prompts": {}, **program_data}
+    if program_data is None:
+        return "Program not found", 404
+
+    # Ensure program_data has required fields with safe defaults
+    program_data = {
+        "code": "",
+        "prompts": {},
+        "metrics": {},
+        "id": "",
+        "island": "",
+        "generation": 0,
+        "parent_id": None,
+        **program_data
+    }
+    # Ensure prompts is a dictionary
+    if not isinstance(program_data["prompts"], dict):
+        program_data["prompts"] = {}
+
     artifacts_json = program_data.get("artifacts_json", None)
 
+    # Handle unicode escape for artifacts JSON display - same as in sidebar.js
+    if artifacts_json and isinstance(artifacts_json, str):
+        try:
+            # Parse and stringify to properly escape unicode
+            parsed = json.loads(artifacts_json)
+            artifacts_json = json.dumps(parsed, indent=2, ensure_ascii=False)
+        except (json.JSONDecodeError, TypeError):
+            # If parsing fails, use original value
+            pass
+
     return render_template(
         "program_page.html",
         program_data=program_data,