fix(penpal): add All Markdown source, lazy scanning, and fix review findings

apps/penpal/ERD.md

@@ -85,7 +85,7 @@ see-also:
 - <a id="E-PENPAL-SRC-MANUAL"></a>**E-PENPAL-SRC-MANUAL**: The `manual` source type is used for user-added sources (via `penpal open` CLI or the add-workspace/project modal). Directory sources create "tree" type entries; individual file sources create "files" type entries. `GroupFiles()` generates directory headings (`Dir`, `ShowDir` fields on `FileInfo`) for subdirectory boundaries. Configuration is persisted in `config.json` under `ProjectSources` (workspace projects) or inline with the `ProjectConfig` entry (standalone projects).
   ← [P-PENPAL-CLI-OPEN](PRODUCT.md#P-PENPAL-CLI-OPEN), [P-PENPAL-STANDALONE](PRODUCT.md#P-PENPAL-STANDALONE)
 
-- <a id="E-PENPAL-SRC-ALL-MD"></a>**E-PENPAL-SRC-ALL-MD**: An "All Markdown" virtual source is always present in every project's sidebar. It shows all `.md` files in the project directory tree, organized by directory structure, regardless of whether they also appear in a typed source. No badge, no deduplication against other sources. The `Auto` field is `true` — the frontend uses this field to control whether the "Remove from Penpal" context menu item appears, and virtual sources cannot be removed. Implemented as a synthetic source group appended after the typed source groups in the project files response.
+- <a id="E-PENPAL-SRC-ALL-MD"></a>**E-PENPAL-SRC-ALL-MD**: An "All Markdown" source is always present in every project. Registered as a `SourceType` with `GroupFiles` returning a single "All Markdown" group, `ShowDirHeadings: true`, and `SkipDirs` for `node_modules`, `.hg`, `.svn`. Always appended by `DetectSources()` after the detection loop — not conditionally detected. The `Auto` field is `true` — the frontend uses this to prevent removal. During scanning, `__all_markdown__` bypasses the per-source dedup (`seen` map) so it claims every `.md` file regardless of other sources. `AllFiles()` deduplicates by project+path, preferring typed-source entries over `__all_markdown__`, so the recent-files list and activity seeding see each file exactly once. The `handleAddSource` conflict check skips `__all_markdown__` so it doesn't block manual file additions.
   ← [P-PENPAL-SRC-ALL-MD](PRODUCT.md#P-PENPAL-SRC-ALL-MD)
 
 - <a id="E-PENPAL-WORKTREE-DISCOVERY"></a>**E-PENPAL-WORKTREE-DISCOVERY**: Worktrees are discovered by parsing `git worktree list --porcelain` output. Each worktree gets a name, path, branch, and `IsMain` flag. The `refs/heads/` prefix is stripped from branch names.
@@ -101,7 +101,7 @@ see-also:
 - <a id="E-PENPAL-CACHE"></a>**E-PENPAL-CACHE**: An in-memory cache (`sync.RWMutex`-protected) holds the full project list and per-project file lists. `RefreshProject()` walks the filesystem; `RefreshAllProjects()` runs in parallel with no concurrency limit. `RescanWith()` replaces the project list while preserving git enrichment.
   ← [P-PENPAL-PROJECT-FILE-TREE](PRODUCT.md#P-PENPAL-PROJECT-FILE-TREE)
 
-- <a id="E-PENPAL-SCAN"></a>**E-PENPAL-SCAN**: `scanProjectSources()` walks `RootPath` recursively for tree sources, skipping `.git`-file directories (nested worktrees), gitignored directories (via `git check-ignore`), source-type `SkipDirs`, and non-`.md` files. Gitignore checking is initialized once per scan via `newGitIgnoreChecker(projectPath)`, which detects whether the project is a git repo; non-git projects skip the gitignore check gracefully. On write or read failure (partial 4-field response), the checker disables itself (`isGitRepo=false`) to prevent permanent stream desync. The source's own `rootPath` is never checked against gitignore (the `path != rootPath` guard ensures registered sources always scan). Files returning `""` from `ClassifyFile()` are hidden. Files are de-duplicated by project-relative path (first source wins) and sorted by `ModTime` descending.
+- <a id="E-PENPAL-SCAN"></a>**E-PENPAL-SCAN**: `scanProjectSources()` walks `RootPath` recursively for tree sources, skipping `.git`-file directories (nested worktrees), gitignored directories (via `git check-ignore`), source-type `SkipDirs`, and non-`.md` files. Gitignore checking is initialized once per scan via `newGitIgnoreChecker(projectPath)`, which detects whether the project is a git repo; non-git projects skip the gitignore check gracefully. On write or read failure (partial 4-field response), the checker disables itself (`isGitRepo=false`) to prevent permanent stream desync. The source's own `rootPath` is never checked against gitignore (the `path != rootPath` guard ensures registered sources always scan). Files returning `""` from `ClassifyFile()` are hidden. Files are de-duplicated by project-relative path (first source wins) and sorted by `ModTime` descending. `EnsureProjectScanned()` is the lazy-scan entry point — it uses write-lock gating (`projectScanned` set under `mu.Lock` before scanning) to prevent concurrent requests from triggering duplicate filesystem walks. `projectHasAnyMarkdown()` performs a cheap startup check that aligns with the full scan: it uses the same gitignore checking, skips `.git`, `node_modules`, `.hg`, `.svn`, and nested worktree directories, and stops at the first `.md` file found.
   ← [P-PENPAL-PROJECT-FILE-TREE](PRODUCT.md#P-PENPAL-PROJECT-FILE-TREE), [P-PENPAL-FILE-TYPES](PRODUCT.md#P-PENPAL-FILE-TYPES), [P-PENPAL-SRC-DEDUP](PRODUCT.md#P-PENPAL-SRC-DEDUP), [P-PENPAL-SRC-GITIGNORE](PRODUCT.md#P-PENPAL-SRC-GITIGNORE)
 
 - <a id="E-PENPAL-TITLE-EXTRACT"></a>**E-PENPAL-TITLE-EXTRACT**: `EnrichTitles()` reads the first 20 lines of each file to extract H1 headings. Titles are cached and shown as the primary display name when present.
@@ -351,7 +351,7 @@ see-also:
 
 ## Search
 
-- <a id="E-PENPAL-SEARCH"></a>**E-PENPAL-SEARCH**: `GET /api/search?q=` handled by `handleAPISearch()`. Iterates `s.cache.Projects()`, doing substring match on project names for the "Projects" section (`matchingProjects`). For file matches, walks each source's `RootPath` via `filepath.Walk`, skipping `SkipDirs` and non-`.md` files. Filenames checked via `strings.Contains(ToLower(...), query)` (flagged `nameMatch: true`); content checked via `bufio.Scanner` line-by-line (stops at first match). Files returning `""` from `ClassifyFile()` are excluded. Results per project sorted name-matches-first then alphabetical; capped at 100 total files with early `break` on project iteration. Response struct: `apiSearchResponse{Query, MatchingProjects, ProjectResults, TotalFiles}`. Frontend `SearchPage` component at route `/search` calls `api.search(query)` and renders project cards, grouped file rows with `FileTypeBadge`, and a "name" `<span class="match-type">` badge for name matches.
+- <a id="E-PENPAL-SEARCH"></a>**E-PENPAL-SEARCH**: `GET /api/search?q=` handled by `handleAPISearch()`. Iterates `s.cache.Projects()`, doing substring match on project names for the "Projects" section (`matchingProjects`). For file matches, walks each source's `RootPath` via `filepath.Walk`, skipping `.git` directories, nested worktree/submodule directories (`.git` file), `SkipDirs`, and non-`.md` files. Filenames checked via `strings.Contains(ToLower(...), query)` (flagged `nameMatch: true`); content checked via `bufio.Scanner` line-by-line (stops at first match). Files returning `""` from `ClassifyFile()` are excluded. Results per project sorted name-matches-first then alphabetical; capped at 100 total files with early `break` on project iteration. Response struct: `apiSearchResponse{Query, MatchingProjects, ProjectResults, TotalFiles}`. Frontend `SearchPage` component at route `/search` calls `api.search(query)` and renders project cards, grouped file rows with `FileTypeBadge`, and a "name" `<span class="match-type">` badge for name matches.
   ← [P-PENPAL-SEARCH](PRODUCT.md#P-PENPAL-SEARCH)
 
 ---
@@ -368,7 +368,7 @@ see-also:
 
 ## Activity Tracking
 
-- <a id="E-PENPAL-ACTIVITY"></a>**E-PENPAL-ACTIVITY**: In-memory `activity.Tracker` stores one event per file (keyed by project + path). Event types: `viewed`, `modified`, `created`, `comment`, `published`. `Record()` always overwrites; `RecordAt()` (for seeding from mtime) does not overwrite.
+- <a id="E-PENPAL-ACTIVITY"></a>**E-PENPAL-ACTIVITY**: In-memory `activity.Tracker` stores one event per file (keyed by project + path). Event types: `viewed`, `modified`, `created`, `comment`, `published`. `Record()` always overwrites; `RecordAt()` (for seeding from mtime) does not overwrite. Activity is seeded per-project when `EnsureProjectScanned()` performs the first lazy scan — each file's `ModTime` is recorded via `RecordAt()`, excluding `__all_markdown__` duplicates. This populates `/api/recent` progressively as projects are opened.
   ← [P-PENPAL-GLOBAL-RECENT](PRODUCT.md#P-PENPAL-GLOBAL-RECENT), [P-PENPAL-PROJECT-RECENT](PRODUCT.md#P-PENPAL-PROJECT-RECENT)
 
 - <a id="E-PENPAL-RECENT-PAGE"></a>**E-PENPAL-RECENT-PAGE**: `GET /api/recent` returns up to 50 recently active files across all projects, sorted by most recent activity. Frontend `RecentPage` renders two-line file rows with filename, workspace / project context, and relative timestamp. Clicking navigates the current tab to the file.

apps/penpal/TESTING.md

@@ -64,7 +64,7 @@ see-also:
 | Source Types — anchors (P-PENPAL-SRC-ANCHORS, SRC-ANCHORS-GROUP, SRC-ANCHORS-NESTED) | discovery_test.go (TestClassifyAnchorsFile, TestGroupAnchorsPaths, TestGroupAnchorsPaths_MarkerOnlyModule, TestAnchorsFileOrder, TestAnchorsRequireSibling) | — | — | — |
 | Source Types — claude-plans (P-PENPAL-SRC-CLAUDE-PLANS) | — | — | — | — |
 | Source Types — manual (P-PENPAL-SRC-MANUAL) | — | — | grouping_test.go (TestBuildFileGroups_ManualSourceDirHeadings) | — |
-| Cache & File Scanning (E-PENPAL-CACHE, SCAN) | cache_test.go | — | — | — |
+| Cache & File Scanning (E-PENPAL-CACHE, SCAN) | cache_test.go (TestCheckAllProjectsHasFiles, TestProjectHasAnyMarkdown_SkipsGitignored, TestProjectHasAnyMarkdown_SkipsVCSDirs, TestAllFiles_DeduplicatesAllMarkdown, TestEnsureProjectScanned_NoDuplicateScans) | — | — | — |
 | Worktree Support (P-PENPAL-WORKTREE) | discovery/worktree_test.go, cache/worktree_test.go | Layout.test.tsx | worktree_test.go (API + MCP) | — |
 | Worktree Dropdown (P-PENPAL-PROJECT-WORKTREE-DROPDOWN) | — | Layout.test.tsx | — | — |
 | Git Integration (P-PENPAL-GIT-INFO) | — | — | — | — |
@@ -89,7 +89,7 @@ see-also:
 | Search (P-PENPAL-SEARCH) | — | SearchPage.test.tsx | — | react-app.spec.ts |
 | Recent Files (P-PENPAL-RECENT) | activity_test.go | RecentPage.test.tsx | integration_test.go | — |
 | CLI Open (P-PENPAL-CLI-OPEN) | — | — | api_manage_test.go | cli-open.spec.ts |
-| Source Management (P-PENPAL-ADD-SOURCE, REMOVE-SOURCE) | — | — | api_manage_test.go | — |
+| Source Management (P-PENPAL-ADD-SOURCE, REMOVE-SOURCE) | — | — | api_manage_test.go (TestAPISources_AddFileNotBlockedByAllMarkdown) | — |
 | Real-Time Updates (P-PENPAL-REALTIME, FOCUS) | watcher_test.go | useSSE.test.ts | api_focus_test.go | — |
 | Config & Migration (E-PENPAL-CONFIG) | config_test.go, migrate_test.go | — | — | — |
 | Install Tools (P-PENPAL-INSTALL) | — | InstallStartup.test.tsx, InstallToolsModal.test.tsx | install_test.go | — |

apps/penpal/frontend/src/components/HomeSidebar.tsx

@@ -144,7 +144,7 @@ export default function HomeSidebar({
                   return (
                     <div key={p.qualifiedName}>
                       <div
-                        className={`tree-item${p.fileCount === 0 ? ' deemphasized' : ''}`}
+                        className={`tree-item${!p.hasFiles ? ' deemphasized' : ''}`}
                         onClick={() => {
                           if (hasWorktrees) {
                             onToggleWorktreeProject(p.qualifiedName);
@@ -201,7 +201,7 @@ export default function HomeSidebar({
         <Link
           key={p.qualifiedName}
           to={`/project/${p.qualifiedName}`}
-          className={`tree-item${p.fileCount === 0 ? ' deemphasized' : ''}`}
+          className={`tree-item${!p.hasFiles ? ' deemphasized' : ''}`}
           onContextMenu={(e) => onShowContextMenu(e, [
             { label: 'Close project', className: 'menu-muted', onClick: () => onCloseStandaloneProject(p) },
           ])}

apps/penpal/frontend/src/components/Layout.close-tab.test.tsx

@@ -51,8 +51,7 @@ vi.mock('../api', () => ({
         qualifiedName: 'ws1/project-a',
         workspace: 'ws1',
         origin: 'workspace',
-        badges: [],
-        fileCount: 5,
+        hasFiles: true,
         lastModified: '2026-01-01T00:00:00Z',
       },
     ]),

apps/penpal/frontend/src/components/Layout.external-links.test.tsx

@@ -37,8 +37,7 @@ vi.mock('../api', () => ({
         qualifiedName: 'ws1/project-a',
         workspace: 'ws1',
         origin: 'workspace',
-        badges: [],
-        fileCount: 5,
+        hasFiles: true,
         lastModified: '2026-01-01T00:00:00Z',
       },
     ]),

apps/penpal/frontend/src/components/Layout.test.tsx

@@ -29,8 +29,8 @@ vi.mock('../api', () => ({
         qualifiedName: 'ws1/project-a',
         workspace: 'ws1',
         origin: 'workspace',
-        badges: [],
-        fileCount: 5,
+
+        hasFiles: true,
         lastModified: '2026-01-01T00:00:00Z',
       },
     ]),
@@ -541,8 +541,8 @@ describe('Layout', () => {
         qualifiedName: 'ws1/project-a',
         workspace: 'ws1',
         origin: 'workspace',
-        badges: [],
-        fileCount: 5,
+
+        hasFiles: true,
         lastModified: '2026-01-01T00:00:00Z',
         worktrees: [
           { name: 'main', path: '/tmp/main', branch: 'main', isMain: true },
@@ -581,8 +581,8 @@ describe('Layout', () => {
         qualifiedName: 'ws1/project-a',
         workspace: 'ws1',
         origin: 'workspace',
-        badges: [],
-        fileCount: 5,
+
+        hasFiles: true,
         lastModified: '2026-01-01T00:00:00Z',
         worktrees: [
           { name: 'main', path: '/tmp/main', branch: 'main', isMain: true },
@@ -642,8 +642,8 @@ describe('Layout', () => {
         workspace: '',
         projectPath: '/tmp/bravo',
         origin: 'standalone' as const,
-        badges: [],
-        fileCount: 3,
+
+        hasFiles: true,
         lastModified: '2026-01-02T00:00:00Z',
       },
       {
@@ -652,8 +652,8 @@ describe('Layout', () => {
         workspace: '',
         projectPath: '/tmp/alpha',
         origin: 'standalone' as const,
-        badges: [],
-        fileCount: 5,
+
+        hasFiles: true,
         lastModified: '2026-01-01T00:00:00Z',
       },
     ]);
@@ -696,8 +696,8 @@ describe('Layout', () => {
         workspace: '',
         projectPath: '/tmp/has-files',
         origin: 'standalone' as const,
-        badges: [],
-        fileCount: 5,
+
+        hasFiles: true,
         lastModified: '2026-01-01T00:00:00Z',
       },
       {
@@ -706,8 +706,8 @@ describe('Layout', () => {
         workspace: '',
         projectPath: '/tmp/empty-project',
         origin: 'standalone' as const,
-        badges: [],
-        fileCount: 0,
+
+        hasFiles: false,
         lastModified: '2026-01-01T00:00:00Z',
       },
     ]);

apps/penpal/frontend/src/components/Layout.tsx

@@ -327,13 +327,13 @@ export default function Layout() {
   );
   // E-PENPAL-SORT: shared comparator — empty projects last, then alpha or API order.
   const projectSort = useCallback((a: APIProject, b: APIProject) => {
-    if ((a.fileCount > 0) !== (b.fileCount > 0)) return b.fileCount > 0 ? 1 : -1;
+    if (a.hasFiles !== b.hasFiles) return b.hasFiles ? 1 : -1;
     if (sortOrder === 'alpha') return a.name.localeCompare(b.name);
     return 0;
   }, [sortOrder]);
 
   // E-PENPAL-VIEW-OPTIONS: filter empty projects when showEmpty is false
-  const filterEmpty = useCallback((p: APIProject) => showEmpty || p.fileCount > 0, [showEmpty]);
+  const filterEmpty = useCallback((p: APIProject) => showEmpty || p.hasFiles, [showEmpty]);
 
   const standaloneProjects = useMemo(() => {
     return projects.filter((p) => p.origin === 'standalone').filter(filterEmpty).sort(projectSort);

apps/penpal/frontend/src/pages/FilePage.test.tsx

@@ -37,7 +37,7 @@ vi.mock('../components/MermaidRenderer', () => ({
 function LayoutWrapper() {
   const ctx: LayoutContext = {
     setHeadings: vi.fn(),
-    projects: [{ name: 'proj', qualifiedName: 'ws/proj', workspace: 'ws', origin: 'workspace' as const, badges: [], fileCount: 1, lastModified: '', projectPath: '/tmp/proj' }],
+    projects: [{ name: 'proj', qualifiedName: 'ws/proj', workspace: 'ws', origin: 'workspace' as const, hasFiles: true, lastModified: '', projectPath: '/tmp/proj' }],
   };
   return <Outlet context={ctx} />;
 }

apps/penpal/frontend/src/types.ts

@@ -1,13 +1,5 @@
 // Types matching Go JSON API responses
 
-export interface APIBadge {
-  text: string;
-  color: string;
-  bg: string;
-  activeBg?: string;
-  activeColor?: string;
-}
-
 export interface APIWorktree {
   name: string;
   path: string;
@@ -22,10 +14,9 @@ export interface APIProject {
   workspacePath?: string;
   projectPath: string;
   origin: 'workspace' | 'standalone';
-  badges: APIBadge[];
   branch?: string;
   dirty?: boolean;
-  fileCount: number;
+  hasFiles: boolean;
   lastModified: string;
   agentConnected?: boolean;
   agentRunning?: boolean;
@@ -147,7 +138,7 @@ export interface ReviewGroup {
 }
 
 export interface ProjectInfo {
-  fileCount: number;
+  hasFiles: boolean;
   dirty: boolean;
   unpushedCommits: number;
 }