add selective TOC item extraction feature

Author: async

README.md

@@ -11,6 +11,7 @@ A command-line tool to scrape and structure GitBook documentation into a single,
 - 🛠️ Configurable output format and structure
 - 🔄 Automatic retry on failed requests
 - 📋 Table of contents generation
+- 🎯 Selective TOC item extraction
 
 ## Installation
 
@@ -32,6 +33,9 @@ gitbook-scraper https://your-gitbook-url.io --toc
 
 # Custom rate limiting
 gitbook-scraper https://your-gitbook-url.io --delay 1.0
+
+# Extract specific TOC items
+gitbook-scraper https://your-gitbook-url.io -t "Getting Started" -t "Advanced Topics"
 ```
 
 ## Advanced Usage
@@ -47,6 +51,7 @@ Options:
   --timeout INTEGER    Request timeout in seconds [default: 10]
   --debug             Enable debug logging [default: False]
   --no-cleanup        Keep intermediate files [default: False]
+  -t, --toc-items TEXT  Specific TOC items to extract (can be specified multiple times)
   --help             Show this message and exit
 ```
 
@@ -55,13 +60,22 @@ Options:
 ```python
 from gitbook_scraper import GitbookScraper
 
+# Basic usage
 scraper = GitbookScraper(
     base_url="https://your-gitbook-url.io",
     output_file="documentation.md",
     generate_toc=True,
     delay=0.5
 )
 
+# Extract specific TOC items
+scraper = GitbookScraper(
+    base_url="https://your-gitbook-url.io",
+    output_file="documentation.md",
+    generate_toc=True,
+    toc_items=["Getting Started", "Advanced Topics"]
+)
+
 scraper.scrape()
 ```
 

gitbook_scraper/cli.py

@@ -26,8 +26,10 @@
               help='Enable debug logging [default: False]')
 @click.option('--selector-file', type=click.Path(exists=True, dir_okay=False),
               help='Path to JSON file with custom CSS selectors')
+@click.option('--toc-items', '-t', multiple=True,
+              help='Specific TOC items to extract (can be specified multiple times)')
 def main(url: str, output: str, toc: bool, delay: float, retries: int,
-         timeout: int, debug: bool, selector_file: Optional[str]) -> int:
+         timeout: int, debug: bool, selector_file: Optional[str], toc_items: tuple) -> int:
     """Scrape and structure GitBook documentation into a single markdown file."""
     try:
         # Validate URL
@@ -43,7 +45,8 @@ def main(url: str, output: str, toc: bool, delay: float, retries: int,
             retries=retries,
             timeout=timeout,
             debug=debug,
-            selector_file=selector_file
+            selector_file=selector_file,
+            toc_items=list(toc_items) if toc_items else None
         )
 
         with console.status("[bold green]Scraping documentation..."):

gitbook_scraper/scraper.py

@@ -22,7 +22,8 @@ def __init__(
         retries: int = 3,
         timeout: int = 10,
         debug: bool = False,
-        selector_file: Optional[str] = None
+        selector_file: Optional[str] = None,
+        toc_items: Optional[List[str]] = None
     ):
         """
         Initialize the GitBook scraper.
@@ -36,6 +37,7 @@ def __init__(
             timeout: Request timeout in seconds
             debug: Enable debug logging
             selector_file: Optional JSON file with custom CSS selectors
+            toc_items: Optional list of TOC item titles to extract. If None, extracts all items.
         """
         self.base_url = self.normalize_url(base_url)
         self.domain = urlparse(self.base_url).netloc
@@ -44,6 +46,7 @@ def __init__(
         self.delay = delay
         self.retries = retries
         self.timeout = timeout
+        self.toc_items = set(toc_items) if toc_items else None
 
         # Setup logging
         log_level = logging.DEBUG if debug else logging.INFO
@@ -369,6 +372,23 @@ def generate_markdown(self, structure: List[Dict], level: int = 1) -> str:
 
         return "\n".join(content)
 
+    def filter_nav_structure(self, structure: List[Dict]) -> List[Dict]:
+        """Filter navigation structure to only include specified TOC items and their children."""
+        if not self.toc_items:
+            return structure
+
+        filtered = []
+        for item in structure:
+            if item['title'] in self.toc_items:
+                filtered.append(item)
+            elif item['children']:
+                filtered_children = self.filter_nav_structure(item['children'])
+                if filtered_children:
+                    item_copy = item.copy()
+                    item_copy['children'] = filtered_children
+                    filtered.append(item_copy)
+        return filtered
+
     def scrape(self):
         """Main scraping method."""
         with Progress(
@@ -381,6 +401,12 @@ def scrape(self):
 
                 if not self.nav_structure:
                     raise NavigationExtractionError("Empty navigation structure")
+
+                # Filter navigation structure if specific TOC items are requested
+                if self.toc_items:
+                    self.nav_structure = self.filter_nav_structure(self.nav_structure)
+                    if not self.nav_structure:
+                        raise NavigationExtractionError("No matching TOC items found")
 
                 progress.add_task("Generating documentation...", total=None)
                 content = self.generate_markdown(self.nav_structure)

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "gitbook-scraper"
-version = "0.1.0"
+version = "0.1.1"
 authors = [
   { name="Async", email="[email protected]" },
 ]