cli.py

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

import requests

from crawler import Downloader, download_pages, extract_nav_tree, plan_output_tree, record_tree_to_manifest
from markdown_converter import convert_html_tree
from models import ToolError


def prompt_if_missing(value: str | None, message: str) -> str:
    if value:
        return value
    return input(message).strip()


def parse_args(argv: list[str]) -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Download a docs sidebar tree into HTML and Markdown folders."
    )
    parser.add_argument("--base-url", help="The URL of the docs page that contains the sidebar.")
    parser.add_argument("--xpath", help="XPath for the sidebar wrapper element.")
    parser.add_argument("--target-dir", help="Directory that will receive html/ and markdown/.")
    return parser.parse_args(argv)


def run(base_url: str, xpath: str, target_dir: Path) -> None:
    target_dir.mkdir(parents=True, exist_ok=True)
    session = requests.Session()
    session.headers.update(
        {
            "User-Agent": "MrSiderDoc/1.0 (+https://openai.com)",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        }
    )

    downloader = Downloader(session, base_url)
    print(f"Reading sidebar from {base_url}")
    sidebar = downloader.fetch_sidebar(xpath)
    nav_items = extract_nav_tree(sidebar)
    records = plan_output_tree(nav_items, target_dir)
    known_pages = download_pages(downloader, base_url, records, target_dir)
    convert_html_tree(target_dir, records, known_pages, xpath)

    manifest_path = target_dir / "manifest.json"
    manifest_path.write_text(
        json.dumps(
            {
                "base_url": base_url,
                "xpath": xpath,
                "items": record_tree_to_manifest(records),
            },
            ensure_ascii=False,
            indent=2,
        )
        + "\n",
        encoding="utf-8",
    )


def main(argv: list[str] | None = None) -> int:
    args = parse_args(argv or sys.argv[1:])

    try:
        base_url = prompt_if_missing(args.base_url, "Base site address: ")
        xpath = prompt_if_missing(args.xpath, "Sidebar XPath: ")
        target_dir = Path(prompt_if_missing(args.target_dir, "Target directory: ")).expanduser()
        run(base_url, xpath, target_dir)
    except KeyboardInterrupt:
        print("\nCancelled.", file=sys.stderr)
        return 130
    except requests.RequestException as exc:
        print(f"Network error: {exc}", file=sys.stderr)
        return 1
    except ToolError as exc:
        print(f"Error: {exc}", file=sys.stderr)
        return 1

    print(f"Finished. Output written to {target_dir}")
    return 0