refactoring and add add a function to verify cached repos

Author: hanouticelina

docs/source/en/guides/cli.md

@@ -671,23 +671,42 @@ As with the other cache commands, `--dry-run`, `--yes`, and `--cache-dir` are av
 
 ## hf cache verify
 
-Use `hf cache verify` to validate cached files against their checksums on the Hub. You can pass one or more targets:
+Use `hf cache verify` to validate local files against their checksums on the Hub. Target a single repo per invocation and choose between verifying the cache snapshot or a regular local directory.
 
-- Repository ID like `model/sentence-transformers/all-MiniLM-L6-v2` to verify all cached revisions for that repo.
-- Specific revision hashes.
+Examples:
 
 ```bash
->>> hf cache verify model/sentence-transformers/all-MiniLM-L6-v2 11c5a3d5811f50298f278a704980280950aedb10
-✅ Verified 60 file(s) across 2 revision(s); no checksum mismatches detected.
+# Verify main revision of a model in cache
+>>> hf cache verify deepseek-ai/DeepSeek-OCR
+
+# Verify a specific revision
+>>> hf cache verify deepseek-ai/DeepSeek-OCR --revision refs/pr/1
+>>> hf cache verify deepseek-ai/DeepSeek-OCR --revision abcdef123
+
+# Verify a private repo
+>>> hf cache verify me/private-model --token hf_***
+
+# Verify a dataset
+>>> hf cache verify karpathy/fineweb-edu-100b-shuffle --repo-type dataset
+
+# Verify files in a local directory
+>>> hf cache verify deepseek-ai/DeepSeek-OCR --local-dir /path/to/repo
 ```
 
-If mismatches are detected, the command prints a detailed list of issues (missing locally, not present on the Hub for this revision, or checksum mismatches) and exits with a non-zero status:
+By default, the command warns about missing or extra files but does not fail. Use flags to make these conditions fail the command:
+
+```bash
+>>> hf cache verify gpt2 --fail-on-missing-files --fail-on-extra-files
+```
+
+On success, you will see a summary:
 
 ```text
-❌ Checksum verification failed for the following file(s):
-  - dataset/nyu-mll/glue@bcdcba79d07bc864c1c254ccfcedcce55bcc9a8c::cola/test-00000-of-00001.parquet: missing locally.
+✅ Verified 60 file(s) at e7da7f221d5bf496a48136c0cd264e630fe9fcc8; no checksum mismatches.
 ```
 
+If mismatches are detected, the command prints a detailed list and exits with a non-zero status.
+
 ## hf repo tag create
 
 The `hf repo tag create` command allows you to tag, untag, and list tags for repositories.

src/huggingface_hub/cli/cache.py

@@ -40,7 +40,7 @@
 )
 from ..utils._parsing import parse_duration, parse_size
 from ..utils.sha import git_hash, sha_fileobj
-from ._cli_utils import TokenOpt, get_hf_api, typer_factory
+from ._cli_utils import RepoIdArg, RepoTypeOpt, RevisionOpt, TokenOpt, get_hf_api, typer_factory
 
 
 cache_cli = typer_factory(help="Manage local cache directory.")
@@ -749,65 +749,97 @@ def prune(
 
 @cache_cli.command()
 def verify(
-    targets: Annotated[
-        list[str],
-        typer.Argument(
-            help="One or more repo IDs (e.g. model/bert-base-uncased) or revision hashes to verify.",
-        ),
-    ],
+    repo_id: RepoIdArg,
+    repo_type: RepoTypeOpt = RepoTypeOpt.model,
+    revision: RevisionOpt = None,
     cache_dir: Annotated[
         Optional[str],
         typer.Option(
-            help="Cache directory to scan (defaults to Hugging Face cache).",
+            help="Cache directory to use when verifying files from cache (defaults to Hugging Face cache).",
         ),
     ] = None,
+    local_dir: Annotated[
+        Optional[str],
+        typer.Option(
+            help="If set, verify files under this directory instead of the cache.",
+        ),
+    ] = None,
+    fail_on_missing_files: Annotated[
+        bool,
+        typer.Option(
+            help="Fail if some files exist on the remote but are missing locally.",
+        ),
+    ] = False,
+    fail_on_extra_files: Annotated[
+        bool,
+        typer.Option(
+            help="Fail if some files exist locally but are not present on the remote revision.",
+        ),
+    ] = False,
     token: TokenOpt = None,
 ) -> None:
-    """Verify cached repos or revisions by comparing against Hub metadata."""
+    """Verify checksums for a single repo revision from cache or a local directory.
 
-    try:
-        hf_cache_info = scan_cache_dir(cache_dir)
-    except CacheNotFound as exc:
-        print(f"Cache directory not found: {str(exc.cache_dir)}")
-        raise typer.Exit(code=1)
+    Examples:
+      - Verify main revision in cache: `hf cache verify gpt2`
+      - Verify specific revision: `hf cache verify gpt2 --revision refs/pr/1`
+      - Verify dataset: `hf cache verify karpathy/fineweb-edu-100b-shuffle --repo-type dataset`
+      - Verify local dir: `hf cache verify deepseek-ai/DeepSeek-OCR --local-dir /path/to/repo`
+    """
 
-    resolution = _resolve_targets(hf_cache_info, targets)
-    if resolution.missing:
-        print("Could not find the following targets in the cache:")
-        for entry in resolution.missing:
-            print(f"  - {entry}")
+    if local_dir is not None and cache_dir is not None:
+        print("Cannot pass both --local-dir and --cache-dir. Use one or the other.")
+        raise typer.Exit(code=2)
 
-    if len(resolution.revisions) == 0:
-        print("Nothing to verify.")
-        raise typer.Exit(code=0)
+    api = get_hf_api(token=token)
 
-    checked_blobs: set[Path] = set()
-    git_hash_cache: dict[Path, str] = {}
-    issues: list[str] = []
+    try:
+        result = api.verify_repo_checksums(
+            repo_id=repo_id,
+            repo_type=repo_type.value if hasattr(repo_type, "value") else str(repo_type),
+            revision=revision,
+            local_dir=local_dir,
+            cache_dir=cache_dir,
+            token=token,
+        )
+    except ValueError as exc:
+        print(str(exc))
+        raise typer.Exit(code=1)
 
-    for repo in sorted(resolution.selected.keys(), key=lambda item: (item.repo_type, item.repo_id.lower())):
-        for revision in sorted(resolution.selected[repo], key=lambda rev: rev.commit_hash):
-            revision_label = f"{repo.cache_id}@{revision.commit_hash}"
-            revision_issues, revision_checked = _verify_revision(
-                repo=repo,
-                revision=revision,
-                revision_label=revision_label,
-                git_hash_cache=git_hash_cache,
-                token=token,
+    # Print mismatches first if any
+    if result.mismatches:
+        print("❌ Checksum verification failed for the following file(s):")
+        for m in result.mismatches:
+            print(f"  - {m['path']}: expected {m['expected']} ({m['algorithm']}), got {m['actual']}")
+
+    # Handle missing/extra
+    exit_code = 0
+    if result.missing_paths:
+        if fail_on_missing_files:
+            print("Missing files (present remotely, absent locally):")
+            for p in result.missing_paths:
+                print(f"  - {p}")
+            exit_code = 1
+        else:
+            print(
+                f"{len(result.missing_paths)} remote file(s) are missing locally. Use --fail-on-missing-files for details."
             )
-            issues.extend(revision_issues)
-            checked_blobs.update(revision_checked)
 
-    if not checked_blobs:
-        print("No cached files found for the requested target.")
-        return
+    if result.extra_paths:
+        if fail_on_extra_files:
+            print("Extra files (present locally, absent remotely):")
+            for p in result.extra_paths:
+                print(f"  - {p}")
+            exit_code = 1
+        else:
+            print(
+                f"{len(result.extra_paths)} local file(s) do not exist on remote repo. Use --fail-on-extra-files for more details."
+            )
 
-    if issues:
-        print("❌ Checksum verification failed for the following file(s):")
-        for entry in issues:
-            print(f"  - {entry}")
-        raise typer.Exit(code=1)
+    if result.mismatches:
+        exit_code = 1
 
-    print(
-        f"✅ Verified {len(checked_blobs)} file(s) across {len(resolution.revisions)} revision(s); no checksum mismatches detected."
-    )
+    if exit_code != 0:
+        raise typer.Exit(code=exit_code)
+
+    print(f"✅ Verified {result.checked_count} file(s) at {result.revision}; no checksum mismatches.")

src/huggingface_hub/hf_api.py

@@ -105,11 +105,13 @@
 from .utils._auth import _get_token_from_environment, _get_token_from_file, _get_token_from_google_colab
 from .utils._deprecation import _deprecate_arguments
 from .utils._typing import CallableT
+from .utils._verification import collect_local_files, resolve_local_root, verify_maps
 from .utils.endpoint_helpers import _is_emission_within_threshold
 
 
 if TYPE_CHECKING:
     from .inference._providers import PROVIDER_T
+    from .utils._verification import Verification
 
 R = TypeVar("R")  # Return type
 CollectionItemType_T = Literal["model", "dataset", "space", "paper", "collection"]
@@ -3057,6 +3059,84 @@ def list_repo_tree(
         for path_info in paginate(path=tree_url, headers=headers, params={"recursive": recursive, "expand": expand}):
             yield (RepoFile(**path_info) if path_info["type"] == "file" else RepoFolder(**path_info))
 
+    @validate_hf_hub_args
+    def verify_repo_checksums(
+        self,
+        repo_id: str,
+        *,
+        repo_type: Optional[str] = None,
+        revision: Optional[str] = None,
+        local_dir: Optional[Union[str, Path]] = None,
+        cache_dir: Optional[Union[str, Path]] = None,
+        token: Union[str, bool, None] = None,
+    ) -> "Verification":
+        """
+        Verify local files for a repo against Hub checksums.
+
+        Args:
+            repo_id (`str`):
+                A namespace (user or an organization) and a repo name separated by a `/`.
+            repo_type (`str`, *optional*):
+                The type of the repository from which to get the tree (`"model"`, `"dataset"` or `"space"`.
+                Defaults to `"model"`.
+            revision (`str`, *optional*):
+                The revision of the repository from which to get the tree. Defaults to `"main"` branch.
+            local_dir (`str` or `Path`, *optional*):
+                The local directory to verify.
+            cache_dir (`str` or `Path`, *optional*):
+                The cache directory to verify.
+            token (Union[bool, str, None], optional):
+                A valid user access token (string). Defaults to the locally saved
+                token, which is the recommended method for authentication (see
+                https://huggingface.co/docs/huggingface_hub/quick-start#authentication).
+                To disable authentication, pass `False`.
+
+        Returns:
+            [`Verification`]: a structured result containing the verification details.
+
+        Raises:
+            [`~utils.RepositoryNotFoundError`]:
+                If repository is not found (error 404): wrong repo_id/repo_type, private but not authenticated or repo
+                does not exist.
+            [`~utils.RevisionNotFoundError`]:
+                If revision is not found (error 404) on the repo.
+            [`~utils.RemoteEntryNotFoundError`]:
+                If the tree (folder) does not exist (error 404) on the repo.
+
+        """
+
+        if repo_type is None:
+            repo_type = constants.REPO_TYPE_MODEL
+
+        if local_dir is not None and cache_dir is not None:
+            raise ValueError("Pass either `local_dir` or `cache_dir`, not both.")
+
+        root, remote_revision = resolve_local_root(
+            repo_id=repo_id,
+            repo_type=repo_type,
+            revision=revision,
+            cache_dir=Path(cache_dir) if cache_dir is not None else None,
+            local_dir=Path(local_dir) if local_dir is not None else None,
+        )
+        local_by_path = collect_local_files(root)
+
+        # get remote entries
+        remote_by_path: dict[str, object] = {}
+        for entry in self.list_repo_tree(
+            repo_id=repo_id, recursive=True, revision=remote_revision, repo_type=repo_type, token=token
+        ):
+            path = getattr(entry, "path", None)
+            if not path:
+                continue
+            lfs = getattr(entry, "lfs", None)
+            has_lfs_sha = (getattr(lfs, "sha256", None) is not None) or (
+                isinstance(lfs, dict) and lfs.get("sha256") is not None
+            )
+            if hasattr(entry, "blob_id") or has_lfs_sha:
+                remote_by_path[path] = entry
+
+        return verify_maps(remote_by_path=remote_by_path, local_by_path=local_by_path, revision=remote_revision)
+
     @validate_hf_hub_args
     def list_repo_refs(
         self,
@@ -10666,6 +10746,7 @@ def _parse_revision_from_pr_url(pr_url: str) -> str:
 list_repo_commits = api.list_repo_commits
 list_repo_tree = api.list_repo_tree
 get_paths_info = api.get_paths_info
+verify_repo_checksums = api.verify_repo_checksums
 
 get_model_tags = api.get_model_tags
 get_dataset_tags = api.get_dataset_tags