fix: paginate large Hugging Face inventories

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Bug Fixes
 
+- paginate and bound large Hugging Face repository inventories before streaming so unfiltered scans preserve complete coverage
 - dispatch logical model directories through their owning scanners, preserving bounded complete SavedModel asset probes before supplemental child-file coverage
 - stream large Flax MessagePack tensor bodies by declared length without tripping the 512 MiB decode budget
 - avoid critical pickle `getattr` findings for proven Ultralytics `Detect.forward` reconstruction while preserving unsafe traversal controls

modelaudit/utils/sources/_huggingface_download_worker.py

@@ -12,28 +12,17 @@
 
 def _run_operation(operation: str, operation_kwargs: dict[str, Any]) -> dict[str, Any]:
     if operation == "list_repo_files":
-        from huggingface_hub import HfApi
+        from modelaudit.utils.sources.huggingface import _list_huggingface_repo_files_at_revision
 
-        repo_info_kwargs: dict[str, Any] = {
-            "timeout": operation_kwargs.get("request_timeout"),
-            "files_metadata": False,
-        }
         requested_revision = operation_kwargs.get("revision")
-        if requested_revision is not None:
-            repo_info_kwargs["revision"] = requested_revision
-        repo_info = HfApi().repo_info(operation_kwargs["repo_id"], **repo_info_kwargs)
-        siblings = getattr(repo_info, "siblings", None)
-        files: list[str] | None = None
-        if siblings is not None:
-            files = []
-            for sibling in siblings:
-                if isinstance(sibling, dict):
-                    file_name = sibling.get("rfilename") or sibling.get("path")
-                else:
-                    file_name = getattr(sibling, "rfilename", None) or getattr(sibling, "path", None)
-                if isinstance(file_name, str) and file_name:
-                    files.append(file_name)
-        return {"value": {"files": files, "revision": getattr(repo_info, "sha", None)}}
+        if requested_revision is None:
+            requested_revision = operation_kwargs.get("requested_revision")
+        files, revision = _list_huggingface_repo_files_at_revision(
+            operation_kwargs["repo_id"],
+            requested_revision=requested_revision,
+            timeout_seconds=operation_kwargs.get("request_timeout", 30),
+        )
+        return {"value": {"files": files, "revision": revision}}
     if operation == "snapshot_download":
         from huggingface_hub import snapshot_download
 
@@ -71,11 +60,17 @@ def _run_operation(operation: str, operation_kwargs: dict[str, Any]) -> dict[str
                 path_repo_info_kwargs["revision"] = requested_revision
             repo_info = api.repo_info(operation_kwargs["repo_id"], **path_repo_info_kwargs)
             resolved_revision = getattr(repo_info, "sha", None)
-        path_info = api.get_paths_info(
-            operation_kwargs["repo_id"],
-            operation_kwargs["filenames"],
-            revision=resolved_revision,
-        )
+        filenames = operation_kwargs["filenames"]
+        path_info = []
+        batch_size = 512
+        for start in range(0, len(filenames), batch_size):
+            path_info.extend(
+                api.get_paths_info(
+                    operation_kwargs["repo_id"],
+                    filenames[start : start + batch_size],
+                    revision=resolved_revision,
+                )
+            )
         return {
             "value": {
                 "revision": resolved_revision,