No need for dataset_info (#7234)

* no need for dataset_info * actually do a single call in dataset_module_factory * update tests * fix tests * again * again * better check for gated error response code
huggingface · Oct 21, 2024 · 59738d7 · 59738d7
1 parent d4422cc
commit 59738d7
Show file tree

Hide file tree

Showing 5 changed files with 200 additions and 114 deletions.
diff --git a/src/datasets/download/download_manager.py b/src/datasets/download/download_manager.py
@@ -189,7 +189,11 @@ def _download_batched(
             download_func = partial(self._download_single, download_config=download_config)
 
             fs: fsspec.AbstractFileSystem
-            fs, path = url_to_fs(url_or_filenames[0], **download_config.storage_options)
+            path = str(url_or_filenames[0])
+            if is_relative_path(path):
+                # append the relative path to the base_path
+                path = url_or_path_join(self._base_path, path)
+            fs, path = url_to_fs(path, **download_config.storage_options)
             size = 0
             try:
                 size = fs.info(path).get("size", 0)