Use huggingface_hub cache

.github/conda/meta.yaml

@@ -24,7 +24,7 @@ requirements:
     - dataclasses
     - multiprocess
     - fsspec
-    - huggingface_hub >=0.21.2,<1.0.0
+    - huggingface_hub >=0.22.0,<1.0.0
     - packaging
     - aiohttp
   run:
@@ -41,7 +41,7 @@ requirements:
     - dataclasses
     - multiprocess
     - fsspec
-    - huggingface_hub >=0.21.2,<1.0.0
+    - huggingface_hub >=0.22.0,<1.0.0
     - packaging
     - aiohttp
 

.github/workflows/ci.yml

@@ -65,7 +65,7 @@ jobs:
         run: uv pip install --system --upgrade pyarrow huggingface-hub dill
       - name: Install dependencies (minimum versions)
         if: ${{ matrix.deps_versions != 'deps-latest' }}
-        run: uv pip install --system pyarrow==15.0.0 huggingface-hub==0.21.2 transformers dill==0.3.1.1
+        run: uv pip install --system pyarrow==15.0.0 huggingface-hub==0.22.0 transformers dill==0.3.1.1
       - name: Test with pytest
         run: |
           python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/

docs/source/audio_dataset.mdx

@@ -523,8 +523,6 @@ The reason you need to use a combination of [`~DownloadManager.download`] and [`
 ```py
 def _split_generators(self, dl_manager):
     """Returns SplitGenerators."""
-    dl_manager.download_config.ignore_url_params = True
-
     audio_path = dl_manager.download(_AUDIO_URL)
     local_extracted_archive = dl_manager.extract(audio_path) if not dl_manager.is_streaming else None
     path_to_clips = "librivox-indonesia"

setup.py

@@ -133,7 +133,7 @@
     # for data streaming via http
     "aiohttp",
     # To get datasets from the Datasets Hub on huggingface.co
-    "huggingface-hub>=0.21.2",
+    "huggingface-hub>=0.22.0",
     # Utilities from PyPA to e.g., compare versions
     "packaging",
     # To parse YAML metadata from dataset cards

src/datasets/builder.py

@@ -51,7 +51,6 @@
 from .dataset_dict import DatasetDict, IterableDatasetDict
 from .download.download_config import DownloadConfig
 from .download.download_manager import DownloadManager, DownloadMode
-from .download.mock_download_manager import MockDownloadManager
 from .download.streaming_download_manager import StreamingDownloadManager, xjoin
 from .exceptions import DatasetGenerationCastError, DatasetGenerationError, FileFormatError, ManualDownloadError
 from .features import Features
@@ -931,14 +930,6 @@ def download_and_prepare(
             )
 
         is_local = not is_remote_filesystem(self._fs)
-
-        if (
-            isinstance(dl_manager, MockDownloadManager)
-            or not is_local
-            or file_format != "arrow"
-            or max_shard_size is not None
-        ):
-            try_from_hf_gcs = False
         self.dl_manager = dl_manager
 
         # Prevent parallel local disk operations

src/datasets/commands/datasets_cli.py

@@ -4,7 +4,6 @@
 from datasets.commands.convert import ConvertCommand
 from datasets.commands.convert_to_parquet import ConvertToParquetCommand
 from datasets.commands.delete_from_hub import DeleteFromHubCommand
-from datasets.commands.dummy_data import DummyDataCommand
 from datasets.commands.env import EnvironmentCommand
 from datasets.commands.test import TestCommand
 from datasets.utils.logging import set_verbosity_info
@@ -25,7 +24,6 @@ def main():
     ConvertCommand.register_subcommand(commands_parser)
     EnvironmentCommand.register_subcommand(commands_parser)
     TestCommand.register_subcommand(commands_parser)
-    DummyDataCommand.register_subcommand(commands_parser)
     ConvertToParquetCommand.register_subcommand(commands_parser)
     DeleteFromHubCommand.register_subcommand(commands_parser)