fix streaming from arrow files (#7083)

fix streaming from local arrow files
huggingface · Aug 30, 2024 · ca58154 · ca58154
1 parent be5cff0
commit ca58154
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/datasets/utils/file_utils.py b/src/datasets/utils/file_utils.py
@@ -480,6 +480,7 @@ def readline(f: io.RawIOBase):
     "pickle",
     "rel",
     "xml",
+    "arrow",
 ]
 COMPRESSION_EXTENSION_TO_PROTOCOL = {
     # single file compression