Fix crash when using --distributed (#114)

LAION-AI · Jan 9, 2024 · 3f1d126 · 3f1d126
1 parent c3e1ebc
commit 3f1d126
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/clip_benchmark/datasets/builder.py b/clip_benchmark/datasets/builder.py
@@ -759,7 +759,7 @@ def read_txt(fname):
     if not cache_dir or not isinstance(cache_dir, str):
         cache_dir = None
     dataset = (
-        wds.WebDataset(filepattern, cache_dir=cache_dir)
+        wds.WebDataset(filepattern, cache_dir=cache_dir, nodesplitter=lambda src: src)
         .decode(wds.autodecode.ImageHandler("pil", extensions=["webp", "png", "jpg", "jpeg"]))
     )
     # Load based on classification or retrieval task