Respect file order in DocumentDataset.read (#487)

Signed-off-by: Praateek <[email protected]>
NVIDIA · Jan 17, 2025 · 7a49ebb · 7a49ebb
1 parent 7cfda44
commit 7a49ebb
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/nemo_curator/utils/distributed_utils.py b/nemo_curator/utils/distributed_utils.py
@@ -485,7 +485,6 @@ def read_data_files_per_partition(
     columns: Optional[List[str]] = None,
     **kwargs,
 ) -> dd.DataFrame:
-    input_files = sorted(input_files)
     if files_per_partition > 1:
         input_files = [
             input_files[i : i + files_per_partition]