fix documents with a lot of paragraphs being removed by the repetitio…

…n filter

src/datatrove/pipeline/filters/gopher_repetition_filter.py

@@ -102,6 +102,7 @@ def __init__(
         self.top_n_grams = top_n_grams
         self.dup_n_grams = dup_n_grams
         self.paragraph_exp = re.compile(r"\n{2,}")
+        self._line_splitter = re.compile("\n+")
 
     def filter(self, doc: Document) -> bool | tuple[bool, str]:
         from nltk.tokenize import word_tokenize
@@ -115,7 +116,7 @@ def filter(self, doc: Document) -> bool | tuple[bool, str]:
         if self.dup_para_char_frac and char_duplicates / len(text) > self.dup_para_char_frac:
             return False, "dup_para_char_frac"
 
-        lines = text.splitlines()
+        lines = self._line_splitter.split(text)
         line_duplicates, char_duplicates = find_duplicates(lines)
         if self.dup_line_frac and line_duplicates / len(lines) > self.dup_line_frac:
             return False, "dup_line_frac"