Remove links from telegram text messages

llama_hub/telegram/base.py

@@ -1,5 +1,6 @@
 """Telegram reader that reads posts/chats and comments to post from Telegram channel or chat."""
 import asyncio
+import re
 from typing import List, Union
 
 from llama_index.readers.base import BaseReader
@@ -102,5 +103,15 @@ async def _load_data(
                 entity_name, reply_to=post_id, limit=limit
             ):
                 if isinstance(message.text, str) and message.text != "":
-                    results.append(Document(text=message.text))
+                    results.append(Document(text=self._remove_links(message.text)))
         return results
+
+    def _remove_links(self, string) -> str:
+        """Removes all URLs from a given string, leaving only the base domain name."""
+
+        def replace_match(match):
+            text = match.group(1)
+            return text if text else ""
+
+        url_pattern = r"https?://(?:www\.)?((?!www\.).)+?"
+        return re.sub(url_pattern, replace_match, string)