[3595][Core] add ability to download files from daemon

When we use thin client mode, we will download tracker's icon from the
client endpoint.
This means we are leaking the IP of the client location, instead using
the daemon, which is already connected to the tracker.
Therefor, an ability to download files from the daemon is added.

Closes https://dev.deluge-torrent.org/ticket/3595

deluge/core/core.py

@@ -10,7 +10,9 @@
 import glob
 import logging
 import os
+import random
 import shutil
+import string
 import tempfile
 from base64 import b64decode, b64encode
 from typing import Any, Dict, List, Optional, Tuple, Union
@@ -508,6 +510,74 @@ async def add_torrents():
 
         return task.deferLater(reactor, 0, add_torrents)
 
+    @maybe_coroutine
+    async def _download_file(
+        self,
+        url,
+        callback=None,
+        headers=None,
+        allow_compression=True,
+        handle_redirects=True,
+    ) -> 'defer.Deferred[Optional[bytes]]':
+        tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.')
+        try:
+            filename = await download_file(
+                url=url,
+                filename=tmp_file,
+                callback=callback,
+                headers=headers,
+                force_filename=True,
+                allow_compression=allow_compression,
+                handle_redirects=handle_redirects,
+            )
+        except Exception:
+            raise
+        else:
+            with open(filename, 'rb') as _file:
+                data = _file.read()
+            return data
+        finally:
+            try:
+                os.close(tmp_fd)
+                os.remove(tmp_file)
+            except OSError as ex:
+                log.warning(f'Unable to delete temp file {tmp_file}: , {ex}')
+
+    @export
+    @maybe_coroutine
+    async def download_file(
+        self,
+        url,
+        callback=None,
+        headers=None,
+        allow_compression=True,
+        handle_redirects=True,
+    ) -> 'defer.Deferred[Optional[bytes]]':
+        """Downloads a file from a URL and returns the content as bytes.
+
+        Use this method to download from the daemon itself (like a proxy).
+
+        Args:
+            url (str): The url to download from.
+            callback (func): A function to be called when partial data is received,
+                it's signature should be: func(data, current_length, total_length).
+            headers (dict): Any optional headers to send.
+            allow_compression (bool): Allows gzip & deflate decoding.
+            handle_redirects (bool): HTTP redirects handled automatically or not.
+
+        Returns:
+            a Deferred which returns the content as bytes or None
+        """
+        log.info(f'Attempting to download URL {url}')
+
+        try:
+            return await self._download_file(
+                url, callback, headers, allow_compression, handle_redirects
+            )
+        except Exception:
+            log.error(f'Failed to download file from URL {url}')
+            raise
+
     @export
     @maybe_coroutine
     async def add_torrent_url(
@@ -524,26 +594,17 @@ async def add_torrent_url(
         Returns:
             a Deferred which returns the torrent_id as a str or None
         """
-        log.info('Attempting to add URL %s', url)
+        log.info(f'Attempting to add URL {url}')
 
-        tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.', suffix='.torrent')
         try:
-            filename = await download_file(
-                url, tmp_file, headers=headers, force_filename=True
-            )
+            data = await self._download_file(url, headers=headers)
         except Exception:
-            log.error('Failed to add torrent from URL %s', url)
+            log.error(f'Failed to add torrent from URL {url}')
             raise
         else:
-            with open(filename, 'rb') as _file:
-                data = _file.read()
-            return self.add_torrent_file(filename, b64encode(data), options)
-        finally:
-            try:
-                os.close(tmp_fd)
-                os.remove(tmp_file)
-            except OSError as ex:
-                log.warning(f'Unable to delete temp file {tmp_file}: , {ex}')
+            chars = string.ascii_letters + string.digits
+            tmp_file_name = ''.join(random.choices(chars, k=7))
+            return self.add_torrent_file(tmp_file_name, b64encode(data), options)
 
     @export
     def add_torrent_magnet(self, uri: str, options: dict) -> str:

deluge/tests/test_core.py

@@ -509,3 +509,58 @@ def test_create_torrent(self, path, tmp_path, piece_length):
             assert f.read() == filecontent
 
         lt.torrent_info(filecontent)
+
+    @pytest.fixture
+    def _download_file_content(self):
+        with open(
+            common.get_test_data_file('ubuntu-9.04-desktop-i386.iso.torrent'), 'rb'
+        ) as _file:
+            data = _file.read()
+        return data
+
+    @pytest_twisted.inlineCallbacks
+    def test_download_file(self, mock_mkstemp, _download_file_content):
+        url = (
+            'http://localhost:%d/ubuntu-9.04-desktop-i386.iso.torrent'
+            % self.listen_port
+        )
+
+        file_content = yield self.core.download_file(url)
+        assert file_content == _download_file_content
+        assert not os.path.isfile(mock_mkstemp[1])
+
+    async def test_download_file_with_cookie(self, _download_file_content):
+        url = 'http://localhost:%d/cookie' % self.listen_port
+        headers = {'Cookie': 'password=deluge'}
+
+        with pytest.raises(Exception):
+            await self.core.download_file(url)
+
+        file_content = await self.core.download_file(url, headers=headers)
+        assert file_content == _download_file_content
+
+    async def test_download_file_with_redirect(self, _download_file_content):
+        url = 'http://localhost:%d/redirect' % self.listen_port
+
+        with pytest.raises(Exception):
+            await self.core.download_file(url, handle_redirects=False)
+
+        file_content = await self.core.download_file(url)
+        assert file_content == _download_file_content
+
+    async def test_download_file_with_callback(self, _download_file_content):
+        url = (
+            'http://localhost:%d/ubuntu-9.04-desktop-i386.iso.torrent'
+            % self.listen_port
+        )
+        called_callback = False
+
+        def on_retrieve_data(data, current_length, total_length):
+            nonlocal called_callback
+            called_callback = True
+            assert data in _download_file_content
+            assert current_length <= total_length
+
+        file_content = await self.core.download_file(url, callback=on_retrieve_data)
+        assert file_content == _download_file_content
+        assert called_callback