From 367116a6224b5a2789499c5ed551f49eb4b81f97 Mon Sep 17 00:00:00 2001 From: teamblue-e2 Date: Fri, 5 Jan 2024 13:55:09 +0100 Subject: [PATCH] Update Downloader.py to be http 1.1 conform --- lib/python/Screens/FlashImage.py | 4 +- lib/python/Tools/Downloader.py | 152 ++++++++++++++++--------------- 2 files changed, 79 insertions(+), 77 deletions(-) diff --git a/lib/python/Screens/FlashImage.py b/lib/python/Screens/FlashImage.py index dc88fa28c7..d81ddeaa31 100644 --- a/lib/python/Screens/FlashImage.py +++ b/lib/python/Screens/FlashImage.py @@ -397,11 +397,11 @@ def startDownload(self, reply=True): def downloadProgress(self, current, total): self["progress"].setValue(int(100 * current / total)) - def downloadError(self, reason, status): + def downloadError(self, reason): self.downloader.stop() self.session.openWithCallback(self.abort, MessageBox, _("Error during downloading image\n%s\n%s") % (self.imagename, reason), type=MessageBox.TYPE_ERROR, simple=True) - def downloadEnd(self): + def downloadEnd(self,filename=None): self.downloader.stop() self.unzip() diff --git a/lib/python/Tools/Downloader.py b/lib/python/Tools/Downloader.py index 656b556885..f85d0c2cd3 100644 --- a/lib/python/Tools/Downloader.py +++ b/lib/python/Tools/Downloader.py @@ -1,89 +1,91 @@ -# -*- coding: utf-8 -*- -from twisted.web import client -from twisted.internet import reactor, defer, ssl -# from urlparse import urlparse +from os import unlink +import requests +from twisted.internet import reactor +from urllib.request import urlopen, Request +from enigma import eTimer -class HTTPProgressDownloader(client.HTTPDownloader): - def __init__(self, url, outfile, headers=None): - client.HTTPDownloader.__init__(self, url, outfile, headers=headers, agent="Enigma2 HbbTV/1.1.1 (+PVR+RTSP+DL;teamBlue;;;)") - self.status = self.progress_callback = self.error_callback = self.end_callback = None - self.deferred = defer.Deferred() - def noPage(self, reason): - if self.status == b"304": - client.HTTPDownloader.page(self, "") - else: - client.HTTPDownloader.noPage(self, reason) - if self.error_callback: - self.error_callback(reason.getErrorMessage(), self.status) +class DownloadWithProgress: + def __init__(self, url, outputFile): + self.url = url + self.outputFile = outputFile + self.userAgent = "HbbTV/1.1.1 (+PVR+RTSP+DL; Sonic; OpenHDF; TV44; 1.32.455; 2.002) Bee/3.5" + # self.agent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5" + self.totalSize = 0 + self.progress = 0 + self.progressCallback = None + self.endCallback = None + self.errorCallback = None + self.stopFlag = False + self.timer = eTimer() + self.timer.callback.append(self.reportProgress) - def gotHeaders(self, headers): - if self.status == b"200": - if b"content-length" in headers: - self.totalbytes = int(headers[b"content-length"][0]) - else: - self.totalbytes = 0 - self.currentbytes = 0.0 - return client.HTTPDownloader.gotHeaders(self, headers) + def start(self): + try: + request = Request(self.url, None, {"User-agent": self.userAgent}) + feedFile = urlopen(request) + metaData = feedFile.headers + self.totalSize = int(metaData.get("Content-Length", 0)) + # Set the transfer block size to a minimum of 1K and a maximum of 1% of the file size (or 128KB if the size is unknown) else use 64K. + self.blockSize = max(min(self.totalSize // 100, 1024), 131071) if self.totalSize else 65536 + except OSError as err: + if self.errorCallback: + self.errorCallback(err) + return self + reactor.callInThread(self.run) + return self - def pagePart(self, packet): - if self.status == b"200": - self.currentbytes += len(packet) - if self.totalbytes and self.progress_callback: - self.progress_callback(self.currentbytes, self.totalbytes) - return client.HTTPDownloader.pagePart(self, packet) + def run(self): + # requests.Response object = requests.get(url, params=None, allow_redirects=True, auth=None, cert=None, cookies=None, headers=None, proxies=None, stream=False, timeout=None, verify=True) + response = requests.get(self.url, headers={"User-agent": self.userAgent}, stream=True) # Streaming, so we can iterate over the response. + try: + with open(self.outputFile, "wb") as fd: + for buffer in response.iter_content(self.blockSize): + if self.stopFlag: + response.close() + fd.close() + unlink(self.outputFile) + return True + self.progress += len(buffer) + if self.progressCallback: + self.timer.start(0, True) + fd.write(buffer) + if self.endCallback: + self.endCallback(self.outputFile) + except OSError as err: + if self.errorCallback: + self.errorCallback(err) + return False - def pageEnd(self): - ret = client.HTTPDownloader.pageEnd(self) - if self.end_callback: - self.end_callback() - return ret + def stop(self): + self.stopFlag = True + def reportProgress(self): + self.progressCallback(self.progress, self.totalSize) -class downloadWithProgress: - def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs): - if hasattr(client, '_parse'): - scheme, host, port, path = client._parse(url) - else: - # _URI class renamed to URI in 15.0.0 - try: - from twisted.web.client import _URI as URI - except ImportError: - from twisted.web.client import URI - # twisted wants bytes - if isinstance(url, str): - url = url.encode("UTF-8") - uri = URI.fromBytes(url) - scheme = uri.scheme - host = uri.host - port = uri.port - path = uri.path -# ======= another twisted fix possibility -# parsed = urlparse(url) -# scheme = parsed.scheme -# host = parsed.hostname -# port = parsed.port or (443 if scheme == 'https' else 80) + def addProgress(self, progressCallback): + self.progressCallback = progressCallback - self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs) - if scheme == b'https': - self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory()) - else: - self.connection = reactor.connectTCP(host, port, self.factory) + def addEnd(self, endCallback): + self.endCallback = endCallback - def start(self): - return self.factory.deferred + def addError(self, errorCallback): + self.errorCallback = errorCallback - def stop(self): - if self.connection: - self.factory.progress_callback = self.factory.end_callback = self.factory.error_callback = None - self.connection.disconnect() + def setAgent(self, userAgent): + self.userAgent = userAgent + + def addErrback(self, errorCallback): # Temporary supprt for deprecated callbacks. + print("[Downloader] Warning: DownloadWithProgress 'addErrback' is deprecated use 'addError' instead!") + self.errorCallback = errorCallback + return self - def addProgress(self, progress_callback): - self.factory.progress_callback = progress_callback + def addCallback(self, endCallback): # Temporary supprt for deprecated callbacks. + print("[Downloader] Warning: DownloadWithProgress 'addCallback' is deprecated use 'addEnd' instead!") + self.endCallback = endCallback + return self - def addEnd(self, end_callback): - self.factory.end_callback = end_callback - def addError(self, error_callback): - self.factory.error_callback = error_callback +class downloadWithProgress(DownloadWithProgress): # Class names should start with a Capital letter, this catches old code until that code can be updated. + pass