From c8ab98d363c60acf54e4940e4fd6ab215492abeb Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sat, 5 Oct 2024 23:28:41 +0200 Subject: [PATCH 01/14] pylint fixes --- lib/inputstreamhelper/utils.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/inputstreamhelper/utils.py b/lib/inputstreamhelper/utils.py index 16b6bd1b..cba8ec62 100644 --- a/lib/inputstreamhelper/utils.py +++ b/lib/inputstreamhelper/utils.py @@ -74,7 +74,7 @@ def download_path(url): def _http_request(url, headers=None, time_out=10): - """Perform an HTTP request and return request""" + """Perform an HTTP request and return the response.""" log(0, 'Request URL: {url}', url=url) try: @@ -82,10 +82,10 @@ def _http_request(url, headers=None, time_out=10): request = Request(url, headers=headers) else: request = Request(url) - req = urlopen(request, timeout=time_out) - log(0, 'Response code: {code}', code=req.getcode()) - if 400 <= req.getcode() < 600: - raise HTTPError('HTTP {} Error for url: {}'.format(req.getcode(), url), response=req) + with urlopen(request, timeout=time_out) as req: + log(0, 'Response code: {code}', code=req.getcode()) + if 400 <= req.getcode() < 600: + raise HTTPError(url, req.getcode(), f'HTTP {req.getcode()} Error for url: {url}', req.headers, req) except (HTTPError, URLError) as err: log(2, 'Download failed with error {}'.format(err)) if yesno_dialog(localize(30004), '{line1}\n{line2}'.format(line1=localize(30063), line2=localize(30065))): # Internet down, try again? @@ -94,7 +94,6 @@ def _http_request(url, headers=None, time_out=10): return req - def http_get(url): """Perform an HTTP GET request and return content""" req = _http_request(url) @@ -106,14 +105,13 @@ def http_get(url): # log(0, 'Response: {response}', response=content) return content.decode("utf-8") - def http_head(url): - """Perform an HTTP HEAD request and return status code""" + """Perform an HTTP HEAD request and return status code.""" req = Request(url) req.get_method = lambda: 'HEAD' try: - resp = urlopen(req) - return resp.getcode() + with urlopen(req) as resp: + return resp.getcode() except HTTPError as exc: return exc.getcode() From fb15e4119caa0b0f84b14b8f589737f3ef5a3735 Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sat, 5 Oct 2024 23:44:00 +0200 Subject: [PATCH 02/14] add more logging in widevines_available_from_repo --- lib/inputstreamhelper/widevine/repo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/inputstreamhelper/widevine/repo.py b/lib/inputstreamhelper/widevine/repo.py index 24cfd8f2..b4a4ec61 100644 --- a/lib/inputstreamhelper/widevine/repo.py +++ b/lib/inputstreamhelper/widevine/repo.py @@ -32,6 +32,8 @@ def widevines_available_from_repo(): http_status = http_head(cdm_url) if http_status == 200: available_cdms.append({'version': cdm_version, 'url': cdm_url}) + continue + log(2, f'Widevine version {cdm_version} is not available from {cdm_url}') if not available_cdms: log(4, "could not find any available cdm in repo") From 122c61d236cdee9e758f407f4d64c998c8badba5 Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sat, 5 Oct 2024 23:49:01 +0200 Subject: [PATCH 03/14] more logging --- lib/inputstreamhelper/widevine/repo.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/inputstreamhelper/widevine/repo.py b/lib/inputstreamhelper/widevine/repo.py index b4a4ec61..5cd4744e 100644 --- a/lib/inputstreamhelper/widevine/repo.py +++ b/lib/inputstreamhelper/widevine/repo.py @@ -19,7 +19,10 @@ def cdm_from_repo(): def widevines_available_from_repo(): """Returns all available Widevine CDM versions and urls from Google's library CDM repository""" - cdm_versions = http_get(config.WIDEVINE_VERSIONS_URL).strip('\n').split('\n') + cdm_versions = http_get(config.WIDEVINE_VERSIONS_URL) + log(0, f"Available Widevine versions from repo: {cdm_versions}") + cdm_versions = cdm_versions.strip('\n').split('\n') + log(0, f"Available Widevine versions from repo: {cdm_versions}") try: cdm_os = config.WIDEVINE_OS_MAP[system_os()] cdm_arch = config.WIDEVINE_ARCH_MAP_REPO[arch()] From 950195ea4fc8d82fdc1899e563ec149c9b4aba1b Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sun, 6 Oct 2024 00:07:10 +0200 Subject: [PATCH 04/14] fix _http_request --- lib/inputstreamhelper/utils.py | 39 ++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/lib/inputstreamhelper/utils.py b/lib/inputstreamhelper/utils.py index cba8ec62..b9b10813 100644 --- a/lib/inputstreamhelper/utils.py +++ b/lib/inputstreamhelper/utils.py @@ -73,37 +73,40 @@ def download_path(url): return os.path.join(temp_path(), filename) + def _http_request(url, headers=None, time_out=10): - """Perform an HTTP request and return the response.""" + """Perform an HTTP request and return the response and content.""" + headers = headers or {} + log(0, 'Request URL: {url}', url=url) + request = Request(url, headers=headers) try: - if headers: - request = Request(url, headers=headers) - else: - request = Request(url) - with urlopen(request, timeout=time_out) as req: - log(0, 'Response code: {code}', code=req.getcode()) - if 400 <= req.getcode() < 600: - raise HTTPError(url, req.getcode(), f'HTTP {req.getcode()} Error for url: {url}', req.headers, req) + with urlopen(request, timeout=time_out) as response: + log(0, 'Response code: {code}', code=response.getcode()) + if 400 <= response.getcode() < 600: + raise HTTPError(url, response.getcode(), f'HTTP {response.getcode()} Error for url: {url}', response.headers, None) + # Read the content inside the `with` block + content = response.read() + return response, content except (HTTPError, URLError) as err: log(2, 'Download failed with error {}'.format(err)) if yesno_dialog(localize(30004), '{line1}\n{line2}'.format(line1=localize(30063), line2=localize(30065))): # Internet down, try again? return _http_request(url, headers, time_out) return None - return req - def http_get(url): - """Perform an HTTP GET request and return content""" - req = _http_request(url) - if req is None: + """Perform an HTTP GET request and return content.""" + response, content = _http_request(url) + if response is None or content is None: return None - content = req.read() - # NOTE: Do not log reponse (as could be large) - # log(0, 'Response: {response}', response=content) - return content.decode("utf-8") + try: + decoded_content = content.decode("utf-8") + return decoded_content + except UnicodeDecodeError as e: + log(2, 'Failed to decode content. Error: {error}', error=str(e)) + return None def http_head(url): """Perform an HTTP HEAD request and return status code.""" From 739a71f2f1eda08b4eedd1cb4b43b74656092307 Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sun, 6 Oct 2024 00:09:08 +0200 Subject: [PATCH 05/14] pylint --- lib/inputstreamhelper/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/inputstreamhelper/utils.py b/lib/inputstreamhelper/utils.py index b9b10813..36f1f89a 100644 --- a/lib/inputstreamhelper/utils.py +++ b/lib/inputstreamhelper/utils.py @@ -104,8 +104,8 @@ def http_get(url): try: decoded_content = content.decode("utf-8") return decoded_content - except UnicodeDecodeError as e: - log(2, 'Failed to decode content. Error: {error}', error=str(e)) + except UnicodeDecodeError as error: + log(2, 'Failed to decode content. Error: {error}', error=str(error)) return None def http_head(url): From f1cb16a4fef9b9dd108a12d642556cce0bcb1e4f Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sun, 6 Oct 2024 00:21:11 +0200 Subject: [PATCH 06/14] try to fix http_download --- lib/inputstreamhelper/utils.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/inputstreamhelper/utils.py b/lib/inputstreamhelper/utils.py index 36f1f89a..667101ce 100644 --- a/lib/inputstreamhelper/utils.py +++ b/lib/inputstreamhelper/utils.py @@ -72,12 +72,10 @@ def download_path(url): return os.path.join(temp_path(), filename) - - def _http_request(url, headers=None, time_out=10): - """Perform an HTTP request and return the response and content.""" - headers = headers or {} - + """Perform an HTTP request and return the response object and content.""" + if headers is None: + headers = {} log(0, 'Request URL: {url}', url=url) request = Request(url, headers=headers) @@ -86,14 +84,14 @@ def _http_request(url, headers=None, time_out=10): log(0, 'Response code: {code}', code=response.getcode()) if 400 <= response.getcode() < 600: raise HTTPError(url, response.getcode(), f'HTTP {response.getcode()} Error for url: {url}', response.headers, None) - # Read the content inside the `with` block content = response.read() + # Return both response object and content return response, content except (HTTPError, URLError) as err: log(2, 'Download failed with error {}'.format(err)) if yesno_dialog(localize(30004), '{line1}\n{line2}'.format(line1=localize(30063), line2=localize(30065))): # Internet down, try again? return _http_request(url, headers, time_out) - return None + return None, None def http_get(url): """Perform an HTTP GET request and return content.""" @@ -131,7 +129,7 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non log(4, 'Invalid hash algorithm specified: {}'.format(hash_alg)) checksum = None - req = _http_request(url) + req, _ = _http_request(url) if req is None: return None @@ -166,7 +164,7 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non return False headers = {'Range': 'bytes={}-{}'.format(size, total_length)} - req = _http_request(url, headers=headers) + req, _ = _http_request(url, headers=headers) if req is None: return None continue From 6076c8d0384c1cb89df84cf337761cab3964319d Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sun, 6 Oct 2024 00:49:43 +0200 Subject: [PATCH 07/14] Revert "try to fix http_download" This reverts commit f1cb16a4fef9b9dd108a12d642556cce0bcb1e4f. --- lib/inputstreamhelper/utils.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/inputstreamhelper/utils.py b/lib/inputstreamhelper/utils.py index 667101ce..36f1f89a 100644 --- a/lib/inputstreamhelper/utils.py +++ b/lib/inputstreamhelper/utils.py @@ -72,10 +72,12 @@ def download_path(url): return os.path.join(temp_path(), filename) + + def _http_request(url, headers=None, time_out=10): - """Perform an HTTP request and return the response object and content.""" - if headers is None: - headers = {} + """Perform an HTTP request and return the response and content.""" + headers = headers or {} + log(0, 'Request URL: {url}', url=url) request = Request(url, headers=headers) @@ -84,14 +86,14 @@ def _http_request(url, headers=None, time_out=10): log(0, 'Response code: {code}', code=response.getcode()) if 400 <= response.getcode() < 600: raise HTTPError(url, response.getcode(), f'HTTP {response.getcode()} Error for url: {url}', response.headers, None) + # Read the content inside the `with` block content = response.read() - # Return both response object and content return response, content except (HTTPError, URLError) as err: log(2, 'Download failed with error {}'.format(err)) if yesno_dialog(localize(30004), '{line1}\n{line2}'.format(line1=localize(30063), line2=localize(30065))): # Internet down, try again? return _http_request(url, headers, time_out) - return None, None + return None def http_get(url): """Perform an HTTP GET request and return content.""" @@ -129,7 +131,7 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non log(4, 'Invalid hash algorithm specified: {}'.format(hash_alg)) checksum = None - req, _ = _http_request(url) + req = _http_request(url) if req is None: return None @@ -164,7 +166,7 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non return False headers = {'Range': 'bytes={}-{}'.format(size, total_length)} - req, _ = _http_request(url, headers=headers) + req = _http_request(url, headers=headers) if req is None: return None continue From 7eb3a46f299f3e9c260610e5ce0e108f7fa702cc Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sun, 6 Oct 2024 00:57:55 +0200 Subject: [PATCH 08/14] update http download logic --- lib/inputstreamhelper/utils.py | 81 ++++++++++++++-------------------- 1 file changed, 32 insertions(+), 49 deletions(-) diff --git a/lib/inputstreamhelper/utils.py b/lib/inputstreamhelper/utils.py index 36f1f89a..40d8f874 100644 --- a/lib/inputstreamhelper/utils.py +++ b/lib/inputstreamhelper/utils.py @@ -72,41 +72,34 @@ def download_path(url): return os.path.join(temp_path(), filename) - - -def _http_request(url, headers=None, time_out=10): - """Perform an HTTP request and return the response and content.""" - headers = headers or {} - - log(0, 'Request URL: {url}', url=url) - request = Request(url, headers=headers) - +def _http_request(url, headers=None, time_out=30): + """Make a robust HTTP request handling redirections.""" try: - with urlopen(request, timeout=time_out) as response: - log(0, 'Response code: {code}', code=response.getcode()) - if 400 <= response.getcode() < 600: - raise HTTPError(url, response.getcode(), f'HTTP {response.getcode()} Error for url: {url}', response.headers, None) - # Read the content inside the `with` block - content = response.read() - return response, content + with urlopen(url, timeout=time_out) as response: + if response.status in [301, 302, 303, 307, 308]: # Handle redirections + new_url = response.getheader('Location') + log(1, f"Redirecting to {new_url}") + return _http_request(new_url, time_out) + return response except (HTTPError, URLError) as err: log(2, 'Download failed with error {}'.format(err)) if yesno_dialog(localize(30004), '{line1}\n{line2}'.format(line1=localize(30063), line2=localize(30065))): # Internet down, try again? return _http_request(url, headers, time_out) return None + except timeout as e: + log(2, f"HTTP request timed out: {e}") + return None def http_get(url): - """Perform an HTTP GET request and return content.""" - response, content = _http_request(url) - if response is None or content is None: + """Perform an HTTP GET request and return content""" + req = _http_request(url) + if req is None: return None - try: - decoded_content = content.decode("utf-8") - return decoded_content - except UnicodeDecodeError as error: - log(2, 'Failed to decode content. Error: {error}', error=str(error)) - return None + content = req.read() + # NOTE: Do not log reponse (as could be large) + # log(0, 'Response: {response}', response=content) + return content.decode("utf-8") def http_head(url): """Perform an HTTP HEAD request and return status code.""" @@ -140,10 +133,13 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non if not message: # display "downloading [filename]" message = localize(30015, filename=filename) # Downloading file - total_length = int(req.info().get('content-length')) + total_length = int(req.info().get('content-length', 0)) + if total_length == 0: + log(2, 'No content-length header available, download may not progress as expected.') + if dl_size and dl_size != total_length: log(2, 'The given file size does not match the request!') - dl_size = total_length # Otherwise size check at end would fail even if dl succeeded + dl_size = total_length if background: progress = bg_progress_dialog() @@ -153,38 +149,27 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non starttime = time() chunk_size = 32 * 1024 + size = 0 with open(compat_path(dl_path), 'wb') as image: - size = 0 - while size < total_length: - try: - chunk = req.read(chunk_size) - except (timeout, SSLError): - req.close() - if not yesno_dialog(localize(30004), '{line1}\n{line2}'.format(line1=localize(30064), - line2=localize(30065))): # Could not finish dl. Try again? - progress.close() - return False - - headers = {'Range': 'bytes={}-{}'.format(size, total_length)} - req = _http_request(url, headers=headers) - if req is None: - return None - continue + while True: + chunk = req.read(chunk_size) + if not chunk: + break image.write(chunk) if checksum: calc_checksum.update(chunk) size += len(chunk) - percent = int(round(size * 100 / total_length)) + percent = int(round(size * 100 / total_length)) if total_length > 0 else 0 if not background and progress.iscanceled(): progress.close() req.close() return False - if time() - starttime > 5: + if time() - starttime > 5 and size > 0: time_left = int(round((total_length - size) * (time() - starttime) / size)) prog_message = '{line1}\n{line2}'.format( line1=message, - line2=localize(30058, mins=time_left // 60, secs=time_left % 60)) # Time remaining + line2=localize(30058, mins=time_left // 60, secs=time_left % 60)) else: prog_message = message @@ -197,15 +182,13 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non size_ok = (not dl_size or stat_file(dl_path).st_size() == dl_size) if not all((checksum_ok, size_ok)): - free_space = sizeof_fmt(diskspace()) log(4, 'Something may be wrong with the downloaded file.') if not checksum_ok: log(4, 'Provided checksum: {}\nCalculated checksum: {}'.format(checksum, calc_checksum.hexdigest())) if not size_ok: free_space = sizeof_fmt(diskspace()) log(4, 'Expected filesize: {}\nReal filesize: {}\nRemaining diskspace: {}'.format(dl_size, stat_file(dl_path).st_size(), free_space)) - - if yesno_dialog(localize(30003), localize(30070, filename=filename)): # file maybe broken. Continue anyway? + if yesno_dialog(localize(30003), localize(30070, filename=filename)): log(4, 'Continuing despite possibly corrupt file!') else: return False From 001e786705915171715998aac03481514cadcb70 Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sun, 6 Oct 2024 01:00:01 +0200 Subject: [PATCH 09/14] pylint --- lib/inputstreamhelper/utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/inputstreamhelper/utils.py b/lib/inputstreamhelper/utils.py index 40d8f874..5cfa5bf8 100644 --- a/lib/inputstreamhelper/utils.py +++ b/lib/inputstreamhelper/utils.py @@ -9,7 +9,6 @@ import struct from functools import total_ordering from socket import timeout -from ssl import SSLError from time import time from typing import NamedTuple from urllib.error import HTTPError, URLError @@ -86,8 +85,8 @@ def _http_request(url, headers=None, time_out=30): if yesno_dialog(localize(30004), '{line1}\n{line2}'.format(line1=localize(30063), line2=localize(30065))): # Internet down, try again? return _http_request(url, headers, time_out) return None - except timeout as e: - log(2, f"HTTP request timed out: {e}") + except timeout as err: + log(2, f"HTTP request timed out: {err}") return None def http_get(url): From 6d937f290cf80cdbea9e6a42798049fd8c9e20eb Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sun, 6 Oct 2024 01:06:11 +0200 Subject: [PATCH 10/14] read content in _http_request --- lib/inputstreamhelper/utils.py | 35 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/lib/inputstreamhelper/utils.py b/lib/inputstreamhelper/utils.py index 5cfa5bf8..f2626b32 100644 --- a/lib/inputstreamhelper/utils.py +++ b/lib/inputstreamhelper/utils.py @@ -71,18 +71,19 @@ def download_path(url): return os.path.join(temp_path(), filename) + def _http_request(url, headers=None, time_out=30): """Make a robust HTTP request handling redirections.""" try: - with urlopen(url, timeout=time_out) as response: - if response.status in [301, 302, 303, 307, 308]: # Handle redirections - new_url = response.getheader('Location') - log(1, f"Redirecting to {new_url}") - return _http_request(new_url, time_out) - return response + response = urlopen(url, timeout=time_out) # pylint: disable=consider-using-with:w + if response.status in [301, 302, 303, 307, 308]: # Handle redirections + new_url = response.getheader('Location') + log(1, f"Redirecting to {new_url}") + return _http_request(new_url, time_out) + return response # Return the response for streaming except (HTTPError, URLError) as err: log(2, 'Download failed with error {}'.format(err)) - if yesno_dialog(localize(30004), '{line1}\n{line2}'.format(line1=localize(30063), line2=localize(30065))): # Internet down, try again? + if yesno_dialog(localize(30004), '{line1}\n{line2}'.format(line1=localize(30063), line2=localize(30065))): return _http_request(url, headers, time_out) return None except timeout as err: @@ -110,8 +111,7 @@ def http_head(url): except HTTPError as exc: return exc.getcode() - -def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=None, background=False): # pylint: disable=too-many-statements +def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=None, background=False): # pylint: disable=too-many-statements """Makes HTTP request and displays a progress dialog on download.""" if checksum: from hashlib import md5, sha1 @@ -123,19 +123,16 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non log(4, 'Invalid hash algorithm specified: {}'.format(hash_alg)) checksum = None - req = _http_request(url) - if req is None: + response = _http_request(url) + if response is None: return None dl_path = download_path(url) filename = os.path.basename(dl_path) - if not message: # display "downloading [filename]" + if not message: message = localize(30015, filename=filename) # Downloading file - total_length = int(req.info().get('content-length', 0)) - if total_length == 0: - log(2, 'No content-length header available, download may not progress as expected.') - + total_length = int(response.info().get('content-length', 0)) if dl_size and dl_size != total_length: log(2, 'The given file size does not match the request!') dl_size = total_length @@ -151,7 +148,7 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non size = 0 with open(compat_path(dl_path), 'wb') as image: while True: - chunk = req.read(chunk_size) + chunk = response.read(chunk_size) if not chunk: break @@ -162,7 +159,7 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non percent = int(round(size * 100 / total_length)) if total_length > 0 else 0 if not background and progress.iscanceled(): progress.close() - req.close() + response.close() return False if time() - starttime > 5 and size > 0: time_left = int(round((total_length - size) * (time() - starttime) / size)) @@ -175,7 +172,7 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non progress.update(percent, prog_message) progress.close() - req.close() + response.close() checksum_ok = (not checksum or calc_checksum.hexdigest() == checksum) size_ok = (not dl_size or stat_file(dl_path).st_size() == dl_size) From f229a68cdac165d97b7db3ca59a91970f6b71e42 Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sun, 6 Oct 2024 11:45:36 +0200 Subject: [PATCH 11/14] remove code for pre-matrix --- lib/inputstreamhelper/kodiutils.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/lib/inputstreamhelper/kodiutils.py b/lib/inputstreamhelper/kodiutils.py index 4408e51a..4fef60c8 100644 --- a/lib/inputstreamhelper/kodiutils.py +++ b/lib/inputstreamhelper/kodiutils.py @@ -3,16 +3,13 @@ """Implements Kodi Helper functions""" from __future__ import absolute_import, division, unicode_literals + from contextlib import contextmanager + import xbmc import xbmcaddon from xbmcgui import DialogProgress, DialogProgressBG - -try: # Kodi v19 or newer - from xbmcvfs import translatePath -except ImportError: # Kodi v18 and older - # pylint: disable=ungrouped-imports - from xbmc import translatePath +from xbmcvfs import translatePath from .unicodes import from_unicode, to_unicode @@ -33,18 +30,10 @@ def __init__(self): def create(self, heading, message=''): # pylint: disable=arguments-differ """Create and show a progress dialog""" - if kodi_version_major() < 19: - lines = message.split('\n', 2) - line1, line2, line3 = (lines + [None] * (3 - len(lines))) - return super(progress_dialog, self).create(heading, line1=line1, line2=line2, line3=line3) return super(progress_dialog, self).create(heading, message=message) def update(self, percent, message=''): # pylint: disable=arguments-differ """Update the progress dialog""" - if kodi_version_major() < 19: - lines = message.split('\n', 2) - line1, line2, line3 = (lines + [None] * (3 - len(lines))) - return super(progress_dialog, self).update(percent, line1=line1, line2=line2, line3=line3) return super(progress_dialog, self).update(percent, message=message) @@ -127,8 +116,6 @@ def ok_dialog(heading='', message=''): from xbmcgui import Dialog if not heading: heading = ADDON.getAddonInfo('name') - if kodi_version_major() < 19: - return Dialog().ok(heading=heading, line1=message) return Dialog().ok(heading=heading, message=message) @@ -155,8 +142,6 @@ def yesno_dialog(heading='', message='', nolabel=None, yeslabel=None, autoclose= from xbmcgui import Dialog if not heading: heading = ADDON.getAddonInfo('name') - if kodi_version_major() < 19: - return Dialog().yesno(heading=heading, line1=message, nolabel=nolabel, yeslabel=yeslabel, autoclose=autoclose) return Dialog().yesno(heading=heading, message=message, nolabel=nolabel, yeslabel=yeslabel, autoclose=autoclose) From 3c52011a59f3314ed903572d62eceef7373dca16 Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sun, 6 Oct 2024 12:43:07 +0200 Subject: [PATCH 12/14] flake8/pylint fixes --- lib/inputstreamhelper/api.py | 1 + lib/inputstreamhelper/config.py | 1 - lib/inputstreamhelper/kodiutils.py | 3 ++- lib/inputstreamhelper/utils.py | 8 +++++--- lib/inputstreamhelper/widevine/arm.py | 9 ++++++--- lib/inputstreamhelper/widevine/arm_chromeos.py | 5 +++-- lib/inputstreamhelper/widevine/widevine.py | 2 +- tests/xbmcgui.py | 6 ++++-- 8 files changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/inputstreamhelper/api.py b/lib/inputstreamhelper/api.py index 941f975e..0a30950e 100644 --- a/lib/inputstreamhelper/api.py +++ b/lib/inputstreamhelper/api.py @@ -3,6 +3,7 @@ """This is the actual InputStream Helper API script""" from __future__ import absolute_import, division, unicode_literals + from . import Helper from .kodiutils import ADDON, log diff --git a/lib/inputstreamhelper/config.py b/lib/inputstreamhelper/config.py index bd5ffe3b..6e0edf0d 100644 --- a/lib/inputstreamhelper/config.py +++ b/lib/inputstreamhelper/config.py @@ -3,7 +3,6 @@ """Configuration variables for inpustreamhelper""" from __future__ import absolute_import, division, unicode_literals - INPUTSTREAM_PROTOCOLS = { 'mpd': 'inputstream.adaptive', 'ism': 'inputstream.adaptive', diff --git a/lib/inputstreamhelper/kodiutils.py b/lib/inputstreamhelper/kodiutils.py index 4fef60c8..548da1f6 100644 --- a/lib/inputstreamhelper/kodiutils.py +++ b/lib/inputstreamhelper/kodiutils.py @@ -94,7 +94,8 @@ def addon_version(addon_name=None): return get_addon_info('version', addon) -def browsesingle(type, heading, shares='', mask='', useThumbs=False, treatAsFolder=False, defaultt=None): # pylint: disable=invalid-name,redefined-builtin +# pylint: disable=invalid-name,redefined-builtin,too-many-positional-arguments +def browsesingle(type, heading, shares='', mask='', useThumbs=False, treatAsFolder=False, defaultt=None): """Show a Kodi browseSingle dialog""" from xbmcgui import Dialog if not heading: diff --git a/lib/inputstreamhelper/utils.py b/lib/inputstreamhelper/utils.py index f2626b32..85d686e6 100644 --- a/lib/inputstreamhelper/utils.py +++ b/lib/inputstreamhelper/utils.py @@ -75,7 +75,7 @@ def download_path(url): def _http_request(url, headers=None, time_out=30): """Make a robust HTTP request handling redirections.""" try: - response = urlopen(url, timeout=time_out) # pylint: disable=consider-using-with:w + response = urlopen(url, timeout=time_out) # pylint: disable=consider-using-with:w if response.status in [301, 302, 303, 307, 308]: # Handle redirections new_url = response.getheader('Location') log(1, f"Redirecting to {new_url}") @@ -90,17 +90,18 @@ def _http_request(url, headers=None, time_out=30): log(2, f"HTTP request timed out: {err}") return None + def http_get(url): """Perform an HTTP GET request and return content""" req = _http_request(url) if req is None: return None - content = req.read() # NOTE: Do not log reponse (as could be large) # log(0, 'Response: {response}', response=content) return content.decode("utf-8") + def http_head(url): """Perform an HTTP HEAD request and return status code.""" req = Request(url) @@ -111,7 +112,8 @@ def http_head(url): except HTTPError as exc: return exc.getcode() -def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=None, background=False): # pylint: disable=too-many-statements + +def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=None, background=False): # pylint: disable=too-many-statements """Makes HTTP request and displays a progress dialog on download.""" if checksum: from hashlib import md5, sha1 diff --git a/lib/inputstreamhelper/widevine/arm.py b/lib/inputstreamhelper/widevine/arm.py index c8a0a49c..a0c9acbe 100644 --- a/lib/inputstreamhelper/widevine/arm.py +++ b/lib/inputstreamhelper/widevine/arm.py @@ -3,12 +3,15 @@ """Implements ARM specific widevine functions""" from __future__ import absolute_import, division, unicode_literals -import os + import json +import os from .. import config -from ..kodiutils import browsesingle, localize, log, ok_dialog, open_file, progress_dialog, yesno_dialog -from ..utils import diskspace, http_download, http_get, parse_version, sizeof_fmt, system_os, update_temp_path, userspace64 +from ..kodiutils import (browsesingle, localize, log, ok_dialog, open_file, + progress_dialog, yesno_dialog) +from ..utils import (diskspace, http_download, http_get, parse_version, + sizeof_fmt, system_os, update_temp_path, userspace64) from .arm_chromeos import ChromeOSImage from .arm_lacros import cdm_from_lacros, install_widevine_arm_lacros diff --git a/lib/inputstreamhelper/widevine/arm_chromeos.py b/lib/inputstreamhelper/widevine/arm_chromeos.py index 9a334d10..8d1621f7 100644 --- a/lib/inputstreamhelper/widevine/arm_chromeos.py +++ b/lib/inputstreamhelper/widevine/arm_chromeos.py @@ -3,13 +3,14 @@ """Implements a class with methods related to the Chrome OS image""" from __future__ import absolute_import, division, unicode_literals + import os +from io import UnsupportedOperation from struct import calcsize, unpack from zipfile import ZipFile -from io import UnsupportedOperation -from ..kodiutils import exists, localize, log, mkdirs from .. import config +from ..kodiutils import exists, localize, log, mkdirs from ..unicodes import compat_path diff --git a/lib/inputstreamhelper/widevine/widevine.py b/lib/inputstreamhelper/widevine/widevine.py index a480fd4f..0204c69f 100644 --- a/lib/inputstreamhelper/widevine/widevine.py +++ b/lib/inputstreamhelper/widevine/widevine.py @@ -40,7 +40,7 @@ def widevine_eula(): cdm_arch = config.WIDEVINE_ARCH_MAP_REPO[arch()] else: # Grab the license from the x86 files log(0, 'Acquiring Widevine EULA from x86 files.') - cdm_version = '4.10.2830.0' # fine to hardcode as it's only used for the EULA + cdm_version = '4.10.2830.0' # fine to hardcode as it's only used for the EULA cdm_os = 'mac' cdm_arch = 'x64' diff --git a/tests/xbmcgui.py b/tests/xbmcgui.py index 4bad5527..f5f1df60 100644 --- a/tests/xbmcgui.py +++ b/tests/xbmcgui.py @@ -3,11 +3,13 @@ # GNU General Public License v3.0 (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) """This file implements the Kodi xbmcgui module, either using stubs or alternative functionality""" -# flake8: noqa: FI14; pylint: disable=invalid-name,super-on-old-class,too-few-public-methods,too-many-arguments,unused-argument,useless-super-delegation +# flake8: noqa: FI14; pylint: disable=invalid-name,super-on-old-class,too-few-public-methods,too-many-arguments,unused-argument,useless-super-delegation,too-many-positional-arguments from __future__ import absolute_import, division, print_function + import sys -from xbmcextra import kodi_to_ansi + +from .xbmcextra import kodi_to_ansi class Control: From 611a32764fd30a0aac9b3eb82a536f89d482653c Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sun, 6 Oct 2024 12:44:48 +0200 Subject: [PATCH 13/14] fix import --- tests/xbmcgui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/xbmcgui.py b/tests/xbmcgui.py index f5f1df60..6405a8ae 100644 --- a/tests/xbmcgui.py +++ b/tests/xbmcgui.py @@ -9,7 +9,7 @@ import sys -from .xbmcextra import kodi_to_ansi +from xbmcextra import kodi_to_ansi class Control: From e57850e86a781599a8fc4cb642cb03cf676504d5 Mon Sep 17 00:00:00 2001 From: emilsvennesson Date: Sun, 6 Oct 2024 13:06:00 +0200 Subject: [PATCH 14/14] more pylint fixes --- lib/inputstreamhelper/utils.py | 110 +++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 34 deletions(-) diff --git a/lib/inputstreamhelper/utils.py b/lib/inputstreamhelper/utils.py index 85d686e6..434b879e 100644 --- a/lib/inputstreamhelper/utils.py +++ b/lib/inputstreamhelper/utils.py @@ -113,17 +113,12 @@ def http_head(url): return exc.getcode() -def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=None, background=False): # pylint: disable=too-many-statements +# pylint: disable=too-many-positional-arguments +def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=None, background=False): """Makes HTTP request and displays a progress dialog on download.""" - if checksum: - from hashlib import md5, sha1 - if hash_alg == 'sha1': - calc_checksum = sha1() - elif hash_alg == 'md5': - calc_checksum = md5() - else: - log(4, 'Invalid hash algorithm specified: {}'.format(hash_alg)) - checksum = None + calc_checksum = _initialize_checksum(checksum, hash_alg) + if checksum and not calc_checksum: + checksum = None response = _http_request(url) if response is None: @@ -139,15 +134,51 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non log(2, 'The given file size does not match the request!') dl_size = total_length - if background: - progress = bg_progress_dialog() - else: - progress = progress_dialog() + progress = _create_progress_dialog(background, message) + + success = _download_file(response, dl_path, calc_checksum, total_length, message, progress, background) + + progress.close() + response.close() + + if not success: + return False + + checksum_ok = _verify_checksum(checksum, calc_checksum) + size_ok = _verify_size(dl_size, dl_path) + + if not checksum_ok or not size_ok: + if not _handle_corrupt_file(dl_size, dl_path, checksum, calc_checksum, filename): + return False + + return dl_path + + +def _initialize_checksum(checksum, hash_alg): + if not checksum: + return None + + from hashlib import md5, sha1 + if hash_alg == 'sha1': + return sha1() + if hash_alg == 'md5': + return md5() + log(4, 'Invalid hash algorithm specified: {}'.format(hash_alg)) + return None + + +def _create_progress_dialog(background, message): + progress = bg_progress_dialog() if background else progress_dialog() progress.create(localize(30014), message=message) # Download in progress + return progress + +# pylint: disable=too-many-positional-arguments +def _download_file(response, dl_path, calc_checksum, total_length, message, progress, background): starttime = time() chunk_size = 32 * 1024 size = 0 + with open(compat_path(dl_path), 'wb') as image: while True: chunk = response.read(chunk_size) @@ -155,14 +186,14 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non break image.write(chunk) - if checksum: + if calc_checksum: calc_checksum.update(chunk) size += len(chunk) percent = int(round(size * 100 / total_length)) if total_length > 0 else 0 + if not background and progress.iscanceled(): - progress.close() - response.close() return False + if time() - starttime > 5 and size > 0: time_left = int(round((total_length - size) * (time() - starttime) / size)) prog_message = '{line1}\n{line2}'.format( @@ -173,25 +204,36 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non progress.update(percent, prog_message) - progress.close() - response.close() + return True - checksum_ok = (not checksum or calc_checksum.hexdigest() == checksum) - size_ok = (not dl_size or stat_file(dl_path).st_size() == dl_size) - - if not all((checksum_ok, size_ok)): - log(4, 'Something may be wrong with the downloaded file.') - if not checksum_ok: - log(4, 'Provided checksum: {}\nCalculated checksum: {}'.format(checksum, calc_checksum.hexdigest())) - if not size_ok: - free_space = sizeof_fmt(diskspace()) - log(4, 'Expected filesize: {}\nReal filesize: {}\nRemaining diskspace: {}'.format(dl_size, stat_file(dl_path).st_size(), free_space)) - if yesno_dialog(localize(30003), localize(30070, filename=filename)): - log(4, 'Continuing despite possibly corrupt file!') - else: - return False - return dl_path +def _verify_checksum(checksum, calc_checksum): + if not checksum: + return True + if calc_checksum: + return calc_checksum.hexdigest() == checksum + return False + + +def _verify_size(dl_size, dl_path): + if not dl_size: + return True + return stat_file(dl_path).st_size() == dl_size + + +def _handle_corrupt_file(dl_size, dl_path, checksum, calc_checksum, filename): + log(4, 'Something may be wrong with the downloaded file.') + if checksum and calc_checksum: + log(4, 'Provided checksum: {}\nCalculated checksum: {}'.format(checksum, calc_checksum.hexdigest())) + if dl_size: + free_space = sizeof_fmt(diskspace()) + log(4, 'Expected filesize: {}\nReal filesize: {}\nRemaining diskspace: {}'.format( + dl_size, stat_file(dl_path).st_size(), free_space)) + if yesno_dialog(localize(30003), localize(30070, filename=filename)): + log(4, 'Continuing despite possibly corrupt file!') + return True + + return False def unzip(source, destination, file_to_unzip=None, result=[]): # pylint: disable=dangerous-default-value