From 003887a13ec461adba332ef651aaa2878961a04f Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 28 Jun 2024 00:24:53 +0100 Subject: [PATCH] fixed+enabled recently disabled tests for download + compressed bodies --- httpie/downloads.py | 14 +++--- httpie/output/streams.py | 7 +++ tests/test_downloads.py | 104 ++++++++++++++++++++------------------- 3 files changed, 68 insertions(+), 57 deletions(-) diff --git a/httpie/downloads.py b/httpie/downloads.py index 4f6f75eed6..805d95918b 100644 --- a/httpie/downloads.py +++ b/httpie/downloads.py @@ -301,11 +301,11 @@ def failed(self): def is_interrupted(self) -> bool: return self.status.is_interrupted - def chunk_downloaded(self, chunk_or_new_total: Union[bytes, int]): + def chunk_downloaded(self, chunk_or_new_total: Union[bytes, int]) -> None: """ A download progress callback. - :param chunk: A chunk of response body data that has just + :param chunk_or_new_total: A chunk of response body data that has just been downloaded and written to the output. """ @@ -372,14 +372,16 @@ def start_display(self, output_file): else: has_reliable_total = self.total_size is not None + if self.decoded_from: + encodings = ', '.join(f'`{enc}`' for enc in self.decoded_from) + message_suffix = DECODED_FROM_SUFFIX.format(encodings=encodings) + summary_suffix = DECODED_SIZE_NOTE_SUFFIX + if has_reliable_total: progress_display_class = ProgressDisplayFull else: - if self.decoded_from: - encodings = ', '.join(f'`{enc}`' for enc in self.decoded_from) - message_suffix = DECODED_FROM_SUFFIX.format(encodings=encodings) - summary_suffix = DECODED_SIZE_NOTE_SUFFIX progress_display_class = ProgressDisplayNoTotal + self.display = progress_display_class( env=self.env, total_size=self.total_size, diff --git a/httpie/output/streams.py b/httpie/output/streams.py index 83d1c5673e..5c1171336a 100644 --- a/httpie/output/streams.py +++ b/httpie/output/streams.py @@ -79,8 +79,15 @@ def __iter__(self) -> Iterable[bytes]: # Useful when the remote compress the body. We use the "untouched" amt of data to determine # the download speed. if hasattr(self.msg, "_orig") and hasattr(self.msg._orig, "download_progress") and self.msg._orig.download_progress: + # this is plan A: using public interfaces! self.on_body_chunk_downloaded(self.msg._orig.download_progress.total) + elif hasattr(self.msg, "_orig") and hasattr(self.msg._orig, "raw") and hasattr(self.msg._orig.raw, "_fp_bytes_read"): + # plan B, falling back on a private property that may disapear from urllib3-future... + # this case is mandatory due to how the mocking library works. it does not use any "socket" but + # rather a simple io.BytesIO. + self.on_body_chunk_downloaded(self.msg._orig.raw._fp_bytes_read) else: + # well. this case will certainly cause issues if the body is compressed. self.on_body_chunk_downloaded(chunk) except DataSuppressedError as e: if self.output_options.headers: diff --git a/tests/test_downloads.py b/tests/test_downloads.py index 4747fa399b..198dda4b29 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -1,6 +1,7 @@ import os import tempfile import time +import zlib from unittest import mock from urllib.request import urlopen @@ -15,7 +16,7 @@ ContentRangeError, Downloader, PARTIAL_CONTENT, - DECODED_SIZE_NOTE_SUFFIX, + DECODED_SIZE_NOTE_SUFFIX, DECODED_FROM_SUFFIX, ) from niquests.structures import CaseInsensitiveDict from .utils import http, MockEnvironment, cd_clean_tmp_dir, DUMMY_URL @@ -264,7 +265,7 @@ def test_download_gzip_content_encoding(self, httpbin): @responses.activate def test_incomplete_response(self): # We have incompleteness checks in the downloader, but it might not be needed as it’s built into (ni|req)uests. - error_msg = 'peer closed connection without sending complete message body (received 2 bytes, expected 1 more)' + error_msg = 'IncompleteRead(2 bytes read, 1 more expected)' responses.add( method=responses.GET, url=DUMMY_URL, @@ -281,55 +282,56 @@ def test_incomplete_response(self): class TestDecodedDownloads: """Test downloading responses with `Content-Encoding`""" - # todo: find an appropriate way to mock compressed bodies within those tests. - # @responses.activate - # def test_decoded_response_no_content_length(self): - # responses.add( - # method=responses.GET, - # url=DUMMY_URL, - # headers={ - # 'Content-Encoding': 'gzip, br', - # }, - # body='123', - # ) - # with cd_clean_tmp_dir(): - # r = http('--download', '--headers', DUMMY_URL) - # print(r.stderr) - # assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr - # assert DECODED_SIZE_NOTE_SUFFIX in r.stderr - # - # @responses.activate - # def test_decoded_response_with_content_length(self): - # responses.add( - # method=responses.GET, - # url=DUMMY_URL, - # headers={ - # 'Content-Encoding': 'gzip, br', - # 'Content-Length': '3', - # }, - # body='123', - # ) - # with cd_clean_tmp_dir(): - # r = http('--download', DUMMY_URL) - # print(r.stderr) - # assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr - # assert DECODED_SIZE_NOTE_SUFFIX in r.stderr - # - # @responses.activate - # def test_decoded_response_without_content_length(self): - # responses.add( - # method=responses.GET, - # url=DUMMY_URL, - # headers={ - # 'Content-Encoding': 'gzip, br', - # }, - # body='123', - # ) - # with cd_clean_tmp_dir(): - # r = http('--download', DUMMY_URL) - # print(r.stderr) - # assert DECODED_FROM_SUFFIX.format(encodings='`gzip`, `br`') in r.stderr - # assert DECODED_SIZE_NOTE_SUFFIX in r.stderr + @responses.activate + def test_decoded_response_no_content_length(self): + responses.add( + method=responses.GET, + url=DUMMY_URL, + headers={ + 'Content-Encoding': 'deflate', + }, + body=zlib.compress(b"foobar"), + ) + with cd_clean_tmp_dir(): + r = http('--download', '--headers', DUMMY_URL) + print(r.stderr) + assert DECODED_FROM_SUFFIX.format(encodings='`deflate`') in r.stderr + assert DECODED_SIZE_NOTE_SUFFIX in r.stderr + + @responses.activate + def test_decoded_response_with_content_length(self): + payload = zlib.compress(b"foobar") + + responses.add( + method=responses.GET, + url=DUMMY_URL, + headers={ + 'Content-Encoding': 'deflate', + 'Content-Length': str(len(payload)), + }, + body=payload, + ) + with cd_clean_tmp_dir(): + r = http('--download', DUMMY_URL) + print(r.stderr) + assert DECODED_FROM_SUFFIX.format(encodings='`deflate`') in r.stderr + assert DECODED_SIZE_NOTE_SUFFIX in r.stderr + + @responses.activate + def test_decoded_response_without_content_length(self): + responses.add( + method=responses.GET, + url=DUMMY_URL, + headers={ + 'Content-Encoding': 'deflate', + }, + body=zlib.compress(b'foobar'), + ) + with cd_clean_tmp_dir(): + r = http('--download', DUMMY_URL) + print(r.stderr) + assert DECODED_FROM_SUFFIX.format(encodings='`deflate`') in r.stderr + assert DECODED_SIZE_NOTE_SUFFIX in r.stderr @responses.activate def test_non_decoded_response_without_content_length(self):