Skip to content

Commit 136f130

Browse files
dhalbertdhalbert
authored
Merge pull request adafruit#110 from klocs/chunked_redirect_bug
Proposed fix for issue adafruit#103. Fixes an issue with _throw_away() and large chunked redirects.
2 parents 6e03832 + ee1ddce commit 136f130

File tree

2 files changed

+149
-4
lines changed

2 files changed

+149
-4
lines changed

‎adafruit_requests.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -317,11 +317,15 @@ def _throw_away(self, nbytes: int) -> None:
317317
nbytes -= self._read_from_buffer(nbytes=nbytes)
318318

319319
buf = self._receive_buffer
320-
for _ in range(nbytes // len(buf)):
321-
self._recv_into(buf)
322-
remaining = nbytes % len(buf)
320+
len_buf = len(buf)
321+
for _ in range(nbytes // len_buf):
322+
to_read = len_buf
323+
while to_read > 0:
324+
to_read -= self._recv_into(buf, to_read)
325+
remaining = nbytes % len_buf
323326
if remaining:
324-
self._recv_into(buf, remaining)
327+
while remaining > 0:
328+
remaining -= self._recv_into(buf, remaining)
325329

326330
def close(self) -> None:
327331
"""Drain the remaining ESP socket buffers. We assume we already got what we wanted."""

‎tests/chunked_redirect_test.py

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# SPDX-FileCopyrightText: 2021 ladyada for Adafruit Industries
2+
#
3+
# SPDX-License-Identifier: Unlicense
4+
5+
""" Redirection Tests """
6+
7+
from unittest import mock
8+
import mocket
9+
from chunk_test import _chunk
10+
import adafruit_requests
11+
12+
IP = "1.2.3.4"
13+
HOST = "docs.google.com"
14+
PATH = (
15+
"/spreadsheets/d/e/2PACX-1vR1WjUKz35-ek6SiR5droDfvPp51MTds4wUs57vEZNh2uDfihSTPhTaiiRo"
16+
"vLbNe1mkeRgurppRJ_Zy/pub?output=tsv"
17+
)
18+
19+
# response headers returned from the initial request
20+
HEADERS_REDIRECT = (
21+
b"HTTP/1.1 307 Temporary Redirect\r\n"
22+
b"Content-Type: text/html; charset=UTF-8\r\n"
23+
b"Cache-Control: no-cache, no-store, max-age=0, must-revalidate\r\n"
24+
b"Pragma: no-cache\r\n"
25+
b"Expires: Mon, 01 Jan 1990 00:00:00 GMT\r\n"
26+
b"Date: Sat, 25 Jun 2022 21:08:48 GMT\r\n"
27+
b"Location: https://doc-14-2g-sheets.googleusercontent.com/pub/70cmver1f290kjsnpar5ku2h9g/3"
28+
b"llvt5u8njbvat22m9l19db1h4/1656191325000"
29+
b"/109226138307867586192/*/e@2PACX-1vR1WjUKz35-ek6SiR5droDfvPp51MTds4wUs57vEZNh2uDfihSTPhTai"
30+
b"iRovLbNe1mkeRgurppRJ_Zy?output=tsv\r\n"
31+
b'P3P: CP="This is not a P3P policy! See g.co/p3phelp for more info."\r\n'
32+
b"X-Content-Type-Options: nosniff\r\n"
33+
b"X-XSS-Protection: 1; mode=block\r\n"
34+
b"Server: GSE\r\n"
35+
b"Set-Cookie: NID=511=EcnO010Porg0NIrxM8tSG6MhfQiVtWrQS42CjhKEpzwIvzBj2PFYH0-H_N--EAXaPBkR2j"
36+
b"FjAWEAxIJNqhvKb0vswOWp9hqcCrO51S8kO5I4C3"
37+
b"Is2ctWe1b_ysRU-6hjnJyLHzqjXotAWzEmr_qA3bpqWDwlRaQIiC6SvxM8L0M; expires=Sun, 25-Dec-2022 "
38+
b"21:08:48 GMT; path=/; "
39+
b"domain=.google.com; HttpOnly\r\n"
40+
b'Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-Q050=":443"; '
41+
b'ma=2592000,h3-Q046=":443";'
42+
b' ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"\r\n'
43+
b"Accept-Ranges: none\r\n"
44+
b"Vary: Accept-Encoding\r\n"
45+
b"Transfer-Encoding: chunked\r\n\r\n"
46+
)
47+
48+
# response body returned from the initial request (needs to be chunked.)
49+
BODY_REDIRECT = (
50+
b"<HTML>\n<HEAD>\n<TITLE>Temporary Redirect</TITLE>\n</HEAD>\n"
51+
b'<BODY BGCOLOR="#FFFFFF" TEXT="#000000">\n'
52+
b"<H1>Temporary Redirect</H1>\nThe document has moved "
53+
b'<A HREF="https://doc-14-2g-sheets.googleusercontent.com/pub'
54+
b"/70cmver1f290kjsnpar5ku2h9g/3llvt5u8njbvat22m9l19db1h4/1656191325000"
55+
b"/109226138307867586192/*/e@2PACX-1vR1WjUKz35-"
56+
b"ek6SiR5droDfvPp51MTds4wUs57vEZNh2uDfihSTPhTaiiRovLbNe1mkeRgurppRJ_Zy?"
57+
b'output=tsv">here</A>.\n</BODY>\n</HTML>\n'
58+
)
59+
60+
# response headers from the request to the redirected location
61+
HEADERS = (
62+
b"HTTP/1.1 200 OK\r\n"
63+
b"Content-Type: text/tab-separated-values\r\n"
64+
b"X-Frame-Options: ALLOW-FROM https://docs.google.com\r\n"
65+
b"X-Robots-Tag: noindex, nofollow, nosnippet\r\n"
66+
b"Expires: Sat, 25 Jun 2022 21:08:49 GMT\r\n"
67+
b"Date: Sat, 25 Jun 2022 21:08:49 GMT\r\n"
68+
b"Cache-Control: private, max-age=300\r\n"
69+
b'Content-Disposition: attachment; filename="WeeklyPlanner-Sheet1.tsv"; '
70+
b"filename*=UTF-8''Weekly%20Planner%20-%20Sheet1.tsv\r\n"
71+
b"Access-Control-Allow-Origin: *\r\n"
72+
b"Access-Control-Expose-Headers: Cache-Control,Content-Disposition,Content-Encoding,"
73+
b"Content-Length,Content-Type,Date,Expires,Server,Transfer-Encoding\r\n"
74+
b"Content-Security-Policy: frame-ancestors 'self' https://docs.google.com\r\n"
75+
b"Content-Security-Policy: base-uri 'self';object-src 'self';report-uri https://"
76+
b"doc-14-2g-sheets.googleusercontent.com/spreadsheets/cspreport;"
77+
b"script-src 'report-sample' 'nonce-6V57medLoq3hw2BWeyGu_A' 'unsafe-inline' 'strict-dynamic'"
78+
b" https: http: 'unsafe-eval';worker-src 'self' blob:\r\n"
79+
b"X-Content-Type-Options: nosniff\r\n"
80+
b"X-XSS-Protection: 1; mode=block\r\n"
81+
b"Server: GSE\r\n"
82+
b'Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-Q050=":443"; '
83+
b'ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443";'
84+
b' ma=2592000,quic=":443"; ma=2592000; v="46,43"\r\n'
85+
b"Accept-Ranges: none\r\n"
86+
b"Vary: Accept-Encoding\r\n"
87+
b"Transfer-Encoding: chunked\r\n\r\n"
88+
)
89+
90+
# response body from the request to the redirected location (needs to be chunked.)
91+
BODY = (
92+
b"Sunday\tMonday\tTuesday\tWednesday\tThursday\tFriday\tSaturday\r\n"
93+
b"Rest\tSpin class\tRowing\tWerewolf Twitter\tWeights\tLaundry\tPoke bowl\r\n"
94+
b"\t\tZoom call\tShow & Tell\t\tMow lawn\tSynth Riders\r\n"
95+
b"\t\tTacos\tAsk an Engineer\t\tTrash pickup\t\r\n"
96+
b"\t\t\t\t\tLeg day\t\r\n"
97+
b"\t\t\t\t\tPizza night\t"
98+
)
99+
100+
101+
class MocketRecvInto(mocket.Mocket): # pylint: disable=too-few-public-methods
102+
"""A special Mocket to cap the number of bytes returned from recv_into()"""
103+
104+
def __init__(self, response):
105+
super().__init__(response)
106+
self.recv_into = mock.Mock(side_effect=self._recv_into)
107+
108+
def _recv_into(self, buf, nbytes=0):
109+
assert isinstance(nbytes, int) and nbytes >= 0
110+
read = nbytes if nbytes > 0 else len(buf)
111+
remaining = len(self._response) - self._position
112+
read = min(read, remaining, 205)
113+
end = self._position + read
114+
buf[:read] = self._response[self._position : end]
115+
self._position = end
116+
return read
117+
118+
119+
def do_test_chunked_redirect():
120+
pool = mocket.MocketPool()
121+
pool.getaddrinfo.return_value = ((None, None, None, None, (IP, 443)),)
122+
chunk = _chunk(BODY_REDIRECT, len(BODY_REDIRECT))
123+
chunk2 = _chunk(BODY, len(BODY))
124+
125+
sock1 = MocketRecvInto(HEADERS_REDIRECT + chunk)
126+
sock2 = mocket.Mocket(HEADERS + chunk2)
127+
pool.socket.side_effect = (sock1, sock2)
128+
129+
requests_session = adafruit_requests.Session(pool, mocket.SSLContext())
130+
response = requests_session.get("https://" + HOST + PATH)
131+
132+
sock1.connect.assert_called_once_with((HOST, 443))
133+
sock2.connect.assert_called_once_with(
134+
("doc-14-2g-sheets.googleusercontent.com", 443)
135+
)
136+
137+
assert response.text == str(BODY, "utf-8")
138+
139+
140+
def test_chunked_redirect():
141+
do_test_chunked_redirect()

0 commit comments

Comments
 (0)