Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix “not enough values to unpack” when parsing headers #5911

Draft
wants to merge 6 commits into
base: master
Choose a base branch
from
Draft

Fix “not enough values to unpack” when parsing headers #5911

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e75e496
Debugging changes
Gallaecio Apr 25, 2023
09bf7e9
Finalize the fix
Gallaecio May 4, 2023
d53851d
Use private names for our customizations
Gallaecio May 4, 2023
06f0437
Add a test for long headers
Gallaecio May 4, 2023
f621a81
Merge remote-tracking branch 'scrapy/master' into fix-355
Gallaecio Aug 31, 2023
5f45494
Take the monkey-patching route
Gallaecio Aug 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 6 additions & 0 deletions scrapy/core/downloader/handlers/http11.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.error import TimeoutError
from twisted.python.failure import Failure
from twisted.web._newclient import HTTPClientParser
from twisted.web.client import (
URI,
Agent,
Expand All @@ -35,6 +36,11 @@
logger = logging.getLogger(__name__)


# Monkey-patch to increase the maximum length for (header) lines, which is
# 2**14 by default as of Twisted 22.10.0.
HTTPClientParser.MAX_LENGTH = max(2**16, HTTPClientParser.MAX_LENGTH)


class HTTP11DownloadHandler:
lazy = False

Expand Down
17 changes: 17 additions & 0 deletions tests/test_downloader_handlers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,12 @@ def render(self, request):
return b""


class LongHeaderResource(resource.Resource):
def render(self, request):
request.responseHeaders.setRawHeaders(b"a", [b"a" * 2**15])
return b""


class HttpTestCase(unittest.TestCase):
scheme = "http"
download_handler_cls: Type = HTTPDownloadHandler
Expand All @@ -241,6 +247,7 @@ def setUp(self):
r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())
r.putChild(b"largechunkedfile", LargeChunkedFileResource())
r.putChild(b"duplicate-header", DuplicateHeaderResource())
r.putChild(b"long-header", LongHeaderResource())
r.putChild(b"echo", Echo())
self.site = server.Site(r, timeout=None)
self.wrapper = WrappingFactory(self.site)
Expand Down Expand Up @@ -424,6 +431,13 @@ def _test(response):
request = Request(self.getURL("duplicate-header"))
return self.download_request(request, Spider("foo")).addCallback(_test)

def test_get_long_header(self):
def _test(response):
self.assertEqual(response.headers.get(b"a"), b"a" * 2**15)

request = Request(self.getURL("long-header"))
return self.download_request(request, Spider("foo")).addCallback(_test)


class Http10TestCase(HttpTestCase):
"""HTTP 1.0 test case"""
Expand All @@ -437,6 +451,9 @@ def test_protocol(self):
d.addCallback(self.assertEqual, "HTTP/1.0")
return d

def test_get_long_header(self):
raise unittest.SkipTest("Scrapy does not support long headers on HTTP/1.0")


class Https10TestCase(Http10TestCase):
scheme = "https"
Expand Down