Fix a bug for header field getting malformed when it extends over multiple lines

tests/test_http.py

@@ -28,12 +28,28 @@ def test_headers_raw_dict_none(self):
         self.assertIsNone(headers_dict_to_raw(None))
 
     def test_headers_raw_to_dict(self):
-        raw = b"Content-type: text/html\n\rAccept: gzip\n\r\
-                Cache-Control: no-cache\n\rCache-Control: no-store\n\n"
-        dct = {b'Content-type': [b'text/html'], b'Accept': [b'gzip'], 
+        raw = b'\r\n'.join((b"Content-type: text/html",
+                            b"Accept: gzip",
+                            b"Cache-Control: no-cache",
+                            b"Cache-Control: no-store"))
+        dct = {b'Content-type': [b'text/html'], b'Accept': [b'gzip'],
                b'Cache-Control': [b'no-cache', b'no-store']}
         self.assertEqual(headers_raw_to_dict(raw), dct)
 
+    def test_headers_raw_to_dict_multiline(self):
+        raw = b'\r\n'.join((b'Content-Type: multipart/related;',
+                            b'  type="application/xop+xml";',
+                            b'\tboundary="example"',
+                            b'Cache-Control: no-cache'))
+        dct = {
+            b'Content-Type': [
+                b'\r\n'.join((b'multipart/related;',
+                              b'  type="application/xop+xml";',
+                              b'\tboundary="example"'))
+            ],
+            b'Cache-Control': [b'no-cache']}
+        self.assertEqual(headers_raw_to_dict(raw), dct)
+
     def test_headers_dict_to_raw(self):
         dct = OrderedDict([
             (b'Content-type', b'text/html'),

w3lib/http.py

@@ -27,7 +27,17 @@ def headers_raw_to_dict(headers_raw):
 
     if headers_raw is None:
         return None
-    headers = headers_raw.splitlines()
+
+    headers = []
+    for line in headers_raw.split(b'\r\n'):
+        if line.startswith(b' ') or line.startswith(b'\t'):
+            try:
+                headers[-1] += (b'\r\n' + line)
+            except IndexError:
+                raise ValueError('Malformed raw headers')
+        else:
+            headers.append(line)
+
     headers_tuples = [header.split(b':', 1) for header in headers]
 
     result_dict = {}