Merge pull request #124 from tomato42/codec-speedup

# Conflicts:
#	tlslite/utils/codec.py

.codeclimate.yml

@@ -1,10 +1,16 @@
 engines:
   radon:
     enabled: true
+    config:
+      threshold: "C"
   pep8:
     enabled: true
   duplication:
     enabled: true
+    exclude_fingerprints:
+    # duplication in tlslite/utils/codec.py Writer.addTwo() .addFour()
+    - 2e783666ce368f4223c1e7f5b162e2d9
+    - 2c398389f33ea2572edefc5370ed49c0
     config:
       languages:
       - python

tlslite/extensions.py

@@ -159,9 +159,10 @@ def write(self):
         assert self.extType is not None
 
         w = Writer()
-        w.add(self.extType, 2)
-        w.add(len(self.extData), 2)
-        w.addFixSeq(self.extData, 1)
+        w.addTwo(self.extType)
+        data = self.extData
+        w.addTwo(len(data))
+        w.bytes += data
         return w.bytes
 
     @staticmethod

tlslite/messages.py

@@ -636,7 +636,7 @@ def _write(self):
         w = Writer()
         w.add(self.client_version[0], 1)
         w.add(self.client_version[1], 1)
-        w.addFixSeq(self.random, 1)
+        w.bytes += self.random
         w.addVarSeq(self.session_id, 1, 1)
         w.addVarSeq(self.cipher_suites, 2, 2)
         w.addVarSeq(self.compression_methods, 1, 1)
@@ -870,7 +870,7 @@ def write(self):
         w = Writer()
         w.add(self.server_version[0], 1)
         w.add(self.server_version[1], 1)
-        w.addFixSeq(self.random, 1)
+        w.bytes += self.random
         w.addVarSeq(self.session_id, 1, 1)
         w.add(self.cipher_suite, 2)
         w.add(self.compression_method, 1)
@@ -1416,7 +1416,7 @@ def write(self):
             if self.version in ((3, 1), (3, 2), (3, 3)):
                 w.addVarSeq(self.encryptedPreMasterSecret, 1, 2)
             elif self.version == (3, 0):
-                w.addFixSeq(self.encryptedPreMasterSecret, 1)
+                w.bytes += self.encryptedPreMasterSecret
             else:
                 raise AssertionError()
         elif self.cipherSuite in CipherSuite.dhAllSuites:
@@ -1612,7 +1612,7 @@ def parse(self, p):
 
     def write(self):
         w = Writer()
-        w.addFixSeq(self.verify_data, 1)
+        w.bytes += self.verify_data
         return self.postWrite(w)
 
 
@@ -1637,7 +1637,7 @@ def write(self):
         """Serialise the message to on the wire data."""
         writer = Writer()
         writer.add(self.handshakeType, 1)
-        writer.addFixSeq(self.verify_data, 1)
+        writer.bytes += self.verify_data
         # does not use postWrite() as it's a SSLv2 message
         return writer.bytes
 

tlslite/utils/codec.py

@@ -5,30 +5,195 @@
 
 from __future__ import division
 
+import sys
+import struct
+from struct import pack
+
 
 class Writer(object):
+    """Serialisation helper for complex byte-based structures."""
+
     def __init__(self):
+        """Initialise the serializer with no data."""
         self.bytes = bytearray(0)
 
-    def add(self, x, length):
-        self.bytes += bytearray(length)
-        newIndex = len(self.bytes) - 1
-        for count in range(length):
-            self.bytes[newIndex] = x & 0xFF
-            x >>= 8
-            newIndex -= 1
-        if x != 0:
-            raise ValueError("Can't represent value in specified length")
+    def addOne(self, val):
+        """Add a single-byte wide element to buffer, see add()."""
+        self.bytes.append(val)
+
+    if sys.version_info < (2, 7):
+        # struct.pack on Python2.6 does not raise exception if the value
+        # is larger than can fit inside the specified size
+        def addTwo(self, val):
+            """Add a double-byte wide element to buffer, see add()."""
+            if not 0 <= val <= 0xffff:
+                raise ValueError("Can't represent value in specified length")
+            self.bytes += pack('>H', val)
+
+        def addThree(self, val):
+            """Add a three-byte wide element to buffer, see add()."""
+            if not 0 <= val <= 0xffffff:
+                raise ValueError("Can't represent value in specified length")
+            self.bytes += pack('>BH', val >> 16, val & 0xffff)
+
+        def addFour(self, val):
+            """Add a four-byte wide element to buffer, see add()."""
+            if not 0 <= val <= 0xffffffff:
+                raise ValueError("Can't represent value in specified length")
+            self.bytes += pack('>I', val)
+    else:
+        def addTwo(self, val):
+            """Add a double-byte wide element to buffer, see add()."""
+            try:
+                self.bytes += pack('>H', val)
+            except struct.error:
+                raise ValueError("Can't represent value in specified length")
+
+        def addThree(self, val):
+            """Add a three-byte wide element to buffer, see add()."""
+            try:
+                self.bytes += pack('>BH', val >> 16, val & 0xffff)
+            except struct.error:
+                raise ValueError("Can't represent value in specified length")
+
+        def addFour(self, val):
+            """Add a four-byte wide element to buffer, see add()."""
+            try:
+                self.bytes += pack('>I', val)
+            except struct.error:
+                raise ValueError("Can't represent value in specified length")
+
+    if sys.version_info >= (3, 0):
+        # the method is called thousands of times, so it's better to extern
+        # the version info check
+        def add(self, x, length):
+            """
+            Add a single positive integer value x, encode it in length bytes
+
+            Encode positive integer x in big-endian format using length bytes,
+            add to the internal buffer.
+
+            @type x: int
+            @param x: value to encode
+
+            @type length: int
+            @param length: number of bytes to use for encoding the value
+            """
+            try:
+                self.bytes += x.to_bytes(length, 'big')
+            except OverflowError:
+                raise ValueError("Can't represent value in specified length")
+    else:
+        _addMethods = {1: addOne, 2: addTwo, 3: addThree, 4: addFour}
+
+        def add(self, x, length):
+            """
+            Add a single positive integer value x, encode it in length bytes
+
+            Encode positive iteger x in big-endian format using length bytes,
+            add to the internal buffer.
+
+            @type x: int
+            @param x: value to encode
+
+            @type length: int
+            @param length: number of bytes to use for encoding the value
+            """
+            try:
+                self._addMethods[length](self, x)
+            except KeyError:
+                self.bytes += bytearray(length)
+                newIndex = len(self.bytes) - 1
+                for i in range(newIndex, newIndex - length, -1):
+                    self.bytes[i] = x & 0xFF
+                    x >>= 8
+                if x != 0:
+                    raise ValueError("Can't represent value in specified "
+                                     "length")
 
     def addFixSeq(self, seq, length):
-        for e in seq:
-            self.add(e, length)
+        """
+        Add a list of items, encode every item in length bytes
+
+        Uses the unbounded iterable seq to produce items, each of
+        which is then encoded to length bytes
 
-    def addVarSeq(self, seq, length, lengthLength):
-        self.add(len(seq)*length, lengthLength)
+        @type seq: iterable of int
+        @param seq: list of positive integers to encode
+
+        @type length: int
+        @param length: number of bytes to which encode every element
+        """
         for e in seq:
             self.add(e, length)
 
+    if sys.version_info < (2, 7):
+        # struct.pack on Python2.6 does not raise exception if the value
+        # is larger than can fit inside the specified size
+        def _addVarSeqTwo(self, seq):
+            """Helper method for addVarSeq"""
+            if not all(0 <= i <= 0xffff for i in seq):
+                raise ValueError("Can't represent value in specified "
+                                 "length")
+            self.bytes += pack('>' + 'H' * len(seq), *seq)
+
+        def addVarSeq(self, seq, length, lengthLength):
+            """
+            Add a bounded list of same-sized values
+
+            Create a list of specific length with all items being of the same
+            size
+
+            @type seq: list of int
+            @param seq: list of positive integers to encode
+
+            @type length: int
+            @param length: amount of bytes in which to encode every item
+
+            @type lengthLength: int
+            @param lengthLength: amount of bytes in which to encode the overall
+                length of the array
+            """
+            self.add(len(seq)*length, lengthLength)
+            if length == 1:
+                self.bytes.extend(seq)
+            elif length == 2:
+                self._addVarSeqTwo(seq)
+            else:
+                for i in seq:
+                    self.add(i, length)
+    else:
+        def addVarSeq(self, seq, length, lengthLength):
+            """
+            Add a bounded list of same-sized values
+
+            Create a list of specific length with all items being of the same
+            size
+
+            @type seq: list of int
+            @param seq: list of positive integers to encode
+
+            @type length: int
+            @param length: amount of bytes in which to encode every item
+
+            @type lengthLength: int
+            @param lengthLength: amount of bytes in which to encode the overall
+                length of the array
+            """
+            seqLen = len(seq)
+            self.add(seqLen*length, lengthLength)
+            if length == 1:
+                self.bytes.extend(seq)
+            elif length == 2:
+                try:
+                    self.bytes += pack('>' + 'H' * seqLen, *seq)
+                except struct.error:
+                    raise ValueError("Can't represent value in specified "
+                                     "length")
+            else:
+                for i in seq:
+                    self.add(i, length)
+
     def addVarTupleSeq(self, seq, length, lengthLength):
         """
         Add a variable length list of same-sized element tuples.
@@ -46,17 +211,24 @@ def addVarTupleSeq(self, seq, length, lengthLength):
         @type lengthLength: int
         @param lengthLength: length in bytes of overall length field
         """
-        if len(seq) == 0:
+        if not seq:
             self.add(0, lengthLength)
         else:
-            tupleSize = len(seq[0])
-            tupleLength = tupleSize*length
-            self.add(len(seq)*tupleLength, lengthLength)
-            for elemTuple in seq:
-                if len(elemTuple) != tupleSize:
-                    raise ValueError("Tuples of different sizes")
-                for elem in elemTuple:
-                    self.add(elem, length)
+            startPos = len(self.bytes)
+            dataLength = len(seq) * len(seq[0]) * length
+            self.add(dataLength, lengthLength)
+            # since at the time of writing, all the calls encode single byte
+            # elements, and it's very easy to speed up that case, give it
+            # special case
+            if length == 1:
+                for elemTuple in seq:
+                    self.bytes.extend(elemTuple)
+            else:
+                for elemTuple in seq:
+                    self.addFixSeq(elemTuple, length)
+            if startPos + dataLength + lengthLength != len(self.bytes):
+                raise ValueError("Tuples of different lengths")
+
 
 class Parser(object):
     """