From 5e3856592c9a5e126bc9ad57c95865534b3e6837 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Mon, 31 Oct 2016 16:27:23 +0100 Subject: [PATCH 1/4] Improve stringNumbers parsing performance by up to 100% --- README.md | 6 ++++-- benchmark/index.js | 2 +- changelog.md | 6 ++++++ lib/parser.js | 26 ++++++++++++++++++++------ test/parsers.spec.js | 6 +++--- 5 files changed, 34 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 6843dc3..392d4af 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ var myParser = new Parser(options); * `returnError`: *function*; mandatory * `returnFatalError`: *function*; optional, defaults to the returnError function * `returnBuffers`: *boolean*; optional, defaults to false +* `stringNumbers`: *boolean*; optional, defaults to false ### Example @@ -66,7 +67,7 @@ You do not have to use the returnFatalError function. Fatal errors will be retur And if you want to return buffers instead of strings, you can do this by adding the `returnBuffers` option. -Big numbers that are too large for JS are automatically stringified. +If you handle with big numbers that are to large for JS (Number.MAX_SAFE_INTEGER === 2^53 - 16) please use the `stringNumbers` option. That way all numbers are going to be returned as String and you can handle them safely. ```js // Same functions as in the first example @@ -78,7 +79,8 @@ var parser = new Parser({ returnError: function(err) { lib.returnError(err); }, - returnBuffers: true // All strings are returned as buffer e.g. + returnBuffers: true, // All strings are returned as Buffer e.g. + stringNumbers: true // All numbers are returned as String }); // The streamHandler as above diff --git a/benchmark/index.js b/benchmark/index.js index a2ba0ff..5387bbf 100644 --- a/benchmark/index.js +++ b/benchmark/index.js @@ -34,7 +34,7 @@ var startBuffer = new Buffer('$100\r\nabcdefghij') var chunkBuffer = new Buffer('abcdefghijabcdefghijabcdefghij') var stringBuffer = new Buffer('+testing a simple string\r\n') var integerBuffer = new Buffer(':1237884\r\n') -var bigIntegerBuffer = new Buffer(':18446744073709551617\r\n') // 2^64 + 1 +var bigIntegerBuffer = new Buffer(':184467440737095516171234567890\r\n') // 2^64 + 1 var errorBuffer = new Buffer('-Error ohnoesitbroke\r\n') var arrayBuffer = new Buffer('*1\r\n*1\r\n$1\r\na\r\n') var endBuffer = new Buffer('\r\n') diff --git a/changelog.md b/changelog.md index 76cdac2..7e32712 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,9 @@ +## v.2.2.0 - 31 Oct, 2016 + +Features + +- Improve `stringNumbers` parsing performance by up to 100% + ## v.2.1.1 - 31 Oct, 2016 Bugfixes diff --git a/lib/parser.js b/lib/parser.js index 73b4303..006fc3f 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -37,26 +37,40 @@ function parseSimpleNumbers (parser) { /** * Used for integer numbers in case of the returnNumbers option + * + * The maximimum possible integer to use is: Math.floor(Number.MAX_SAFE_INTEGER / 10) - 9 + * * @param parser * @returns {*} */ function parseStringNumbers (parser) { var offset = parser.offset - var length = parser.buffer.length - var number = '' + var length = parser.buffer.length - 1 + var number = 0 + var res = '' if (parser.buffer[offset] === 45) { - number += '-' + res += '-' offset++ } while (offset < length) { var c1 = parser.buffer[offset++] - if (c1 === 13 && parser.buffer[offset] === 10) { // \r\n + if (c1 === 13) { // \r\n parser.offset = offset + 1 - return number + if (number !== 0) { + res += number + } + return res + } else if (number > 900719925474090) { + res += number + res += c1 - 48 + number = 0 + } else if (c1 === 48 && number === 0) { + res += 0 + } else { + number = (number * 10) + (c1 - 48) } - number += c1 - 48 } } diff --git a/test/parsers.spec.js b/test/parsers.spec.js index 69b23a5..43686a9 100644 --- a/test/parsers.spec.js +++ b/test/parsers.spec.js @@ -541,7 +541,7 @@ describe('parsers', function () { return this.skip() } var replyCount = 0 - var entries = ['123', '590295810358705700002', '-99999999999999999'] + var entries = ['123', '590295810358705700002', '-99999999999999999', '9007199254740992', '90071992547409920', '10000040000000000000000000000000000000020'] function checkReply (reply) { assert.strictEqual(typeof reply, 'string') assert.strictEqual(reply, entries[replyCount]) @@ -551,8 +551,8 @@ describe('parsers', function () { returnReply: checkReply, stringNumbers: true }) - parser.execute(new Buffer(':123\r\n:590295810358705700002\r\n:-99999999999999999\r\n')) - assert.strictEqual(replyCount, 3) + parser.execute(new Buffer(':123\r\n:590295810358705700002\r\n:-99999999999999999\r\n:9007199254740992\r\n:90071992547409920\r\n:10000040000000000000000000000000000000020\r\n')) + assert.strictEqual(replyCount, 6) }) it('handle big numbers', function () { From f4c80d74e192f1d1fdb71c5ad3e4b11c424c580e Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Mon, 31 Oct 2016 17:27:56 +0100 Subject: [PATCH 2/4] Simplify concat bulk string logic --- lib/parser.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/parser.js b/lib/parser.js index 006fc3f..7099119 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -319,14 +319,14 @@ function concatBulkString (parser) { if (chunks === 2) { return list[0].toString('utf8', parser.bigOffset, list[0].length - 1) } - } else { - chunks++ + chunks-- + offset = list[list.length - 2].length + 1 } var res = decoder.write(list[0].slice(parser.bigOffset)) - for (var i = 1; i < chunks - 2; i++) { + for (var i = 1; i < chunks - 1; i++) { res += decoder.write(list[i]) } - res += decoder.end(list[i].slice(0, offset === 1 ? list[i].length - 1 : offset - 2)) + res += decoder.end(list[i].slice(0, offset - 2)) return res } From c9a55bee2aebd02cb9201b8d70cc6a74236deb15 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 1 Nov 2016 05:36:27 +0100 Subject: [PATCH 3/4] Improve simple string and number performance by 5-15% --- lib/parser.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/parser.js b/lib/parser.js index 7099119..7acad1e 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -16,20 +16,20 @@ var notDecreased = 0 */ function parseSimpleNumbers (parser) { var offset = parser.offset - var length = parser.buffer.length + var length = parser.buffer.length - 1 var number = 0 - var sign = false + var sign = 1 if (parser.buffer[offset] === 45) { - sign = true + sign = -1 offset++ } while (offset < length) { var c1 = parser.buffer[offset++] - if (c1 === 13 && parser.buffer[offset] === 10) { // \r\n + if (c1 === 13) { // \r\n parser.offset = offset + 1 - return sign ? -number : number + return sign * number } number = (number * 10) + (c1 - 48) } @@ -103,12 +103,12 @@ function convertBufferRange (parser, start, end) { function parseSimpleStringViaOffset (parser) { var start = parser.offset var offset = parser.offset - var length = parser.buffer.length + var length = parser.buffer.length - 1 var buffer = parser.buffer while (offset < length) { - if (buffer[offset++] === 10) { // \r\n - return convertBufferRange(parser, start, offset - 2) + if (buffer[offset++] === 13) { // \r\n + return convertBufferRange(parser, start, offset - 1) } } } @@ -207,7 +207,7 @@ function parseArray (parser) { } var responses = new Array(length) - var bufferLength = parser.buffer.length + var bufferLength = parser.buffer.length - 3 for (var i = 0; i < length; i++) { if (parser.offset >= bufferLength) { return From 583046b0e3377ecadacd22d6c5bbdafaabacc92b Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 8 Nov 2016 10:53:47 +0100 Subject: [PATCH 4/4] Minimal improvement --- lib/parser.js | 14 +++++++------- test/parsers.spec.js | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/parser.js b/lib/parser.js index 7acad1e..053716e 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -38,7 +38,8 @@ function parseSimpleNumbers (parser) { /** * Used for integer numbers in case of the returnNumbers option * - * The maximimum possible integer to use is: Math.floor(Number.MAX_SAFE_INTEGER / 10) - 9 + * The maximimum possible integer to use is: Math.floor(Number.MAX_SAFE_INTEGER / 10) + * Staying in a SMI Math.floor((Math.pow(2, 32) / 10) - 1) is even more efficient though * * @param parser * @returns {*} @@ -62,9 +63,8 @@ function parseStringNumbers (parser) { res += number } return res - } else if (number > 900719925474090) { - res += number - res += c1 - 48 + } else if (number > 429496728) { + res += (number * 10) + (c1 - 48) number = 0 } else if (c1 === 48 && number === 0) { res += 0 @@ -102,9 +102,9 @@ function convertBufferRange (parser, start, end) { */ function parseSimpleStringViaOffset (parser) { var start = parser.offset - var offset = parser.offset - var length = parser.buffer.length - 1 + var offset = start var buffer = parser.buffer + var length = buffer.length - 1 while (offset < length) { if (buffer[offset++] === 13) { // \r\n @@ -121,7 +121,7 @@ function parseSimpleStringViaOffset (parser) { function parseLength (parser) { var string = parseSimpleNumbers(parser) if (string !== undefined) { - return +string + return string } } diff --git a/test/parsers.spec.js b/test/parsers.spec.js index 43686a9..b84cdc2 100644 --- a/test/parsers.spec.js +++ b/test/parsers.spec.js @@ -541,7 +541,7 @@ describe('parsers', function () { return this.skip() } var replyCount = 0 - var entries = ['123', '590295810358705700002', '-99999999999999999', '9007199254740992', '90071992547409920', '10000040000000000000000000000000000000020'] + var entries = ['123', '590295810358705700002', '-99999999999999999', '4294967290', '90071992547409920', '10000040000000000000000000000000000000020'] function checkReply (reply) { assert.strictEqual(typeof reply, 'string') assert.strictEqual(reply, entries[replyCount]) @@ -551,7 +551,7 @@ describe('parsers', function () { returnReply: checkReply, stringNumbers: true }) - parser.execute(new Buffer(':123\r\n:590295810358705700002\r\n:-99999999999999999\r\n:9007199254740992\r\n:90071992547409920\r\n:10000040000000000000000000000000000000020\r\n')) + parser.execute(new Buffer(':123\r\n:590295810358705700002\r\n:-99999999999999999\r\n:4294967290\r\n:90071992547409920\r\n:10000040000000000000000000000000000000020\r\n')) assert.strictEqual(replyCount, 6) })