Skip to content

Commit

Permalink
v0.1.1
Browse files Browse the repository at this point in the history 
* Added trim, ltrim, and rtrim to parsing options
  • Loading branch information
@doug-martin
doug-martin committed Mar 5, 2014
1 parent 4401b0a commit b60acd2
Show file tree
Hide file tree
Showing 12 changed files with 250 additions and 102 deletions.
87 changes: 46 additions & 41 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ All methods accept the following `options`
* `ignoreEmpty=false`: If you wish to ignore empty rows.
* `delimiter=','`: If your data uses an alternate delimiter such as `;` or `\t`.
* **NOTE** When specifying an alternate `delimiter` you may only pass in a single character delimeter
* The following are options for parsing only.
* `trim=false`: If you want to trim all values parsed set to true.
* `rtrim=false`: If you want to right trim all values parsed set to true.
* `ltrim=false`: If you want to left trim all values parsed set to true.


**events**

Expand Down Expand Up @@ -223,22 +228,22 @@ Create a readable stream to read data from.
```javascript
var ws = fs.createWritableStream("my.csv");
csv
.write([
["a", "b"],
["a1", "b1"],
["a2", "b2"]
], {headers: true})
.pipe(ws);
.write([
["a", "b"],
["a1", "b1"],
["a2", "b2"]
], {headers: true})
.pipe(ws);
```
```javascript
var ws = fs.createWritableStream("my.csv");
csv
.write([
{a: "a1", b: "b1"},
{a: "a2", b: "b2"}
], {headers: true})
.pipe(ws);
.write([
{a: "a1", b: "b1"},
{a: "a2", b: "b2"}
], {headers: true})
.pipe(ws);
```
**`writeToStream(stream,arr[, options])`**
Expand All @@ -247,20 +252,20 @@ Write an array of values to a `WritableStream`
```javascript
csv
.writeToStream(fs.createWritableStream("my.csv"), [
["a", "b"],
["a1", "b1"],
["a2", "b2"]
], {headers: true});
.writeToStream(fs.createWritableStream("my.csv"), [
["a", "b"],
["a1", "b1"],
["a2", "b2"]
], {headers: true});
```
```javascript
csv
.writeToStream(fs.createWritableStream("my.csv"), [
{a: "a1", b: "b1"},
{a: "a2", b: "b2"}
], {headers: true})
.pipe(ws);
.writeToStream(fs.createWritableStream("my.csv"), [
{a: "a1", b: "b1"},
{a: "a2", b: "b2"}
], {headers: true})
.pipe(ws);
```
**`writeToPath(arr[, options])`**
Expand All @@ -269,41 +274,41 @@ Write an array of values to the specified path
```javascript
csv
.writeToPath("my.csv", [
["a", "b"],
["a1", "b1"],
["a2", "b2"]
], {headers: true})
.on("finish", function(){
console.log("done!");
});
.writeToPath("my.csv", [
["a", "b"],
["a1", "b1"],
["a2", "b2"]
], {headers: true})
.on("finish", function(){
console.log("done!");
});
```
```javascript
csv
.writeToStream("my.csv", [
{a: "a1", b: "b1"},
{a: "a2", b: "b2"}
], {headers: true})
.on("finish", function(){
console.log("done!");
});
.writeToStream("my.csv", [
{a: "a1", b: "b1"},
{a: "a2", b: "b2"}
], {headers: true})
.on("finish", function(){
console.log("done!");
});
```
**`writeToString(arr[, options])`**
```javascript
csv.writeToString([
["a", "b"],
["a1", "b1"],
["a2", "b2"]
["a", "b"],
["a1", "b1"],
["a2", "b2"]
], {headers: true}); //"a,b\na1,b1\na2,b2\n"
```
```javascript
csv.writeToString([
{a: "a1", b: "b1"},
{a: "a2", b: "b2"}
{a: "a1", b: "b1"},
{a: "a2", b: "b2"}
], {headers: true}); //"a,b\na1,b1\na2,b2\n"
```
Expand Down
2 changes: 1 addition & 1 deletion benchmark/benchmark.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
var fastCsv = require("../lib"),
csv = require("csv"),
path = require("path"),
COUNT = 1000000,
COUNT = 20000,
TEST_FILE = path.resolve(__dirname, "./assets/" + COUNT + ".csv");


Expand Down
88 changes: 47 additions & 41 deletions docs/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,12 @@ <h3>Parsing</h3>
<li><strong>NOTE</strong> When specifying an alternate <code>delimiter</code> you may only pass in a single character delimeter</li>
</ul>
</li>
<li>The following are options for parsing only.<ul>
<li><code>trim=false</code>: If you want to trim all values parsed set to true.</li>
<li><code>rtrim=false</code>: If you want to right trim all values parsed set to true.</li>
<li><code>ltrim=false</code>: If you want to left trim all values parsed set to true.</li>
</ul>
</li>
</ul>
<p><strong>events</strong></p>
<p><code>parse-error</code>: Emitted if there was an error parsing a row.
Expand Down Expand Up @@ -332,61 +338,61 @@ <h3>Formatting</h3>
<p>Create a readable stream to read data from.</p>
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">var ws = fs.createWritableStream(&quot;my.csv&quot;);
csv
.write([
[&quot;a&quot;, &quot;b&quot;],
[&quot;a1&quot;, &quot;b1&quot;],
[&quot;a2&quot;, &quot;b2&quot;]
], {headers: true})
.pipe(ws);</code></pre>
.write([
[&quot;a&quot;, &quot;b&quot;],
[&quot;a1&quot;, &quot;b1&quot;],
[&quot;a2&quot;, &quot;b2&quot;]
], {headers: true})
.pipe(ws);</code></pre>
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">var ws = fs.createWritableStream(&quot;my.csv&quot;);
csv
.write([
{a: &quot;a1&quot;, b: &quot;b1&quot;},
{a: &quot;a2&quot;, b: &quot;b2&quot;}
], {headers: true})
.pipe(ws);</code></pre>
.write([
{a: &quot;a1&quot;, b: &quot;b1&quot;},
{a: &quot;a2&quot;, b: &quot;b2&quot;}
], {headers: true})
.pipe(ws);</code></pre>
<p><strong><code>writeToStream(stream,arr[, options])</code></strong></p>
<p>Write an array of values to a <code>WritableStream</code></p>
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
.writeToStream(fs.createWritableStream(&quot;my.csv&quot;), [
[&quot;a&quot;, &quot;b&quot;],
[&quot;a1&quot;, &quot;b1&quot;],
[&quot;a2&quot;, &quot;b2&quot;]
], {headers: true});</code></pre>
.writeToStream(fs.createWritableStream(&quot;my.csv&quot;), [
[&quot;a&quot;, &quot;b&quot;],
[&quot;a1&quot;, &quot;b1&quot;],
[&quot;a2&quot;, &quot;b2&quot;]
], {headers: true});</code></pre>
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
.writeToStream(fs.createWritableStream(&quot;my.csv&quot;), [
{a: &quot;a1&quot;, b: &quot;b1&quot;},
{a: &quot;a2&quot;, b: &quot;b2&quot;}
], {headers: true})
.pipe(ws);</code></pre>
.writeToStream(fs.createWritableStream(&quot;my.csv&quot;), [
{a: &quot;a1&quot;, b: &quot;b1&quot;},
{a: &quot;a2&quot;, b: &quot;b2&quot;}
], {headers: true})
.pipe(ws);</code></pre>
<p><strong><code>writeToPath(arr[, options])</code></strong></p>
<p>Write an array of values to the specified path</p>
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
.writeToPath(&quot;my.csv&quot;, [
[&quot;a&quot;, &quot;b&quot;],
[&quot;a1&quot;, &quot;b1&quot;],
[&quot;a2&quot;, &quot;b2&quot;]
], {headers: true})
.on(&quot;finish&quot;, function(){
console.log(&quot;done!&quot;);
});</code></pre>
.writeToPath(&quot;my.csv&quot;, [
[&quot;a&quot;, &quot;b&quot;],
[&quot;a1&quot;, &quot;b1&quot;],
[&quot;a2&quot;, &quot;b2&quot;]
], {headers: true})
.on(&quot;finish&quot;, function(){
console.log(&quot;done!&quot;);
});</code></pre>
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
.writeToStream(&quot;my.csv&quot;, [
{a: &quot;a1&quot;, b: &quot;b1&quot;},
{a: &quot;a2&quot;, b: &quot;b2&quot;}
], {headers: true})
.on(&quot;finish&quot;, function(){
console.log(&quot;done!&quot;);
});</code></pre>
.writeToStream(&quot;my.csv&quot;, [
{a: &quot;a1&quot;, b: &quot;b1&quot;},
{a: &quot;a2&quot;, b: &quot;b2&quot;}
], {headers: true})
.on(&quot;finish&quot;, function(){
console.log(&quot;done!&quot;);
});</code></pre>
<p><strong><code>writeToString(arr[, options])</code></strong></p>
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv.writeToString([
[&quot;a&quot;, &quot;b&quot;],
[&quot;a1&quot;, &quot;b1&quot;],
[&quot;a2&quot;, &quot;b2&quot;]
[&quot;a&quot;, &quot;b&quot;],
[&quot;a1&quot;, &quot;b1&quot;],
[&quot;a2&quot;, &quot;b2&quot;]
], {headers: true}); //&quot;a,b\na1,b1\na2,b2\n&quot;</code></pre>
<pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv.writeToString([
{a: &quot;a1&quot;, b: &quot;b1&quot;},
{a: &quot;a2&quot;, b: &quot;b2&quot;}
{a: &quot;a1&quot;, b: &quot;b1&quot;},
{a: &quot;a2&quot;, b: &quot;b2&quot;}
], {headers: true}); //&quot;a,b\na1,b1\na2,b2\n&quot;</code></pre>
<h2>Benchmarks</h2>
<p><code>Parsing 20000 records AVG over 3 runs</code></p>
Expand Down
5 changes: 5 additions & 0 deletions lib/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@
* * `ignoreEmpty=false`: If you wish to ignore empty rows.
* * `delimiter=','`: If your data uses an alternate delimiter such as `;` or `\t`.
* * **NOTE** When specifying an alternate `delimiter` you may only pass in a single character delimeter
* * The following are options for parsing only.
* * `trim=false`: If you want to trim all values parsed set to true.
* * `rtrim=false`: If you want to right trim all values parsed set to true.
* * `ltrim=false`: If you want to left trim all values parsed set to true.
*
*
* **events**
*
Expand Down
63 changes: 51 additions & 12 deletions lib/parser.js
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,31 @@
var SINGLE_QUOTE = "'",
var extended = require("./extended"),
trim = extended.trim,
trimLeft = extended.trimLeft,
trimRight = extended.trimRight,
SINGLE_QUOTE = "'",
DOUBLE_QUOTE = '"';

function createParser(delimiter) {

var VALUE_REGEXP = new RegExp("([^" + delimiter + "'\"\\s\\\\]*(?:\\s+[^" + delimiter + "'\"\\s\\\\]+)*)"),
function createParser(options) {
options = options || {};
var delimiter = options.delimiter || ",",
doLtrim = options.ltrim || false,
doRtrim = options.rtrim || false,
doTrim = options.trim || false,
VALUE_REGEXP = new RegExp("([^" + delimiter + "'\"\\s\\\\]*(?:\\s+[^" + delimiter + "'\"\\s\\\\]+)*)"),
SEARCH_REGEXP = new RegExp("[^\\\\]" + delimiter),
ESCAPE_CHAR = "\\";
ESCAPE_CHAR = "\\",
WHITE_SPACE = /\s/;

function formatItem(item) {
if (doTrim) {
item = trim(item);
} else if (doLtrim) {
item = trimLeft(item);
} else if (doRtrim) {
item = trimRight(item);
}
return item;
}

function getTokensBetween(str, start, items, cursor) {
var depth = 0, ret = [];
Expand Down Expand Up @@ -35,10 +55,10 @@ function createParser(delimiter) {
++cursor;
}
}
if (++cursor < str.length && str[cursor].search(delimiter) !== 0) {
if (++cursor < str.length && getNextToken(str, cursor).token.search(delimiter) !== 0) {
throw new Error("Invalid row " + str);
}
items.push(ret.join(""));
items.push(formatItem(ret.join("")));
return ++cursor;
}

Expand All @@ -52,7 +72,7 @@ function createParser(delimiter) {
nextIndex = searchStr.length - 1;
}
}
items.push(searchStr.substr(0, nextIndex + 1));
items.push(formatItem(searchStr.substr(0, nextIndex + 1)));
return cursor + (nextIndex + 2);
}

Expand All @@ -69,20 +89,39 @@ function createParser(delimiter) {
return findNextToken(line, items, cursor);
}

function getNextToken(line, cursor) {
var l = line.length, ret, token;
do {
token = line[cursor];
if (token === delimiter || !WHITE_SPACE.test(token)) {
ret = token;
} else {
token = null;
}

} while (!token && cursor++ < l);
if (!token) {
throw new Error("Invalid row " + line);
}
return {token: token, cursor: cursor};
}

return function parseLine(line) {
var i = 0, l = line.length, items = [], token;
var i = 0, l = line.length, items = [], token, nextToken;
while (i < l) {
token = line[i];
nextToken = getNextToken(line, i);
token = nextToken.token;
if (token === delimiter) {
items.push("");
i++;
} else if (token === SINGLE_QUOTE) {
i = parseSingleQuoteItem(line, items, i);
i = parseSingleQuoteItem(line, items, nextToken.cursor);
} else if (token === DOUBLE_QUOTE) {
i = parseDoubleQuoteItem(line, items, i);
i = parseDoubleQuoteItem(line, items, nextToken.cursor);
} else {
i = parseItem(line, items, i);
}

}
return items;
};
Expand Down
3 changes: 2 additions & 1 deletion lib/parser_stream.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ function ParserStream(options) {
} else {
delimiter = DEFAULT_DELIMITER;
}
this.parser = createParser(delimiter);
options.delimiter = delimiter;
this.parser = createParser(options);
this._headers = options.headers;
this._ignoreEmpty = options.ignoreEmpty;
return this;
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "fast-csv",
"version": "0.1.0",
"version": "0.1.1",
"description": "CSV parser for node.js",
"main": "index.js",
"scripts": {
Expand Down
Loading

0 comments on commit b60acd2

Please sign in to comment.