allow quoted strings and pretty file (#149)

Co-authored-by: Max Gruenfelder <[email protected]>

src/parser.ts

@@ -10,15 +10,15 @@ export type Token = {
 export function tokenizer(f: string): Token[] {
   const ret: Token[] = [];
   let rest = f;
-  const patterns = /^(?:(\s+)|(-?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?(?![-\w._:\/\)\s]))|("(?:[^"]|\\.|\n)*")|([[()]|]\.?)|(\w[-\w._:\/%]*))/;
+  const patterns = /^(?:(\s+)|(-?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?(?![-\w._:\/\)\s]))|("(?:[^"\\]|\\.|\n)*")|([[()]|]\.?)|(\w[-\w._:\/%]*))/;
   let n;
   while ((n = patterns.exec(rest))) {
     if (n[1] || n[0].length === 0) {
       //
     } else if (n[2]) {
       ret.push({ literal: n[2], type: "Number" });
     } else if (n[3]) {
-      const literal = n[3].replace(/\\/g, '\\\\');
+      const literal = n[3].replace(/\\(?!")/g, '\\\\');
       ret.push({ literal, type: "Quoted" });
     } else if (n[4]) {
       ret.push({ literal: n[4], type: "Bracket" });

test/tokenizer.test.ts

@@ -1,11 +1,11 @@
 import "mocha";
 import { tokenizer, Token } from "../src/parser";
-import { EOT } from './test_util';
+import { EOT } from "./test_util";
 import chai = require("chai");
 const assert = chai.assert;
 
 describe("tokenizer", () => {
-  const tok = (literal: string, type: string) => ({ literal, type } as Token);
+  const tok = (literal: string, type: string) => ({ literal, type }) as Token;
 
   it("eot", () => {
     assert.deepEqual(tokenizer(""), [EOT]);
@@ -14,46 +14,55 @@ describe("tokenizer", () => {
   it("false", () => {
     assert.deepEqual(tokenizer("false"), [
       { literal: "false", type: "Word" },
-      EOT
+      EOT,
     ]);
   });
 
   it("userName is AttrPath", () => {
     assert.deepEqual(tokenizer("userName"), [
       { literal: "userName", type: "Word" },
-      EOT
+      EOT,
     ]);
   });
 
   it("userName eq -12", () => {
     assert.deepEqual(
       [tok("userName", "Word"), tok("eq", "Word"), tok("-12", "Number"), EOT],
-      tokenizer("userName eq -12")
+      tokenizer("userName eq -12"),
     );
   });
 
   it("0Field1 eq -12", () => {
     assert.deepEqual(
       [tok("0Field1", "Word"), tok("eq", "Word"), tok("-12", "Number"), EOT],
-      tokenizer("0Field1 eq -12")
+      tokenizer("0Field1 eq -12"),
     );
   });
 
   it("sub-attribute after ValPath", () => {
     assert.deepEqual(
-        tokenizer('emails[type eq "work"].value eq "[email protected]"'),
-        [
-          tok("emails", "Word"),
-          tok("[", "Bracket"),
-          tok("type", "Word"),
-          tok("eq", "Word"),
-          tok("\"work\"", "Quoted"),
-          tok("].", "Bracket"),
-          tok("value", "Word"),
-          tok("eq", "Word"),
-          tok("\"[email protected]\"", "Quoted"),
-          EOT,
-      ]
-    )
-  })
+      tokenizer('emails[type eq "work"].value eq "[email protected]"'),
+      [
+        tok("emails", "Word"),
+        tok("[", "Bracket"),
+        tok("type", "Word"),
+        tok("eq", "Word"),
+        tok('"work"', "Quoted"),
+        tok("].", "Bracket"),
+        tok("value", "Word"),
+        tok("eq", "Word"),
+        tok('"[email protected]"', "Quoted"),
+        EOT,
+      ],
+    );
+  });
+
+  it("support of quoted values", () => {
+    assert.deepEqual(tokenizer('displayName eq "Alice \\"and\\" Bob"'), [
+      tok("displayName", "Word"),
+      tok("eq", "Word"),
+      tok('"Alice \\"and\\" Bob"', "Quoted"),
+      EOT,
+    ]);
+  });
 });