initial implementation of deserialize transform

JairusSW · Jan 11, 2025 · 7902b8e · 7902b8e
1 parent 67dcfb5
commit 7902b8e
Show file tree

Hide file tree

Showing 26 changed files with 1,700 additions and 156 deletions.
diff --git a/.prettierrc.json b/.prettierrc.json
@@ -2,5 +2,6 @@
   "pluginSearchDirs": ["node_modules"],
   "plugins": ["assemblyscript-prettier"],
   "singleQuote": false,
-  "printWidth": 65536
+  "printWidth": 65536,
+  "tabWidth": 2
 }
diff --git a/assembly/__benches__/misc.bench.ts b/assembly/__benches__/misc.bench.ts
@@ -1,82 +1,19 @@
-import { deserializeString_SIMD } from "../deserialize/simd/string";
-import { serializeString_SIMD } from "../serialize/simd/string";
-import { bench } from "as-bench/assembly/index";
-import { serialize_simple } from "../serialize/simple";
+import { bench } from "as-bench/assembly/bench";
 import { bs } from "as-bs";
+import { deserializeString_SIMD } from "../deserialize/simd/string";
+import { deserializeString } from "../deserialize/simple/string";
+import { bytes } from "../util/bytes";
 const str = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^&*()\\\"\t\r\f\n\u0000';
 const str2 = '"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^&*()\\\\\\"\\t\\r\\f\\n\\u0000"';
-
-// bench("Serialize String (Simple)", () => {
-//   serializeString(str);
-// });
-
-// bench("Serialize String (BS)", () => {
-//     serializeString_BS(str);
-//     bs.reset();
-// });
-
-@json
-class Vec3 {
-  x: i32 = 1;
-  y: i32 = 2;
-  z: i32 = 3;
-
-  __SERIALIZE(ptr: usize = 0): string {
-      if (ptr == 0) ptr = changetype<usize>(this);
-      let out = `{"x":${load<i32>(ptr, offsetof<this>("x")).toString()},"y":${load<i32>(ptr, offsetof<this>("y")).toString()},"z":${load<i32>(ptr, offsetof<this>("z")).toString()},`;
-      store<u16>(changetype<usize>(out) + ((out.length - 1) << 1), 125);
-      return out;
-    }
-
-  __SERIALIZE_BS(ptr: usize = 0): void {
-    ptr = ptr || changetype<usize>(this);
-    bs.ensureSize(128);
-    store<u64>(bs.offset, 9570664606466171);
-    store<u16>(bs.offset, 58, 8);
-    bs.offset += 10;
-
-    serialize_simple<i32>(load<i32>(ptr, offsetof<this>("x")));
-
-    store<u64>(bs.offset, 9570668901433388);
-    store<u16>(bs.offset, 58, 8);
-    bs.offset += 10;
-
-    serialize_simple<i32>(load<i32>(ptr, offsetof<this>("y")));
-
-    store<u64>(bs.offset, 9570673196400684);
-    store<u16>(bs.offset, 58, 8);
-    bs.offset += 10;
-
-    serialize_simple<i32>(load<i32>(ptr, offsetof<this>("z")));
-
-    store<u16>(bs.offset, 125);
-    bs.offset += 2;
-  }
-}
-
-const vec: Vec3 = {
-  x: 1,
-  y: 2,
-  z: 3
-}
-
-bench("Serialize Object (New)", () => {
-  vec.__SERIALIZE_BS(changetype<usize>(vec));
-  bs.out<string>();
+const srcStart = changetype<usize>(str);
+const srcEnd = srcStart + bytes(str);
+bs.ensureSize(2048);
+bench("Deserialize String (Simple)", () => {
+    deserializeString(str2);
 });
-
-// bench("Serialize Object (Old)", () => {
-//   vec.__SERIALIZE(changetype<usize>(vec));
-// });
-// }
-
-// const out = new ArrayBuffer(256);
-// bench("Serialize String (SIMD)", () => {
-//   // ~5.07GB/s
-//   serializeString_SIMD(str, changetype<usize>(out));
-// });
-
 // bench("Deserialize String (SIMD)", () => {
-//   // ~4.03GB/s
-//   deserializeString_SIMD(str2, changetype<usize>(out));
+//   (str2, bs.buffer);
 // });
+bench("Deserialize String (SIMD)", () => {
+    deserializeString_SIMD(str2, srcStart, srcEnd, __new(158, idof<string>()));
+});
diff --git a/assembly/deserialize/simd/string.ts b/assembly/deserialize/simd/string.ts
@@ -1,4 +1,3 @@
-import { OBJECT, TOTAL_OVERHEAD } from "rt/common";
 import { BACK_SLASH } from "../../custom/chars";
 import { DESERIALIZE_ESCAPE_TABLE, ESCAPE_HEX_TABLE } from "../../globals/tables";
 
@@ -12,11 +11,11 @@ const SPLAT_92 = i16x8.splat(92); /* \ */
  */
 // todo: optimize and stuff. it works, its not pretty. ideally, i'd like this to be (nearly) branchless
 // @ts-ignore: Decorator
-@inline export function deserializeString_SIMD(src: string, dst: usize): usize {
-  let src_ptr = changetype<usize>(src) + 2;
+@inline export function deserializeString_SIMD(src: string, srcStart: usize, srcEnd: usize, dst: usize): usize {
+  let src_ptr = srcStart + 2;
   let dst_ptr = changetype<usize>(dst);
 
-  const src_end = src_ptr + changetype<OBJECT>(changetype<usize>(src) - TOTAL_OVERHEAD).rtSize - 4;
+  const src_end = srcEnd - 2;
   const src_end_15 = src_end - 15;
 
   while (src_ptr < src_end_15) {

diff --git a/assembly/deserialize/simple/arbitrary.ts b/assembly/deserialize/simple/arbitrary.ts
@@ -15,6 +15,7 @@ import { deserializeString } from "./string";
   if (firstChar > 47 && firstChar < 58) return JSON.Value.from(data.includes(".") ? deserializeInteger<u64>(data) : deserializeFloat<f64>(data));
   if (firstChar == 45) return JSON.Value.from(data.includes(".") ? deserializeInteger<i64>(data) : deserializeFloat<f64>(data));
   if (firstChar == 91) {
+
     // array
   }
   if (firstChar == 116 || firstChar == 102) return JSON.Value.from(deserializeBoolean(data));

diff --git a/assembly/deserialize/simple/array/float.ts b/assembly/deserialize/simple/array/float.ts
@@ -1,4 +1,4 @@
-import { isSpace } from "util/string";
+import { isSpace } from "../../../util";
 import { unsafeCharCodeAt } from "../../../custom/util";
 import { COMMA, BRACKET_RIGHT } from "../../../custom/chars";
 import { deserializeFloat } from "../float";

diff --git a/assembly/deserialize/simple/array/integer.ts b/assembly/deserialize/simple/array/integer.ts
@@ -1,4 +1,4 @@
-import { isSpace } from "util/string";
+import { isSpace } from "../../../util";
 import { unsafeCharCodeAt } from "../../../custom/util";
 import { COMMA, BRACKET_RIGHT } from "../../../custom/chars";
 import { deserializeInteger } from "../integer";

diff --git a/assembly/deserialize/simple/bool.ts b/assembly/deserialize/simple/bool.ts
@@ -16,3 +16,12 @@ import { unsafeCharCodeAt } from "../../custom/util";
   else if (len == 5 && firstChar == CHAR_F && load<u64>(ptr, 2) == 28429466576093281) return false;
   return false; //ERROR(`Expected to find boolean, but found "${data.slice(0, 100)}" instead!`);
 }
+
+// @ts-ignore: Decorator valid here
+@inline export function deserializeBoolean_NEW(srcStart: usize, srcEnd: usize): boolean {
+  const srcSize = srcEnd - srcStart;
+  const firstChar = load<u16>(srcStart);
+  if (srcSize == 4 && firstChar == CHAR_T && load<u64>(srcStart) == 28429475166421108) return true;
+  else if (srcSize == 5 && firstChar == CHAR_F && load<u64>(srcSize, 2) == 28429466576093281) return false;
+  return false; //ERROR(`Expected to find boolean, but found "${data.slice(0, 100)}" instead!`);
+}
diff --git a/assembly/deserialize/simple/float.ts b/assembly/deserialize/simple/float.ts
@@ -7,3 +7,9 @@
   // @ts-ignore
   return f32.parse(data);
 }
+
+// @ts-ignore: Decorator valid here
+@inline export function deserializeFloat_NEW<T>(srcStart: usize, srcEnd: usize): T {
+  // todo: actually implement this
+  return 3.14
+}
diff --git a/assembly/deserialize/simple/integer.ts b/assembly/deserialize/simple/integer.ts
@@ -1,7 +1,13 @@
 import { snip_fast } from "../../custom/util";
+import { strToInt } from "../../util/strToInt";
 
 // @ts-ignore: Decorator valid here
 @inline export function deserializeInteger<T>(data: string): T {
   // @ts-ignore
   return snip_fast<T>(data);
 }
+
+// @ts-ignore: Decorator valid here
+@inline export function deserializeInteger_NEW<T>(srcStart: usize, srcEnd: usize): T {
+  return strToInt<T>(srcStart, srcEnd);
+}
diff --git a/assembly/deserialize/simple/map.ts b/assembly/deserialize/simple/map.ts
@@ -4,7 +4,7 @@ import { CHAR_A, BACK_SLASH, COLON, COMMA, CHAR_E, CHAR_F, CHAR_L, BRACE_LEFT, B
 import { deserializeBoolean } from "./bool";
 import { JSON } from "../..";
 import { deserializeString } from "./string";
-import { isSpace } from "util/string";
+import { isSpace } from "../../util";
 import { deserializeInteger } from "./integer";
 import { deserializeFloat } from "./float";
 

diff --git a/assembly/deserialize/simple/object.ts b/assembly/deserialize/simple/object.ts
@@ -1,6 +1,6 @@
 import { unsafeCharCodeAt } from "../../custom/util";
-import { CHAR_A, BACK_SLASH, COMMA, CHAR_E, CHAR_F, CHAR_L, BRACE_LEFT, BRACKET_LEFT, CHAR_N, QUOTE, CHAR_R, BRACE_RIGHT, BRACKET_RIGHT, CHAR_S, CHAR_T, CHAR_U } from "../../custom/chars";
-import { isSpace } from "util/string";
+import { CHAR_A, BACK_SLASH, COMMA, CHAR_E, CHAR_F, CHAR_L, BRACE_LEFT, BRACKET_LEFT, CHAR_N, QUOTE, CHAR_R, BRACE_RIGHT, BRACKET_RIGHT, CHAR_S, CHAR_T, CHAR_U, COLON } from "../../custom/chars";
+import { isSpace } from "../../util";
 
 export function deserializeObject<T>(data: string): T {
   const schema: nonnull<T> = changetype<nonnull<T>>(__new(offsetof<nonnull<T>>(), idof<nonnull<T>>()));
@@ -103,3 +103,124 @@ export function deserializeObject<T>(data: string): T {
   }
   return schema;
 }
+
+export function deserializeObject_NEW<T>(srcStart: usize, srcEnd: usize): T {
+  console.log("Data: " + str(srcStart, srcEnd));
+  const schema: nonnull<T> = changetype<nonnull<T>>(__new(offsetof<nonnull<T>>(), idof<nonnull<T>>()));
+
+  const srcPtr = srcStart;
+
+  let keyStart: usize = 0;
+  let keyEnd: usize = 0;
+  let isKey = false;
+  let depth = 0;
+  let lastIndex = 0;
+
+  // while (srcStart < srcEnd && isSpace(load<u16>(srcStart))) srcStart += 2;
+  // while (srcEnd > srcStart && isSpace(load<u16>(srcEnd))) srcEnd -= 2;
+
+  while (srcStart < srcEnd) {
+    let code = load<u16>(srcStart);// while (isSpace(code)) code = load<u16>(srcStart += 2);
+    if (keyStart == 0) {
+      if (code == QUOTE && load<u16>(srcStart - 2) !== BACK_SLASH) {
+        if (isKey) {
+          keyStart = lastIndex;
+          keyEnd = srcStart;
+          console.log("Key: " + str(lastIndex, srcStart));
+          while (isSpace(code = load<u16>(srcStart += 2))) { /* empty */ }
+          if (code !== COLON)
+            throw new Error(
+              "Expected ':' after key at position " + (srcStart - srcPtr).toString(),
+            );
+          isKey = false;
+        } else {
+          isKey = true; // i don't like this
+          lastIndex = srcStart + 2;
+        }
+      }
+      // isKey = !isKey;
+      srcStart += 2;
+    } else {
+      if (code == QUOTE) {
+        lastIndex = srcStart;
+        srcStart += 2;
+        while (srcStart < srcEnd) {
+          const code = load<u16>(srcStart);
+          if (code == QUOTE && load<u16>(srcStart - 2) !== BACK_SLASH) {
+            while (isSpace(load<u16>(srcStart += 2))) { /* empty */ }
+            console.log("Value (string): " + str(lastIndex, srcStart));
+            // @ts-ignore: exists
+            schema.__DESERIALIZE(keyStart, keyEnd, lastIndex, srcStart);
+            keyStart = 0;
+            break;
+          }
+          srcStart += 2;
+        }
+      } else if (code - 48 <= 9 || code == 45) {
+        lastIndex = srcStart;
+        srcStart += 2;
+        while (srcStart < srcEnd) {
+          const code = load<u16>(srcStart);
+          if (code == COMMA || code == BRACE_RIGHT || isSpace(code)) {
+            // @ts-ignore: exists
+            schema.__DESERIALIZE(keyStart, keyEnd, lastIndex, srcStart);
+            console.log("Value (number): " + str(lastIndex, srcStart));
+            while (isSpace(load<u16>(srcStart += 2))) { /* empty */ }
+            keyStart = 0;
+            break;
+          }
+          srcStart += 2;
+        }
+      } else if (code == BRACE_LEFT) {
+        lastIndex = srcStart;
+        depth++;
+        srcStart += 2;
+        while (srcStart < srcEnd) {
+          const code = load<u16>(srcStart);
+          if (((code ^ BRACE_RIGHT) | (code ^ BRACKET_RIGHT)) == 32) {
+            if (--depth == 0) {
+              // @ts-ignore: exists
+              schema.__DESERIALIZE(keyStart, keyEnd, lastIndex, srcStart);
+              console.log("Value (object): " + str(lastIndex, srcStart));
+              keyStart = 0;
+              while (isSpace(load<u16>(srcStart += 2))) { /* empty */ }
+              break;
+            }
+          } else if (((code ^ BRACE_LEFT) | (code ^ BRACKET_LEFT)) == 220) depth++;
+          srcStart += 2;
+        }
+      } else if (code == CHAR_T) {
+        if (load<u64>(srcStart) == 28429475166421108) {
+          // @ts-ignore: exists
+          schema.__DESERIALIZE(keyStart, keyEnd, srcStart, srcStart += 8);
+          console.log("Value (bool): " + str(srcStart - 8, srcStart));
+          while (isSpace(load<u16>(srcStart += 2))) { /* empty */ }
+          keyStart = 0;
+        }
+      } else if (code == CHAR_F) {
+        if (load<u64>(srcStart, 2) == 28429466576093281) {
+          // @ts-ignore: exists
+          schema.__DESERIALIZE(keyStart, keyEnd, srcStart, srcStart += 10);
+          console.log("Value (bool): " + str(srcStart - 10, srcStart));
+          while (isSpace(load<u16>(srcStart += 2))) { /* empty */ }
+          keyStart = 0;
+        }
+      } else if (code == CHAR_N) {
+        if (load<u64>(srcStart) == 30399761348886638) {
+          // @ts-ignore: exists
+          schema.__DESERIALIZE(keyStart, keyEnd, srcStart, srcStart += 8);
+          console.log("Value (null): " + str(srcStart - 8, srcStart));
+          while (isSpace(load<u16>(srcStart += 2))) { /* empty */ }
+        }
+      }
+    }
+  }
+  return schema;
+}
+
+function str(start: usize, end: usize): string {
+  const size = end - start;
+  const out = __new(size, idof<string>());
+  memory.copy(out, start, size);
+  return changetype<string>(out);
+}
diff --git a/assembly/deserialize/simple/string.ts b/assembly/deserialize/simple/string.ts
@@ -1,6 +1,8 @@
 import { CHAR_B, BACK_SLASH, BACKSPACE, CARRIAGE_RETURN, CHAR_F, FORM_FEED, CHAR_N, NEW_LINE, QUOTE, CHAR_R, CHAR_T, TAB, CHAR_U } from "../../custom/chars";
 import { Sink } from "../../custom/sink";
 import { unsafeCharCodeAt } from "../../custom/util";
+import { DESERIALIZE_ESCAPE_TABLE, ESCAPE_HEX_TABLE } from "../../globals/tables";
+import { bytes } from "../../util/bytes";
 
 // @ts-ignore: Decorator valid here
 @inline export function deserializeString(data: string, start: i32 = 0, end: i32 = 0): string {
@@ -66,3 +68,51 @@ import { unsafeCharCodeAt } from "../../custom/util";
   }
   return result.toString();
 }
+
+// @ts-ignore: Decorator valid here
+@inline export function deserializeString_NEW(srcStart: usize, srcEnd: usize, dst: usize): string {
+  // const srcSize = srcEnd - srcStart;
+  // const dstSize = bytes(dst);
+
+  let dstPtr = dst;
+
+  let lastPtr = srcStart;
+  while (srcStart < srcEnd) {
+    let code = load<u16>(srcStart);
+    if (code == BACK_SLASH) {
+      code = load<u16>(DESERIALIZE_ESCAPE_TABLE + load<u8>(srcStart, 2));
+      if (code == 117 && load<u32>(srcStart, 4) == 3145776) {
+        const block = load<u32>(srcStart, 8);
+        const codeA = block & 0xffff;
+        const codeB = (block >> 16) & 0xffff;
+        const escapedA = load<u8>(ESCAPE_HEX_TABLE + codeA);
+        const escapedB = load<u8>(ESCAPE_HEX_TABLE + codeB);
+        const escaped = (escapedA << 4) + escapedB;
+        const remBytes = srcStart - lastPtr;
+        memory.copy(dstPtr, lastPtr, remBytes);
+        dstPtr += remBytes;
+        store<u16>(dst, escaped);
+        dstPtr += 2;
+        srcStart += 12;
+        lastPtr = srcStart;
+      } else {
+        const remBytes = srcStart - lastPtr;
+        memory.copy(dstPtr, lastPtr, remBytes);
+        dstPtr += remBytes;
+        store<u16>(dstPtr, code);
+        dstPtr += 2;
+        srcStart += 4;
+        lastPtr = srcStart;
+      }
+    } else {
+      srcStart += 2;
+    }
+  }
+
+  const remBytes = srcEnd - lastPtr;
+  memory.copy(dstPtr, lastPtr, remBytes);
+  dstPtr += remBytes;
+
+  if (lastPtr != srcStart) dst = __renew(dst, dstPtr - dst);
+  return changetype<string>(dst);
+}