Merge pull request #54 from hyperoracle/dev

Dev - ready for v1.0

README.md

@@ -2,7 +2,19 @@
 
 ## Getting Started
 
-To create your zkGraph project based on this template, click `Use this template`, and `Creating a new repository`.
+To create your zkGraph project based on this template:
+
+Option 1:
+
+Click `Use this template`, and `Creating a new repository`.
+
+Option 2:
+
+Use `gh` cli
+
+```bash
+gh repo create zkgraph-new --public --template="https://github.com/hyperoracle/zkgraph.git"
+```
 
 ### Configuration
 
@@ -11,9 +23,12 @@ After clone your project, you need to create `config.js` file at root folder bas
 ```js
 // ./config.js
 export const config = {
-  // Etherum JSON RPC provider URL:
-  // (Please note the provider must support debug_getRawReceipts RPC method.)
+  // Update your Etherum JSON RPC provider URL here.
+  // Please note that the provider must support debug_getRawReceipts RPC method.
+  // Recommended provider: ANKR.
   JsonRpcProviderUrl: "https://{URL}",
+  UserPrivateKey: "0x{PRIVATE_KEY}",
+  // ...and other configs.
 };
 ```
 
@@ -49,11 +64,44 @@ npm run compile-local
 npm run exec-local -- {block_id}
 ```
 
-### Prove Local (input generation / pre-test)
+### Set Up Local Image
+
+```bash
+npm run setup-local
+```
+
+### Prove Local Image (input generation / pre-test / prove)
 
 ```bash
 npm run prove-local -- --inputgen {block_id} {expected_state}
 npm run prove-local -- --pretest {block_id} {expected_state}
+npm run prove-local -- --prove {block_id} {expected_state}
+```
+
+### Compile (with Compile Server)
+
+```bash
+npm run compile
+```
+
+### Set Up Image (Link Compiled with Compiler Server)
+
+```bash
+npm run setup
+```
+
+### Prove Full Image (Link Compiled with Compiler Server)
+
+```bash
+npm run prove -- --inputgen {block_id} {expected_state}
+npm run prove -- --pretest {block_id} {expected_state}
+npm run prove -- --prove {block_id} {expected_state}
+```
+
+### Verifier Contract Interface
+
+```AggregatorVerifier
+https://github.com/DelphinusLab/halo2aggregator-s/blob/main/sol/contracts/AggregatorVerifier.sol#L40
 ```
 
 ## Develop
@@ -114,16 +162,16 @@ More info and API reference can be found in [Hyper Oracle zkGraph docs](https://
 ## Lib Dev Tips
 
 1. Don't use `I8.parseInt` because it will be compiled to `i32.extend8_s (aka. Unknown opcode 192 (0xC0) in WASM)`.
-2. Don't use template literals (`${}`), for example when throwing errors, because it will be compiled to too many WASM instructions (~1000 diff).
-3. Don't use `FC extensions` opcodes, because it will be compiled to `Unknown opcode 252 (0xFC) in WASM`.
+2. Try not to use template literals (`${}`), for example when throwing errors, because it will be compiled to too many WASM instructions (~1000 diff).
+3. Try not to use `FC extensions` opcodes (`<u32>parseInt(...)`, `f64`, or `Math`), because it will be compiled to `Unknown opcode 252 (0xFC) in WASM`, and generates too many instructions.
 
 References: [WebAssembly Opcodes](https://pengowray.github.io/wasm-ops/).
 
 ## Structure
 
 This repo has the following folders relevant to zkGraph development:
 
-- `bundle-js`: APIs (the scripts in `package.json`) for compile, execute, prove, and deploy zkGraph for testing locally.
+- `api`: APIs (the scripts in `package.json`) for compile, execute, prove, and deploy zkGraph for testing locally, and fully with zkWASM node.
 - `example`: Example zkGraphs.
 - `lib`: AssemblyScript library for zkGraph development, with data structure such as Bytes, ByteArray and BigInt.
 - `src`: Where your actual zkGraph should be in. Contains `mapping.ts` and `zkgraph.yaml`.

bundle-js/common/api_helper.js → api/common/api_helper.js

@@ -62,12 +62,26 @@ export function genStreamAndMatchedEventOffsets(rawreceiptList, eventList) {
 }
 
 // Format inputs with length and input value
-export function formatVarLenInput(inputs) {
-  var formatted = "";
-  inputs.map((input) => {
-    formatted += `0x${Math.ceil(input.length / 2).toString(
-      16,
-    )}:i64 0x${input}:bytes-packed `;
-  });
+export function formatIntInput(input) {
+  return `0x${input.toString(16)}:i64 `;
+}
+
+// Format bytes input
+export function formatHexStringInput(input) {
+  return `0x${trimPrefix(input, "0x")}:bytes-packed `;
+}
+
+// Format inputs with length and input value
+export function formatVarLenInput(input) {
+  //   var formatted = "";
+  //   inputs.map((input) => {
+  //     var inp = trimPrefix(input, '0x')
+  //     formatted += `${formatIntInput(Math.ceil(inp.length / 2))}${formatHexStringInput(inp)}`;
+  //   });
+
+  var inp = trimPrefix(input, "0x");
+  var formatted = `${formatIntInput(
+    Math.ceil(inp.length / 2),
+  )}${formatHexStringInput(inp)}`;
   return formatted;
 }

bundle-js/common/bundle_local.js → api/common/bundle.js

@@ -22,6 +22,14 @@ async function instantiate(module, imports = {}) {
         // lib/common/zkwasm/wasm_input(i32) => i64
         return zkwasmmock.wasm_input(x) || 0n;
       },
+      js_log(arg) {
+        // to compatible with c-wasm
+        console.log(arg);
+      },
+      js_log_u64(arg) {
+        // to compatible with c-wasm
+        console.log(arg);
+      },
     }),
   };
   const { exports } = await WebAssembly.instantiate(module, adaptedImports);
@@ -123,8 +131,28 @@ async function instantiate(module, imports = {}) {
   }
   return adaptedExports;
 }
-export const { memory, asmain, zkmain } = await (async (url) =>
-  instantiate(
+// export const { memory, asmain, zkmain } = await (async (url) =>
+//   instantiate(
+//     await (async () => {
+//       try {
+//         return await globalThis.WebAssembly.compileStreaming(
+//           globalThis.fetch(url),
+//         );
+//       } catch {
+//         return globalThis.WebAssembly.compile(
+//           await (await import("node:fs/promises")).readFile(url),
+//         );
+//       }
+//     })(),
+//     {},
+//   ))(new URL("../../build/zkgraph_full.wasm", import.meta.url));
+
+export const instantiateWasm = async (wasmpath) => {
+  // update this when move bundle.js
+  let curPathToRoot = "../../";
+
+  let url = new URL(curPathToRoot + wasmpath, import.meta.url);
+  return instantiate(
     await (async () => {
       try {
         return await globalThis.WebAssembly.compileStreaming(
@@ -137,4 +165,5 @@ export const { memory, asmain, zkmain } = await (async (url) =>
       }
     })(),
     {},
-  ))(new URL("../../build/zkgraph_local.wasm", import.meta.url));
+  );
+};

bundle-js/common/config.js → api/common/config_utils.js

@@ -13,7 +13,7 @@ function loadYaml(fname) {
   }
 }
 
-export function loadConfig(fname) {
+export function loadZKGraphConfig(fname) {
   const config = loadYaml(fname);
   const source_address = config.dataSources[0].source.address;
 
@@ -36,4 +36,4 @@ export function loadConfig(fname) {
   return [source_address, source_esigs];
 }
 
-export function applyConfig(configObj) {} //placeholder
+export function applyZKGraphConfig(configObj) {} //placeholder

api/common/ethers_helper.js

@@ -0,0 +1,16 @@
+import { ethers } from "ethers";
+
+export async function getRawReceipts(ethersProvider, blockid) {
+  if (Number.isFinite(blockid)) {
+    blockid = "0x" + blockid.toString(16);
+  }
+  return ethersProvider.send("debug_getRawReceipts", [blockid]);
+}
+
+export async function getBlockByNumber(ethersProvider, blockNumber) {
+  const fullBlock = await ethersProvider.send("eth_getBlockByNumber", [
+    ethers.utils.hexValue(blockNumber),
+    false,
+  ]);
+  return fullBlock;
+}

api/common/utils.js

@@ -0,0 +1,118 @@
+export function fromHexString(hexString) {
+  hexString = hexString.startsWith("0x") ? hexString.slice(2) : hexString;
+  hexString = hexString.length % 2 ? "0" + hexString : hexString;
+  return Uint8Array.from(Buffer.from(hexString, "hex"));
+}
+
+export function toHexString(bytes) {
+  return Buffer.from(bytes).toString("hex");
+}
+
+export function toHexStringBytes32Reverse(arr) {
+  let result = "";
+  for (let i = 0; i < arr.length / 32; i++) {
+    result +=
+      "0x" + toHexString(arr.slice(i * 32, (i + 1) * 32).reverse()) + "\n";
+  }
+  return result;
+}
+
+export function areEqualArrays(first, second) {
+  return (
+    first.length === second.length &&
+    first.every((value, index) => value === second[index])
+  );
+}
+
+export function trimPrefix(str, prefix) {
+  if (str.startsWith(prefix)) {
+    str = str.substring(prefix.length);
+  }
+  return str;
+}
+
+export function logDivider() {
+  const line = "=".repeat(process.stdout.columns);
+  console.log(line);
+}
+
+export function logLoadingAnimation() {
+  // If width is equal to process.stdout.columns, the bar will overflow into the next line.
+  // 4 is the length of the prefix "[*] ".
+  // 55 is about the same length as the longest message in this script.
+  const width = Math.min(process.stdout.columns - 4, 55);
+  let frame = 0;
+  let stop = false;
+
+  const frames = ["▓"];
+  let position = 0;
+  const intervalId = setInterval(() => {
+    if (stop) {
+      clearInterval(intervalId);
+      process.stdout.clearLine();
+      process.stdout.cursorTo(0);
+      return;
+    }
+
+    const currentFrame = frames[frame % frames.length];
+    const loadingBar = `[*] ${currentFrame.repeat(
+      position,
+    )}▒${currentFrame.repeat(width - position - 1)}`;
+
+    process.stdout.cursorTo(0);
+    process.stdout.write(loadingBar);
+
+    position = (position + 1) % width;
+
+    frame++;
+  }, 400);
+
+  return {
+    stopAndClear: () => {
+      stop = true;
+      process.stdout.clearLine();
+      process.stdout.cursorTo(0);
+    },
+  };
+}
+
+export function concatHexStrings(hexStrings) {
+  let result = "";
+  for (let hexString of hexStrings) {
+    result += hexString.startsWith("0x") ? hexString.slice(2) : hexString;
+  }
+  return "0x" + result;
+}
+
+export function currentNpmScriptName() {
+  return process.env.npm_lifecycle_event;
+}
+
+export function logReceiptAndEvents(
+  rawreceiptList,
+  blockid,
+  matchedEventOffsets,
+  filteredEventList,
+) {
+  console.log(
+    "[*]",
+    rawreceiptList.length,
+    rawreceiptList.length > 1
+      ? "receipts fetched from block"
+      : "receipt fetched from block",
+    blockid,
+  );
+  console.log(
+    "[*]",
+    matchedEventOffsets.length / 7,
+    matchedEventOffsets.length / 7 > 1 ? "events matched" : "event matched",
+  );
+  for (let i in filteredEventList) {
+    for (let j in filteredEventList[i]) {
+      filteredEventList[i][j].prettyPrint(
+        "\tTx[" + i + "]Event[" + j + "]",
+        false,
+      );
+    }
+  }
+}