feat: Modify bb.js to be compatible with next.js (#544)

Co-authored-by: ludamad <[email protected]>

acir_tests/Dockerfile

@@ -10,4 +10,4 @@ COPY . .
 # Run all native tests.
 RUN ./run_acir_tests.sh
 # Just run double_verify_proof as a sanity check as bb.js is quite slow.
-RUN BB=../ts/dest/main.js ./run_acir_tests.sh double_verify_proof
+RUN BB=./run_bb.sh ./run_acir_tests.sh double_verify_proof

acir_tests/run_bb.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Used to call this script from a stable path
+DIR=$(dirname "$0")
+exec node "$DIR/../ts/dest/node/main.js" $@

ts/.eslintrc.cjs

@@ -26,6 +26,13 @@ const contexts = [
   'EnumExpression',
 ];
 
+let tsconfigPaths;
+if (process.env.DOCKER_ENV) {
+  tsconfigPaths = ['./tsconfig.node.json', './tsconfig.browser.json'];
+} else {
+  tsconfigPaths = ['./ts/tsconfig.node.json', './ts/tsconfig.browser.json'];
+}
+
 module.exports = {
   extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended', 'prettier'],
   root: true,
@@ -36,7 +43,7 @@ module.exports = {
     {
       files: ['*.ts', '*.tsx'],
       parserOptions: {
-        project: true,
+        project: tsconfigPaths,
       },
     },
     {

ts/.gitignore

@@ -3,5 +3,7 @@
 node_modules
 dest
 .tsbuildinfo
+.tsbuildinfo.browser
+.tsbuildinfo.node
 *.log
 crs

ts/Dockerfile

@@ -2,6 +2,7 @@
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-wasm-linux-clang
 
 FROM node:18-alpine
+
 COPY --from=0 /usr/src/barretenberg /usr/src/barretenberg
 WORKDIR /usr/src/barretenberg/ts
 COPY .yarn .yarn
@@ -10,5 +11,6 @@ COPY yarn.lock yarn.lock
 COPY .yarnrc.yml .yarnrc.yml
 RUN yarn --immutable
 COPY . .
-RUN yarn formatting && yarn build:ts
-CMD ["yarn", "test"]
+ENV DOCKER_ENV=true
+RUN yarn formatting && yarn build:ts:browser && yarn build:ts:node
+CMD ["yarn", "test"]

ts/bb.js-dev

@@ -2,5 +2,5 @@
 # Add a symlink to this somewhere in your path.
 # Now you can run bb.js-dev anywhere to execute latest code, no 'yarn build' required.
 SCRIPT_PATH=$(dirname $(realpath $0))
-export TS_NODE_PROJECT="$SCRIPT_PATH/tsconfig.json"
-NODE_OPTIONS="--loader $SCRIPT_PATH/node_modules/ts-node/esm/transpile-only.mjs --no-warnings" node $SCRIPT_PATH/src/main.ts $@
+export TS_NODE_PROJECT="$SCRIPT_PATH/tsconfig.node.json"
+npx nodemon --ext '.ts' --watch 'src/**/*.ts' --exec "ts-node -r tsconfig-paths/register" $SCRIPT_PATH/src/main.ts $@

ts/package.json

@@ -8,52 +8,41 @@
     "displayName": "bb.js",
     "tsconfig": "./tsconfig.json"
   },
+  "main": "./dest/index.js",
   "bin": "./dest/main.js",
   "files": [
     "src/",
     "dest/",
+    "!dest/browser/*.wasm",
     "README.md"
   ],
   "scripts": {
-    "clean": "rm -rf ./dest .tsbuildinfo",
-    "build": "yarn clean && yarn build:wasm && yarn build:ts",
+    "clean": "rm -rf ./dest .tsbuildinfo.browser .tsbuildinfo.node",
+    "build": "yarn clean && yarn build:wasm && yarn build:ts:browser && yarn build:ts:node",
     "build:dev": "tsc -b --watch",
     "build:wasm": "cd ../cpp && cmake --preset wasm-threads && cmake --build --preset wasm-threads && cmake --preset wasm && cmake --build --preset wasm",
-    "build:ts": "tsc -b && webpack && chmod +x ./dest/main.js",
+    "build:ts:browser": "tsc -b tsconfig.browser.json && BUILD_TARGET=browser webpack && chmod +x ./dest/browser/main.js && BUILD_TARGET=browser node replace_imports.cjs",
+    "build:ts:node": "tsc -b tsconfig.node.json && chmod +x ./dest/node/main.js && BUILD_TARGET=node node replace_imports.cjs",
     "build:bindings": "cd .. && ./scripts/bindgen.sh",
     "serve": "webpack serve",
     "formatting": "prettier --check ./src && eslint --max-warnings 0 ./src",
     "formatting:fix": "prettier -w ./src",
-    "test": "yarn test:jest && yarn test:bin",
-    "test:jest": "NODE_OPTIONS='--loader ts-node/esm' NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --no-cache --passWithNoTests",
+    "test": "yarn build:ts:browser && yarn build:ts:node && yarn test:jest && yarn test:bin",
+    "test:jest": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --no-cache --passWithNoTests",
     "test:bin": "cd ./bin-test && ./bin-test.sh",
-    "test-debug": "NODE_OPTIONS='--loader ts-node/esm' NODE_NO_WARNINGS=1 node --inspect-brk=0.0.0.0 --experimental-vm-modules $(yarn bin jest) --no-cache --passWithNoTests --runInBand",
-    "simple_test": "NODE_OPTIONS='--loader ts-node/esm' NODE_NO_WARNINGS=1 node ./src/examples/simple.rawtest.ts",
+    "test-debug": "NODE_NO_WARNINGS=1 node --inspect-brk=0.0.0.0 --experimental-vm-modules $(yarn bin jest) --no-cache --passWithNoTests --runInBand",
     "prepack": "yarn build",
     "deploy": "npm publish --access public"
   },
   "jest": {
-    "preset": "ts-jest/presets/default-esm",
-    "transform": {
-      "./src/.*\\.ts": [
-        "ts-jest",
-        {
-          "useESM": true
-        }
-      ]
-    },
-    "moduleNameMapper": {
-      "^(\\.{1,2}/.*)\\.js$": "$1"
-    },
-    "testRegex": "./src/.*\\.test\\.ts$",
-    "rootDir": "./src"
+    "testRegex": "./dest/node/.*\\.test\\.js$",
+    "rootDir": "./dest"
   },
   "dependencies": {
     "comlink": "^4.4.1",
     "commander": "^10.0.1",
     "debug": "^4.3.4",
-    "ts-node": "^10.9.1",
-    "tslib": "^2.4.0"
+    "idb-keyval": "^6.2.1"
   },
   "devDependencies": {
     "@jest/globals": "^29.4.3",
@@ -69,12 +58,13 @@
     "eslint": "^8.35.0",
     "eslint-config-prettier": "^8.8.0",
     "html-webpack-plugin": "^5.5.1",
-    "idb-keyval": "^6.2.1",
     "jest": "^29.5.0",
     "prettier": "^2.8.4",
+    "replace-in-file": "^7.0.1",
     "resolve-typescript-plugin": "^2.0.1",
     "ts-jest": "^29.1.0",
     "ts-loader": "^9.4.2",
+    "tsconfig-paths-webpack-plugin": "^4.0.1",
     "typescript": "^5.0.4",
     "webpack": "^5.82.1",
     "webpack-cli": "^5.1.1",

ts/replace_imports.cjs

@@ -0,0 +1,33 @@
+const replaceInFile = require('replace-in-file');
+const fs = require('fs');
+const path = require('path');
+
+const buildTarget = process.env.BUILD_TARGET;
+const dynamic_imports = ['barretenberg_wasm', 'crs', 'random', 'types'];
+
+async function replaceImports() {
+  try {
+    dynamic_imports.forEach(async item => {
+      await replaceInFile({
+        files: path.resolve(__dirname, `dest/${buildTarget}/${item}/*`),
+        from: new RegExp(`'dynamic\\/${item}';`, 'g'),
+        to: `'./${buildTarget}/index.js';`,
+      });
+    });
+    const filePath = path.resolve(__dirname, `dest/${buildTarget}/barretenberg_wasm/${buildTarget}/index.js`);
+    // Grab the contents for a hacky check if this has ran twice
+    const contents = fs.readFileSync(filePath, 'utf8');
+    // hack to allow for shared .wasm files between build targets
+    if (contents.includes('../../') && !contents.includes('../../../')) {
+      await replaceInFile({
+        files: filePath,
+        from: /\.\.\/\.\.\//g,
+        to: `../../../`,
+      });
+    }
+  } catch (error) {
+    console.error('Error occurred:', error);
+  }
+}
+
+replaceImports();

ts/src/barretenberg_wasm/barretenberg_wasm.test.ts

@@ -37,7 +37,7 @@ describe('barretenberg wasm worker', () => {
   let wasm!: BarretenbergWasmWorker;
 
   beforeAll(async () => {
-    ({ wasm, worker } = await BarretenbergWasm.newWorker(2));
+    ({ wasm, worker } = (await BarretenbergWasm.newWorker(2)) as any);
   }, 20000);
 
   afterAll(async () => {

ts/src/barretenberg_wasm/barretenberg_wasm.ts

@@ -3,10 +3,14 @@ import { EventEmitter } from 'events';
 import createDebug from 'debug';
 import { Remote, proxy } from 'comlink';
 import { randomBytes } from '../random/index.js';
-// Webpack config swaps this import with ./browser/index.js
-// You can toggle between these two imports to sanity check the type-safety.
-import { fetchCode, getNumCpu, createWorker, getRemoteBarretenbergWasm, threadLogger, killSelf } from './node/index.js';
-// import { fetchCode, getNumCpu, createWorker, randomBytes } from './browser/index.js';
+import {
+  fetchCode,
+  getNumCpu,
+  createWorker,
+  getRemoteBarretenbergWasm,
+  threadLogger,
+  killSelf,
+} from 'dynamic/barretenberg_wasm';
 
 const debug = createDebug('bb.js:wasm');
 
@@ -68,7 +72,7 @@ export class BarretenbergWasm {
 
     // Annoyingly the wasm declares if it's memory is shared or not. So now we need two wasms if we want to be
     // able to fallback on "non shared memory" situations.
-    const code = await fetchCode(threads > 1 ? 'barretenberg-threads.wasm' : 'barretenberg.wasm');
+    const code = await fetchCode(threads > 1);
     const { instance, module } = await WebAssembly.instantiate(code, this.getImportObj(this.memory));
 
     this.instance = instance;
@@ -78,8 +82,8 @@ export class BarretenbergWasm {
 
     // Create worker threads. Create 1 less than requested, as main thread counts as a thread.
     this.logger('creating worker threads...');
-    this.workers = await Promise.all(Array.from({ length: threads - 1 }).map(createWorker));
-    this.remoteWasms = await Promise.all(this.workers.map(getRemoteBarretenbergWasm));
+    this.workers = (await Promise.all(Array.from({ length: threads - 1 }).map(createWorker))) as any;
+    this.remoteWasms = await Promise.all(this.workers.map(getRemoteBarretenbergWasm as any));
     await Promise.all(this.remoteWasms.map(w => w.initThread(module, this.memory)));
     this.logger('init complete.');
   }

ts/src/barretenberg_wasm/browser/index.ts

@@ -2,13 +2,16 @@ import { wrap } from 'comlink';
 import { BarretenbergWasmWorker, type BarretenbergWasm } from '../barretenberg_wasm.js';
 import debug from 'debug';
 
-export async function fetchCode(name: string) {
-  const res = await fetch('/' + name);
+export async function fetchCode(multithreading: boolean) {
+  const wasmModuleUrl = multithreading
+    ? new URL(`../../barretenberg-threads.wasm`, import.meta.url)
+    : new URL(`../../barretenberg.wasm`, import.meta.url);
+  const res = await fetch(wasmModuleUrl.href);
   return await res.arrayBuffer();
 }
 
 export function createWorker() {
-  const worker = new Worker('barretenberg_wasm.js');
+  const worker = new Worker(new URL(`./worker.js`, import.meta.url));
   const debugStr = debug.disable();
   debug.enable(debugStr);
   worker.postMessage({ debug: debugStr });

ts/src/barretenberg_wasm/node/index.ts

@@ -8,9 +8,9 @@ import { wrap } from 'comlink';
 import { nodeEndpoint } from './node_endpoint.js';
 import { writeSync } from 'fs';
 
-export async function fetchCode(name: string) {
+export async function fetchCode(multithreading: boolean) {
   const __dirname = dirname(fileURLToPath(import.meta.url));
-  return await readFile(__dirname + '/../../' + name);
+  return await readFile(__dirname + `/../../${multithreading ? 'barretenberg-threads.wasm' : 'barretenberg.wasm'}`);
 }
 
 export function createWorker() {

ts/src/crs/browser/cached_net_crs.ts

@@ -10,7 +10,9 @@ export class CachedNetCrs {
 
   constructor(public readonly numPoints: number) {}
 
-  static async new(numPoints: number) {
+  // This is to keep signrature equal with the node version of CRS
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  static async new(numPoints: number, _?: string) {
     const crs = new CachedNetCrs(numPoints);
     await crs.init();
     return crs;

ts/src/crs/index.ts

@@ -1 +1 @@
-export { Crs } from './node/index.js';
+export { Crs } from 'dynamic/crs';

ts/src/random/index.ts

@@ -1 +1 @@
-export * from './node/index.js';
+export * from 'dynamic/random';

ts/src/types/browser/index.ts

@@ -0,0 +1 @@
+export * from './point.js';

ts/src/types/browser/point.ts

@@ -0,0 +1,50 @@
+import { Fr } from '../index.js';
+import { BufferReader } from '../../serialize/buffer_reader.js';
+
+export class Point {
+  static SIZE_IN_BYTES = 64;
+  static EMPTY = new Point(Fr.ZERO, Fr.ZERO);
+
+  constructor(public readonly x: Fr, public readonly y: Fr) {}
+
+  static random() {
+    // TODO: This is not a point on the curve!
+    return new Point(Fr.random(), Fr.random());
+  }
+
+  static fromBuffer(buffer: Uint8Array | BufferReader) {
+    const reader = BufferReader.asReader(buffer);
+    return new this(Fr.fromBuffer(reader), Fr.fromBuffer(reader));
+  }
+
+  static fromString(address: string) {
+    address = address.replace(/^0x/i, '');
+    const byteValues = new Uint8Array(Math.ceil(address.length / 2));
+    for (let i = 0; i < byteValues.length; i++) {
+      byteValues[i] = Number.parseInt(address.substr(i * 2, 2), 16);
+    }
+    return Point.fromBuffer(byteValues);
+  }
+
+  toBuffer() {
+    const xBuffer = this.x.toBuffer();
+    const yBuffer = this.y.toBuffer();
+    const combined = new Uint8Array(xBuffer.length + yBuffer.length);
+    combined.set(xBuffer, 0);
+    combined.set(yBuffer, xBuffer.length);
+    return combined;
+  }
+
+  toString() {
+    const buffer = this.toBuffer();
+    let hexString = '0x';
+    for (let i = 0; i < buffer.length; i++) {
+      hexString += buffer[i].toString(16).padStart(2, '0');
+    }
+    return hexString;
+  }
+
+  equals(rhs: Point) {
+    return this.x.equals(rhs.x) && this.y.equals(rhs.y);
+  }
+}

ts/src/types/index.ts

@@ -1,5 +1,5 @@
 export * from './ptr.js';
 export * from './fields.js';
-export * from './point.js';
+export * from 'dynamic/types';
 export * from './fixed_size_buffer.js';
 export * from './raw_buffer.js';

ts/src/types/node/index.ts

@@ -0,0 +1 @@
+export * from './point.js';

ts/src/types/point.ts → ts/src/types/node/point.ts

@@ -1,5 +1,5 @@
-import { Fr } from './index.js';
-import { BufferReader } from '../serialize/buffer_reader.js';
+import { Fr } from '../index.js';
+import { BufferReader } from '../../serialize/buffer_reader.js';
 
 export class Point {
   static SIZE_IN_BYTES = 64;