From 18ec31dec8fdb702efea65f52c4c0c5395bb4344 Mon Sep 17 00:00:00 2001 From: Aleix Date: Fri, 1 Dec 2023 13:04:58 +0100 Subject: [PATCH 1/5] Downloading anon circuits using webworkers --- .../src/election/use-election-provider.ts | 32 ++++++++-- .../src/worker/circuitWorkerScript.ts | 60 +++++++++++++++++++ .../src/worker/useWebWorker.ts | 37 ++++++++++++ .../react-providers/src/worker/webWorker.ts | 5 ++ 4 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 packages/react-providers/src/worker/circuitWorkerScript.ts create mode 100644 packages/react-providers/src/worker/useWebWorker.ts create mode 100644 packages/react-providers/src/worker/webWorker.ts diff --git a/packages/react-providers/src/election/use-election-provider.ts b/packages/react-providers/src/election/use-election-provider.ts index 5122b89c..45a512e6 100644 --- a/packages/react-providers/src/election/use-election-provider.ts +++ b/packages/react-providers/src/election/use-election-provider.ts @@ -7,9 +7,13 @@ import { PublishedElection, Vote, } from '@vocdoni/sdk' -import { ComponentType, useCallback, useEffect } from 'react' +import { ComponentType, useCallback, useEffect, useMemo, useState } from 'react' import { useClient } from '../client' import { useElectionReducer } from './use-election-reducer' +import { ChainAPI } from '@vocdoni/sdk' +import { createWebWorker } from '../worker/webWorker' +import { useWebWorker } from '../worker/useWebWorker' +import worker, { ICircuit, ICircuitWorkerRequest } from '../worker/circuitWorkerScript' export type ElectionProviderProps = { id?: string @@ -40,6 +44,7 @@ export const useElectionProvider = ({ loaded, sik: { password, signature }, } = state + const [anonCircuitsFetched, setAnonCircuitsFetched] = useState(false) const fetchElection = useCallback( async (id: string) => { @@ -87,7 +92,10 @@ export const useElectionProvider = ({ } }, [actions, client, election, password, signature]) - const fetchAnonCircuits = useCallback(() => { + const workerInstance = useMemo(() => createWebWorker(worker), []) + const { result: circuits, startProcessing } = useWebWorker(workerInstance) + + const fetchAnonCircuits = useCallback(async () => { const hasOverwriteEnabled = typeof election !== 'undefined' && typeof election.voteType.maxVoteOverwrites !== 'undefined' && @@ -96,16 +104,32 @@ export const useElectionProvider = ({ const votable = state.isAbleToVote || (hasOverwriteEnabled && state.isInCensus && state.voted) if (votable && election?.census.type === CensusType.ANONYMOUS) { - client.anonymousService.fetchCircuits() + const chainCircuits = await ChainAPI.circuits(client.url) + const circuits = { + zKeyURI: chainCircuits.uri + '/' + chainCircuits.circuitPath + '/' + chainCircuits.zKeyFilename, + zKeyHash: chainCircuits.zKeyHash, + vKeyURI: chainCircuits.uri + '/' + chainCircuits.circuitPath + '/' + chainCircuits.vKeyFilename, + vKeyHash: chainCircuits.vKeyHash, + wasmURI: chainCircuits.uri + '/' + chainCircuits.circuitPath + '/' + chainCircuits.wasmFilename, + wasmHash: chainCircuits.wasmHash, + } + startProcessing({ circuits }) } }, [election, state.isAbleToVote, state.isInCensus, state.voted, client.anonymousService]) // pre-fetches circuits needed for voting in anonymous elections useEffect(() => { - if (!fetchCensus || !election || loading.census || !client.wallet) return + if (!fetchCensus || !election || loading.census || !client.wallet || anonCircuitsFetched) return fetchAnonCircuits() }, [fetchAnonCircuits, client.wallet, election, loading.census, fetchCensus]) + // sets circuits in the anonymous service + useEffect(() => { + if (!circuits) return + setAnonCircuitsFetched(true) + client.anonymousService.setCircuits(circuits) + }, [circuits]) + // CSP OAuth flow // As vote setting and voting token are async, we need to wait for both to be set useEffect(() => { diff --git a/packages/react-providers/src/worker/circuitWorkerScript.ts b/packages/react-providers/src/worker/circuitWorkerScript.ts new file mode 100644 index 00000000..62eef0b8 --- /dev/null +++ b/packages/react-providers/src/worker/circuitWorkerScript.ts @@ -0,0 +1,60 @@ +/* eslint-disable no-restricted-globals */ +/* eslint-disable import/no-anonymous-default-export */ + +import { IBaseWorkerResponse } from './useWebWorker' + +export interface ICircuit { + zKeyData: Uint8Array + zKeyHash: string + zKeyURI: string + vKeyData: Uint8Array + vKeyHash: string + vKeyURI: string + wasmData: Uint8Array + wasmHash: string + wasmURI: string +} + +export interface ICircuitWorkerRequest { + circuits: { + zKeyURI: string + zKeyHash: string + vKeyURI: string + vKeyHash: string + wasmURI: string + wasmHash: string + } +} + +export interface IWorkerResponse extends IBaseWorkerResponse {} + +export default () => { + self.addEventListener('message', async (e: MessageEvent) => { + console.time('Worker run') + try { + const { circuits } = e.data + + const circuitsData: ICircuit = { + zKeyData: await fetch(circuits.zKeyURI) + .then((res) => res.arrayBuffer()) + .then((res) => new Uint8Array(res)), + zKeyURI: circuits.zKeyURI, + zKeyHash: circuits.zKeyHash, + vKeyData: await fetch(circuits.vKeyURI) + .then((res) => res.arrayBuffer()) + .then((res) => new Uint8Array(res)), + vKeyURI: circuits.vKeyURI, + vKeyHash: circuits.vKeyHash, + wasmData: await fetch(circuits.wasmURI) + .then((res) => res.arrayBuffer()) + .then((res) => new Uint8Array(res)), + wasmURI: circuits.wasmURI, + wasmHash: circuits.wasmHash, + } + console.timeEnd('Worker run') + return postMessage({ result: circuitsData } as IWorkerResponse) + } catch (error) { + return postMessage({ error } as IWorkerResponse) + } + }) +} diff --git a/packages/react-providers/src/worker/useWebWorker.ts b/packages/react-providers/src/worker/useWebWorker.ts new file mode 100644 index 00000000..eb34d364 --- /dev/null +++ b/packages/react-providers/src/worker/useWebWorker.ts @@ -0,0 +1,37 @@ +import { useCallback, useEffect, useState } from 'react' + +export interface IBaseWorkerResponse { + result: T + error?: any +} + +export const useWebWorker = (worker: Worker) => { + const [running, setRunning] = useState(false) + const [error, setError] = useState() + const [result, setResult] = useState() + + const startProcessing = useCallback( + (data: TWorkerPayload) => { + setRunning(true) + worker.postMessage(data) + }, + [worker] + ) + + useEffect(() => { + const onMessage = (event: MessageEvent>) => { + setRunning(false) + setError(event.data.error) + setResult(event.data.result) + } + worker.addEventListener('message', onMessage) + return () => worker.removeEventListener('message', onMessage) + }, [worker]) + + return { + startProcessing, + running, + error, + result, + } +} diff --git a/packages/react-providers/src/worker/webWorker.ts b/packages/react-providers/src/worker/webWorker.ts new file mode 100644 index 00000000..a276fed4 --- /dev/null +++ b/packages/react-providers/src/worker/webWorker.ts @@ -0,0 +1,5 @@ +export const createWebWorker = (worker: any) => { + const code = worker.toString() + const blob = new Blob(['(' + code + ')()']) + return new Worker(URL.createObjectURL(blob)) +} From eeaa338c3cfdf66ff316acd1de680e2ca7f25615 Mon Sep 17 00:00:00 2001 From: Aleix Date: Fri, 1 Dec 2023 22:13:19 +0100 Subject: [PATCH 2/5] Bugfixing for the tests --- .../react-providers/src/worker/circuitWorkerScript.ts | 2 -- packages/react-providers/src/worker/webWorker.ts | 4 +++- setup-tests.ts | 10 +++++++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/react-providers/src/worker/circuitWorkerScript.ts b/packages/react-providers/src/worker/circuitWorkerScript.ts index 62eef0b8..de2b0b5a 100644 --- a/packages/react-providers/src/worker/circuitWorkerScript.ts +++ b/packages/react-providers/src/worker/circuitWorkerScript.ts @@ -30,7 +30,6 @@ export interface IWorkerResponse extends IBaseWorkerResponse {} export default () => { self.addEventListener('message', async (e: MessageEvent) => { - console.time('Worker run') try { const { circuits } = e.data @@ -51,7 +50,6 @@ export default () => { wasmURI: circuits.wasmURI, wasmHash: circuits.wasmHash, } - console.timeEnd('Worker run') return postMessage({ result: circuitsData } as IWorkerResponse) } catch (error) { return postMessage({ error } as IWorkerResponse) diff --git a/packages/react-providers/src/worker/webWorker.ts b/packages/react-providers/src/worker/webWorker.ts index a276fed4..09d67bb7 100644 --- a/packages/react-providers/src/worker/webWorker.ts +++ b/packages/react-providers/src/worker/webWorker.ts @@ -1,5 +1,7 @@ export const createWebWorker = (worker: any) => { const code = worker.toString() const blob = new Blob(['(' + code + ')()']) - return new Worker(URL.createObjectURL(blob)) + const workerURL = (window as any).MockedWindowURL || window.URL || window.webkitURL + const blobURL = workerURL.createObjectURL(blob) + return new Worker(blobURL) } diff --git a/setup-tests.ts b/setup-tests.ts index 2b801822..b7959183 100644 --- a/setup-tests.ts +++ b/setup-tests.ts @@ -25,10 +25,18 @@ class Worker { postMessage(msg) { this.onmessage(msg) } + addEventListener() {} + removeEventListener() {} } // required due to SDK dependency -Object.defineProperty(window, 'Worker', Worker) +Object.defineProperty(window, 'Worker', { value: Worker }) +Object.defineProperty(window, 'MockedWindowURL', { + value: { + createObjectURL: () => 'blob:mocked', + revokeObjectURL: () => {}, + }, +}) // required by any react component (almost all of them) global.React = React From 657ce7a672a43d5666275f8e6d6d07aa696c546f Mon Sep 17 00:00:00 2001 From: Aleix Date: Tue, 5 Dec 2023 13:09:35 +0100 Subject: [PATCH 3/5] Semaphore implemented --- .../src/election/use-election-provider.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/react-providers/src/election/use-election-provider.ts b/packages/react-providers/src/election/use-election-provider.ts index 45a512e6..e1a7cb29 100644 --- a/packages/react-providers/src/election/use-election-provider.ts +++ b/packages/react-providers/src/election/use-election-provider.ts @@ -7,7 +7,7 @@ import { PublishedElection, Vote, } from '@vocdoni/sdk' -import { ComponentType, useCallback, useEffect, useMemo, useState } from 'react' +import { ComponentType, useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useClient } from '../client' import { useElectionReducer } from './use-election-reducer' import { ChainAPI } from '@vocdoni/sdk' @@ -46,6 +46,8 @@ export const useElectionProvider = ({ } = state const [anonCircuitsFetched, setAnonCircuitsFetched] = useState(false) + const isAnonCircuitsFetching = useRef(false) + const fetchElection = useCallback( async (id: string) => { actions.load(id) @@ -119,7 +121,16 @@ export const useElectionProvider = ({ // pre-fetches circuits needed for voting in anonymous elections useEffect(() => { - if (!fetchCensus || !election || loading.census || !client.wallet || anonCircuitsFetched) return + if ( + !fetchCensus || + !election || + loading.census || + !client.wallet || + anonCircuitsFetched || + isAnonCircuitsFetching.current + ) + return + isAnonCircuitsFetching.current = true fetchAnonCircuits() }, [fetchAnonCircuits, client.wallet, election, loading.census, fetchCensus]) From ce2d4588090892adb5be0f0b4754b3f1347ff361 Mon Sep 17 00:00:00 2001 From: Aleix Date: Mon, 11 Dec 2023 12:51:27 +0100 Subject: [PATCH 4/5] Circuits in batch --- .../src/election/use-election-provider.ts | 8 ++- .../src/worker/circuitWorkerScript.ts | 55 ++++++++++++++++--- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/packages/react-providers/src/election/use-election-provider.ts b/packages/react-providers/src/election/use-election-provider.ts index e1a7cb29..01095a10 100644 --- a/packages/react-providers/src/election/use-election-provider.ts +++ b/packages/react-providers/src/election/use-election-provider.ts @@ -136,9 +136,11 @@ export const useElectionProvider = ({ // sets circuits in the anonymous service useEffect(() => { - if (!circuits) return - setAnonCircuitsFetched(true) - client.anonymousService.setCircuits(circuits) + ;(async () => { + if (!circuits) return + setAnonCircuitsFetched(true) + client.anonymousService.setCircuits(circuits) + })() }, [circuits]) // CSP OAuth flow diff --git a/packages/react-providers/src/worker/circuitWorkerScript.ts b/packages/react-providers/src/worker/circuitWorkerScript.ts index de2b0b5a..21160b4d 100644 --- a/packages/react-providers/src/worker/circuitWorkerScript.ts +++ b/packages/react-providers/src/worker/circuitWorkerScript.ts @@ -28,25 +28,62 @@ export interface ICircuitWorkerRequest { export interface IWorkerResponse extends IBaseWorkerResponse {} +export const fetchDataInChunks = async (url: string) => { + return fetch(url) + .then((res) => res.arrayBuffer()) + .then((res) => new Uint8Array(res)) +} + export default () => { self.addEventListener('message', async (e: MessageEvent) => { try { const { circuits } = e.data + async function fetchDataInChunks(uri: string) { + const response = await fetch(uri) + const reader = response.body?.getReader() as ReadableStreamDefaultReader + const contentLength = +response.headers.get('Content-Length') + + const chunks = [] + let receivedLength = 0 + + while (true) { + const { done, value } = await reader.read() + + if (done) break + + chunks.push(value) + receivedLength += value.length + + // Check for content length, break if all content received + if (contentLength && receivedLength >= contentLength) break + } + + const concatenatedArray = new Uint8Array(receivedLength) + let offset = 0 + + for (const chunk of chunks) { + concatenatedArray.set(chunk, offset) + offset += chunk.length + } + + return concatenatedArray + } + + const [zKeyData, vKeyData, wasmData] = await Promise.all([ + fetchDataInChunks(circuits.zKeyURI), + fetchDataInChunks(circuits.vKeyURI), + fetchDataInChunks(circuits.wasmURI), + ]) + const circuitsData: ICircuit = { - zKeyData: await fetch(circuits.zKeyURI) - .then((res) => res.arrayBuffer()) - .then((res) => new Uint8Array(res)), + zKeyData, zKeyURI: circuits.zKeyURI, zKeyHash: circuits.zKeyHash, - vKeyData: await fetch(circuits.vKeyURI) - .then((res) => res.arrayBuffer()) - .then((res) => new Uint8Array(res)), + vKeyData, vKeyURI: circuits.vKeyURI, vKeyHash: circuits.vKeyHash, - wasmData: await fetch(circuits.wasmURI) - .then((res) => res.arrayBuffer()) - .then((res) => new Uint8Array(res)), + wasmData, wasmURI: circuits.wasmURI, wasmHash: circuits.wasmHash, } From 509f2201c416f624bcc94e434a6d6cfdced96f09 Mon Sep 17 00:00:00 2001 From: Aleix Date: Thu, 14 Dec 2023 15:36:01 +0100 Subject: [PATCH 5/5] Bugfixing linter --- packages/react-providers/src/worker/circuitWorkerScript.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-providers/src/worker/circuitWorkerScript.ts b/packages/react-providers/src/worker/circuitWorkerScript.ts index 21160b4d..d0a40e38 100644 --- a/packages/react-providers/src/worker/circuitWorkerScript.ts +++ b/packages/react-providers/src/worker/circuitWorkerScript.ts @@ -42,7 +42,7 @@ export default () => { async function fetchDataInChunks(uri: string) { const response = await fetch(uri) const reader = response.body?.getReader() as ReadableStreamDefaultReader - const contentLength = +response.headers.get('Content-Length') + const contentLength = +(response.headers?.get('Content-Length') ?? 0) const chunks = [] let receivedLength = 0