[web-wasm] Refactor to Web Worker (add camera)

ts/@live-compositor/core/src/api/input.ts

@@ -2,21 +2,38 @@ import type { Api } from '../api.js';
 import type { RegisterMp4Input, RegisterRtpInput, Inputs } from 'live-compositor';
 import { _liveCompositorInternals } from 'live-compositor';
 
-export type RegisterInputRequest = Api.RegisterInput;
+/**
+ * It represents HTTP request that can be sent to
+ * to compositor, but also additional variants that are specific to WASM like camera
+ */
+export type RegisterInputRequest =
+  | Api.RegisterInput
+  | { type: 'camera' }
+  | { type: 'screen_capture' };
 
 export type InputRef = _liveCompositorInternals.InputRef;
 export const inputRefIntoRawId = _liveCompositorInternals.inputRefIntoRawId;
 export const parseInputRef = _liveCompositorInternals.parseInputRef;
 
 export type RegisterInput =
   | ({ type: 'rtp_stream' } & RegisterRtpInput)
-  | ({ type: 'mp4' } & RegisterMp4Input);
+  | ({ type: 'mp4' } & RegisterMp4Input)
+  | { type: 'camera' }
+  | { type: 'screen_capture' };
 
+/**
+ * Converts object passed by user (or modified by platform specific interface) into
+ * HTTP request
+ */
 export function intoRegisterInput(input: RegisterInput): RegisterInputRequest {
   if (input.type === 'mp4') {
     return intoMp4RegisterInput(input);
   } else if (input.type === 'rtp_stream') {
     return intoRtpRegisterInput(input);
+  } else if (input.type === 'camera') {
+    return { type: 'camera' };
+  } else if (input.type === 'screen_capture') {
+    return { type: 'screen_capture' };
   } else {
     throw new Error(`Unknown input type ${(input as any).type}`);
   }

ts/@live-compositor/core/src/live/compositor.ts

@@ -5,14 +5,14 @@ import Output from './output.js';
 import type { CompositorManager } from '../compositorManager.js';
 import type { RegisterOutput } from '../api/output.js';
 import { intoRegisterOutput } from '../api/output.js';
-import type { RegisterInput } from '../api/input.js';
 import { intoRegisterInput } from '../api/input.js';
 import { parseEvent } from '../event.js';
 import { intoRegisterImage, intoRegisterWebRenderer } from '../api/renderer.js';
 import { handleEvent } from './event.js';
 import type { ReactElement } from 'react';
 import type { Logger } from 'pino';
 import type { ImageRef } from '../api/image.js';
+import type { RegisterInput } from '../index.js';
 
 export class LiveCompositor {
   private manager: CompositorManager;

ts/@live-compositor/core/src/offline/compositor.ts

@@ -88,6 +88,8 @@ export class OfflineCompositor {
     const inputRef = { type: 'global', id: inputId } as const;
     const result = await this.api.registerInput(inputRef, intoRegisterInput(request));
 
+    const offsetMs = 'offsetMs' in request && request.offsetMs ? request.offsetMs : 0;
+
     if (request.type === 'mp4' && request.loop) {
       this.store.addInput({
         inputId,
@@ -98,18 +100,18 @@ export class OfflineCompositor {
     } else {
       this.store.addInput({
         inputId,
-        offsetMs: request.offsetMs ?? 0,
+        offsetMs: offsetMs ?? 0,
         videoDurationMs: result.video_duration_ms,
         audioDurationMs: result.audio_duration_ms,
       });
-      if (request.offsetMs) {
-        this.inputTimestamps.push(request.offsetMs);
+      if (offsetMs) {
+        this.inputTimestamps.push(offsetMs);
       }
       if (result.video_duration_ms) {
-        this.inputTimestamps.push((request.offsetMs ?? 0) + result.video_duration_ms);
+        this.inputTimestamps.push((offsetMs ?? 0) + result.video_duration_ms);
       }
       if (result.audio_duration_ms) {
-        this.inputTimestamps.push((request.offsetMs ?? 0) + result.audio_duration_ms);
+        this.inputTimestamps.push((offsetMs ?? 0) + result.audio_duration_ms);
       }
     }
     return result;

ts/@live-compositor/node/src/utils.ts

@@ -1,8 +1,8 @@
-export async function sleep(timeout_ms: number): Promise<void> {
+export async function sleep(timeoutMs: number): Promise<void> {
   await new Promise<void>(res => {
     setTimeout(() => {
       res();
-    }, timeout_ms);
+    }, timeoutMs);
   });
 }
 

ts/@live-compositor/web-wasm/src/compositor.ts → ts/@live-compositor/web-wasm/src/compositor/compositor.ts

@@ -1,15 +1,14 @@
-import { loadWasmModule, Renderer } from '@live-compositor/browser-render';
 import { LiveCompositor as CoreLiveCompositor } from '@live-compositor/core';
-import WasmInstance from './manager/wasmInstance';
-import type { RegisterOutput } from './output/registerOutput';
-import { intoRegisterOutput } from './output/registerOutput';
-import type { RegisterInput } from './input/registerInput';
-import { intoRegisterInput } from './input/registerInput';
-import type { RegisterImage } from './renderers';
 import type { ReactElement } from 'react';
 import type { Logger } from 'pino';
 import { pino } from 'pino';
-import { assert } from './utils';
+import { assert } from '../utils';
+import type { RegisterOutput } from './output';
+import { intoRegisterOutput } from './output';
+import type { RegisterInput } from './input';
+import { intoRegisterInput } from './input';
+import WasmInstance from '../wasmInstance';
+import type { RegisterImage } from './renderers';
 
 export type LiveCompositorOptions = {
   framerate?: Framerate;
@@ -34,9 +33,8 @@ export function setWasmBundleUrl(url: string) {
 export default class LiveCompositor {
   private coreCompositor?: CoreLiveCompositor;
   private instance?: WasmInstance;
-  private renderer?: Renderer;
   private options: LiveCompositorOptions;
-  private logger: Logger = pino({ level: 'warn' });
+  private logger: Logger = pino({ level: 'debug' });
 
   public constructor(options: LiveCompositorOptions) {
     this.options = options;
@@ -47,13 +45,11 @@ export default class LiveCompositor {
    * Outputs won't produce any results until `start()` is called.
    */
   public async init(): Promise<void> {
-    await ensureWasmModuleLoaded();
-    this.renderer = await Renderer.create({
-      streamFallbackTimeoutMs: this.options.streamFallbackTimeoutMs ?? 500,
-    });
+    assert(wasmBundleUrl, 'Location of WASM bundle is not defined, call setWasmBundleUrl() first.');
     this.instance = new WasmInstance({
-      renderer: this.renderer!,
       framerate: this.options.framerate ?? { num: 30, den: 1 },
+      wasmBundleUrl,
+      logger: this.logger.child({ element: 'wasmInstance' }),
     });
     this.coreCompositor = new CoreLiveCompositor(this.instance, this.logger);
 
@@ -94,9 +90,9 @@ export default class LiveCompositor {
     await this.coreCompositor.unregisterImage(imageId);
   }
 
-  public async registerFont(fontUrl: string): Promise<void> {
-    assert(this.renderer);
-    await this.renderer.registerFont(fontUrl);
+  public async registerFont(_fontUrl: string): Promise<void> {
+    //assert(this.renderer);
+    //await this.renderer.registerFont(fontUrl);
   }
 
   /**
@@ -114,14 +110,3 @@ export default class LiveCompositor {
     await this.instance?.terminate();
   }
 }
-
-const ensureWasmModuleLoaded = (() => {
-  let loadedState: Promise<void> | undefined = undefined;
-  return async () => {
-    assert(wasmBundleUrl, 'Location of WASM bundle is not defined, call setWasmBundleUrl() first.');
-    if (!loadedState) {
-      loadedState = loadWasmModule(wasmBundleUrl);
-    }
-    await loadedState;
-  };
-})();

ts/@live-compositor/web-wasm/src/compositor/input.ts

@@ -0,0 +1,22 @@
+import type { RegisterInput as CoreRegisterInput } from '@live-compositor/core';
+
+export type RegisterInput =
+  | ({ type: 'mp4' } & RegisterMP4Input)
+  | { type: 'camera' }
+  | { type: 'screen_capture' };
+
+export type RegisterMP4Input = {
+  url: string;
+};
+
+export function intoRegisterInput(input: RegisterInput): CoreRegisterInput {
+  if (input.type === 'mp4') {
+    return { type: 'mp4', url: input.url };
+  } else if (input.type === 'camera') {
+    return { type: 'camera' };
+  } else if (input.type === 'screen_capture') {
+    return { type: 'screen_capture' };
+  } else {
+    throw new Error(`Unknown input type ${(input as any).type}`);
+  }
+}

ts/@live-compositor/web-wasm/src/compositor/output.ts

@@ -0,0 +1,24 @@
+import type { Resolution } from '@live-compositor/browser-render';
+import type { RegisterOutput as CoreRegisterOutput } from '@live-compositor/core';
+
+export type RegisterOutput = {
+  type: 'canvas';
+  video: {
+    canvas: HTMLCanvasElement;
+    resolution: Resolution;
+  };
+};
+
+export function intoRegisterOutput(output: RegisterOutput): CoreRegisterOutput {
+  if (output.type === 'canvas') {
+    return {
+      type: 'canvas',
+      video: {
+        resolution: output.video.resolution,
+        canvas: output.video.canvas.transferControlToOffscreen(),
+      },
+    };
+  } else {
+    throw new Error(`Unknown output type ${(output as any).type}`);
+  }
+}

ts/@live-compositor/web-wasm/src/index.ts

@@ -1,4 +1,3 @@
-import WasmInstance from './manager/wasmInstance';
-import LiveCompositor, { setWasmBundleUrl } from './compositor';
+import LiveCompositor, { setWasmBundleUrl } from './compositor/compositor';
 
-export { WasmInstance, LiveCompositor, setWasmBundleUrl };
+export { LiveCompositor, setWasmBundleUrl };