fix: LEAP-1490: Fix AudioUltra working with relations and comments (#…

…6494) Co-authored-by: Gondragos <[email protected]> Co-authored-by: bmartel <[email protected]>
HumanSignal · Oct 21, 2024 · 6c2e4ea · 6c2e4ea
1 parent 47d1a2f
commit 6c2e4ea
Show file tree

Hide file tree

Showing 15 changed files with 560 additions and 30 deletions.
diff --git a/web/libs/editor/src/components/InteractiveOverlays/BoundingBox.js b/web/libs/editor/src/components/InteractiveOverlays/BoundingBox.js
@@ -1,3 +1,4 @@
+import { FF_DEV_2715, isFF } from "../../utils/feature-flags";
 import { wrapArray } from "../../utils/utilities";
 import { Geometry } from "./Geometry";
 
@@ -80,11 +81,20 @@ const _detect = (region) => {
   if (isResult) {
     return Geometry.getDOMBBox(region.from_name.elementRef?.current);
   }
-  switch (region.type) {
+
+  let type = region.type;
+  if (type === "audioregion") {
+    if (isFF(FF_DEV_2715)) {
+      type = "audioregion::ultra";
+    } else {
+      type = "audioregion::old";
+    }
+  }
+  switch (type) {
     case "textrange":
     case "richtextregion":
     case "textarearegion":
-    case "audioregion":
+    case "audioregion::old":
     case "paragraphs":
     case "timeseriesregion": {
       const regionBbox = Geometry.getDOMBBox(region.getRegionElement());
@@ -104,6 +114,22 @@ const _detect = (region) => {
 
       return regionBbox;
     }
+    case "audioregion::ultra": {
+      const bbox = region.bboxCoordsCanvas;
+      const stageEl = region.parent?.stageRef?.current;
+      const stageBbox = Geometry.getDOMBBox(stageEl, true);
+
+      return bbox
+        ? stageBbox
+          ? {
+              x: stageBbox.x + bbox.left,
+              y: stageBbox.y + bbox.top,
+              width: bbox.right - bbox.left,
+              height: bbox.bottom - bbox.top,
+            }
+          : bbox
+        : DEFAULT_BBOX;
+    }
     case "rectangleregion":
     case "ellipseregion":
     case "polygonregion":

diff --git a/web/libs/editor/src/components/InteractiveOverlays/NodesConnector.js b/web/libs/editor/src/components/InteractiveOverlays/NodesConnector.js
@@ -1,5 +1,5 @@
 import { debounce } from "../../utils/debounce";
-import { FF_PER_FIELD_COMMENTS, isFF } from "../../utils/feature-flags";
+import { FF_DEV_2715, FF_PER_FIELD_COMMENTS, isFF } from "../../utils/feature-flags";
 import { wrapArray } from "../../utils/utilities";
 import { Geometry } from "./Geometry";
 import { RelationShape } from "./RelationShape";
@@ -19,9 +19,17 @@ const parentImagePropsWatch = {
 const obtainWatcher = (node) => {
   switch (node.type) {
     case "richtextregion":
-    case "audioregion":
     case "paragraphs":
       return DOMWatcher;
+    case "audioregion": {
+      if (isFF(FF_DEV_2715)) {
+        return createPropertyWatcher(["bboxTriggers"]);
+      }
+      if (node.getRegionElement) {
+        return DOMWatcher;
+      }
+      return null;
+    }
     case "rectangleregion":
       return createPropertyWatcher(["x", "y", "width", "height", "hidden", parentImagePropsWatch]);
     case "ellipseregion":

diff --git a/web/libs/editor/src/lib/AudioUltra/Regions/Regions.ts b/web/libs/editor/src/lib/AudioUltra/Regions/Regions.ts
@@ -74,6 +74,7 @@ export class Regions {
     container.addEventListener("mousedown", this.handleMouseDown);
     container.addEventListener("mouseup", this.handleMouseUp);
     container.addEventListener("click", this.handleClick);
+    container.addEventListener("mouseleave", this.handleMouseLeave);
   }
 
   handleDraw = () => {
@@ -86,7 +87,7 @@ export class Regions {
     const currentTime = this.waveform.currentTime;
 
     this.regions.forEach((region) => {
-      region.highlighted = region.start <= currentTime && region.end >= currentTime;
+      region.active = region.start <= currentTime && region.end >= currentTime;
       region.render();
     });
   }
@@ -226,6 +227,7 @@ export class Regions {
     container.removeEventListener("mousedown", this.handleMouseDown);
     container.removeEventListener("mouseup", this.handleMouseUp);
     container.removeEventListener("click", this.handleClick);
+    container.removeEventListener("mouseleave", this.handleMouseLeave);
 
     this.regions.forEach((region) => region.destroy());
     this.regions = [];
@@ -399,6 +401,15 @@ export class Regions {
     }
   };
 
+  private handleMouseLeave = (e: MouseEvent) => {
+    if (this.hoveredRegions.size) {
+      this.hoveredRegions.forEach((region) => {
+        region.invoke("mouseLeave", [region, e]);
+      });
+      this.hoveredRegions.clear();
+    }
+  };
+
   private get cursorLockedByPlayhead() {
     return this.waveform.cursor.hasFocus() && this.waveform.cursor.isFocused("playhead");
   }

diff --git a/web/libs/editor/src/lib/AudioUltra/Regions/Segment.ts b/web/libs/editor/src/lib/AudioUltra/Regions/Segment.ts
@@ -48,6 +48,8 @@ export class Segment extends Events<SegmentEvents> {
   color: RgbaColorArray = rgba("#afafaf");
   selected = false;
   highlighted = false;
+  // active means that segment intersects with the cursor
+  active = false;
   updateable = true;
   locked = false;
   deleteable = true;
@@ -168,6 +170,21 @@ export class Segment extends Events<SegmentEvents> {
     return this.xStart + this.width;
   }
 
+  get yStart() {
+    const { timelinePlacement, timelineHeight } = this;
+    const timelineLayer = this.visualizer.getLayer("timeline");
+    const timelineTop = timelinePlacement === defaults.timelinePlacement;
+    const top = timelineLayer?.isVisible && timelineTop ? timelineHeight : 0;
+
+    return top;
+  }
+
+  get yEnd() {
+    const { height } = this.visualizer;
+    const { timelineHeight } = this;
+    return this.yStart + (height - timelineHeight);
+  }
+
   get width() {
     const { start, end } = this;
     const { width } = this.visualizer;
@@ -203,7 +220,7 @@ export class Segment extends Events<SegmentEvents> {
 
   private get inViewport() {
     const { xStart: startX, xEnd: endX } = this;
-    const width = this.visualizer.width * this.zoom;
+    const width = this.visualizer.width;
 
     // Both coordinates are less than or equal to 0
     if (startX <= 0 && endX <= 0) return false;
@@ -322,16 +339,14 @@ export class Segment extends Events<SegmentEvents> {
       return;
     }
 
-    const { color: _color, selected, highlighted, timelinePlacement, timelineHeight } = this;
+    const { color: _color, selected, highlighted, active } = this;
     const { height } = this.visualizer;
 
     const color = _color.clone();
-    const timelineLayer = this.visualizer.getLayer("timeline");
-    const timelineTop = timelinePlacement === defaults.timelinePlacement;
-    const top = timelineLayer?.isVisible && timelineTop ? timelineHeight : 0;
+    const top = this.yStart;
     const layer = this.controller.layerGroup;
 
-    if (selected || highlighted) {
+    if (selected || highlighted || active) {
       color.darken(0.4);
     }
 
@@ -359,7 +374,7 @@ export class Segment extends Events<SegmentEvents> {
   };
 
   handleHighlighted = (highlighted?: boolean) => {
-    if (!this.updateable || this.selected) return;
+    if (!this.updateable || (this.isDragging && this.selected)) return;
     this.highlighted = highlighted ?? !this.highlighted;
     this.invoke("update", [this]);
     this.waveform.invoke("regionUpdated", [this]);

diff --git a/web/libs/editor/src/lib/AudioUltra/Waveform.ts b/web/libs/editor/src/lib/AudioUltra/Waveform.ts
@@ -160,6 +160,14 @@ export interface WaveformOptions {
     denoize: boolean;
   };
 }
+
+export type WaveformFrameState = {
+  width: number;
+  height: number;
+  zoom: number;
+  scroll: number;
+};
+
 interface WaveformEventTypes extends RegionsGlobalEvents, RegionGlobalEvents {
   load: () => void;
   error: (error: Error) => void;
@@ -176,6 +184,7 @@ interface WaveformEventTypes extends RegionsGlobalEvents, RegionGlobalEvents {
   durationChanged: (duration: number) => void;
   scroll: (scroll: number) => void;
   layersUpdated: (layers: Map<string, Layer>) => void;
+  frameDrawn: (frameState: WaveformFrameState) => void;
 }
 
 export class Waveform extends Events<WaveformEventTypes> {
@@ -549,8 +558,19 @@ export class Waveform extends Events<WaveformEventTypes> {
   private initEvents() {
     this.cursor.on("mouseMove", this.handleCursorMove);
     this.visualizer.on("layersUpdated", () => this.invoke("layersUpdated", [this.getLayers()]));
+    this.visualizer.on("draw", () => this.handleDrawn());
   }
 
+  private handleDrawn = () => {
+    const frameState = {
+      width: this.visualizer.width,
+      height: this.visualizer.height,
+      zoom: this.zoom,
+      scroll: this.visualizer.getScrollLeftPx(),
+    };
+    this.invoke("frameDrawn", [frameState]);
+  };
+
   /**
    * Handle cursor move event
    */

diff --git a/web/libs/editor/src/lib/AudioUltra/react/index.ts b/web/libs/editor/src/lib/AudioUltra/react/index.ts
@@ -1,8 +1,8 @@
-import { type MutableRefObject, useEffect, useRef, useState } from "react";
+import { type MutableRefObject, useEffect, useMemo, useRef, useState } from "react";
 
 import { isTimeRelativelySimilar } from "../Common/Utils";
 import type { Layer } from "../Visual/Layer";
-import { Waveform, type WaveformOptions } from "../Waveform";
+import { Waveform, type WaveformFrameState, type WaveformOptions } from "../Waveform";
 
 export const useWaveform = (
   containter: MutableRefObject<HTMLElement | null | undefined>,
@@ -14,6 +14,7 @@ export const useWaveform = (
     onError?: (error: Error) => void;
     autoLoad?: boolean;
     showLabels?: boolean;
+    onFrameChanged?: (frame: { width: number; height: number; zoom: number; scroll: number }) => void;
   },
 ) => {
   const waveform = useRef<Waveform>();
@@ -29,6 +30,29 @@ export const useWaveform = (
   const [layers, setLayers] = useState<Layer[]>([]);
   const [layerVisibility, setLayerVisibility] = useState(new Map());
 
+  const onFrameChangedRef = useRef(options?.onFrameChanged);
+  onFrameChangedRef.current = options?.onFrameChanged;
+
+  const updateAfterRegionDraw = useMemo(() => {
+    let prevFrameState: WaveformFrameState | null = null;
+    let requestId = -1;
+    return (frameState: WaveformFrameState) => {
+      cancelAnimationFrame(requestId);
+      requestId = requestAnimationFrame(() => {
+        if (
+          !prevFrameState ||
+          frameState.width !== prevFrameState.width ||
+          frameState.height !== prevFrameState.height ||
+          frameState.zoom !== prevFrameState.zoom ||
+          frameState.scroll !== prevFrameState.scroll
+        ) {
+          onFrameChangedRef.current?.(frameState);
+          prevFrameState = frameState;
+        }
+      });
+    };
+  }, []);
+
   useEffect(() => {
     const wf = new Waveform({
       ...(options ?? {}),
@@ -64,6 +88,7 @@ export const useWaveform = (
       }
     });
     wf.on("zoom", setZoom);
+    wf.on("frameDrawn", updateAfterRegionDraw);
     wf.on("muted", setMuted);
     wf.on("durationChanged", setDuration);
     wf.on("volumeChanged", setVolume);

diff --git a/web/libs/editor/src/regions/AudioRegion/AudioUltraRegionModel.js b/web/libs/editor/src/regions/AudioRegion/AudioUltraRegionModel.js
@@ -2,6 +2,7 @@ import { types } from "mobx-state-tree";
 import { AudioModel } from "../../tags/object/AudioNext";
 import Utils from "../../utils";
 import Constants from "../../core/Constants";
+import { clamp } from "../../utils/utilities";
 
 export const AudioUltraRegionModel = types
   .model("AudioUltraRegionModel", {
@@ -16,8 +17,30 @@ export const AudioUltraRegionModel = types
   })
   .volatile(() => ({
     hideable: true,
+    _ws_region: null,
   }))
   .views((self) => ({
+    get bboxTriggers() {
+      return [self.start, self.end, self._ws_region, self.object?._ws, self.object?._wfFrame];
+    },
+    get bboxCoordsCanvas() {
+      if (!self.bboxTriggers) {
+        return null;
+      }
+
+      const { _ws_region } = self;
+      if (!_ws_region) return null;
+      if (!_ws_region.inViewport) return null;
+
+      const { xStart, xEnd, yStart, yEnd, visualizer } = _ws_region;
+      return {
+        left: clamp(xStart, 0, visualizer.width),
+        top: yStart,
+        right: clamp(xEnd, 0, visualizer.width),
+        bottom: yEnd,
+      };
+    },
+
     wsRegionOptions() {
       const reg = {
         id: self.id,
@@ -143,5 +166,14 @@ export const AudioUltraRegionModel = types
           self.updatePosition();
         }
       },
+
+      setWSRegion(wsRegion) {
+        self._ws_region = wsRegion;
+
+        if (wsRegion) {
+          wsRegion.on("mouseOver", self.onMouseOver);
+          wsRegion.on("mouseLeave", self.onMouseLeave);
+        }
+      },
     };
   });