Add visualization of the matching result to viewer

src/viewer/src/App.tsx

@@ -1,18 +1,21 @@
 // Created by O. Vysotska in 2023
 import { useState, useEffect } from "react";
 import "./App.css";
-import { ImageCostMatrix, ZoomBlockParams } from "./ImageCostMatrix";
-import ProtoLoader from "./ProtoLoader";
+import { ProtoLoader, MessageType } from "./ProtoLoader";
 import { CostMatrix, CostMatrixElement } from "./costMatrix";
-import InteractiveCostMatrix from "./InteractiveCostMatrix";
 import { ImagesLoader } from "./ImagesLoader";
+import {
+  MatchingResultElement,
+  readMatchingResultFromProto,
+} from "./matchingResult";
+import CostMatrixComponent from "./CostMatrixComponent";
 
 function App() {
-  const [image, setImage] = useState<ImageBitmap>();
   const [costMatrixProto, setCostMatrixProto] = useState(null);
   const [costMatrix, setCostMatrix] = useState<CostMatrix>();
-  const [zoomParams, setZoomParams] = useState<ZoomBlockParams>();
-  const [zoomedCostMatrix, setZoomedCostMatrix] = useState<CostMatrix>();
+  const [matchingResultProto, setMatchingResultProto] = useState(null);
+  const [matchingResult, setMatchingResult] =
+    useState<MatchingResultElement[]>();
   const [selectedCostMatrixElement, setSelectedCostMatrixElement] =
     useState<CostMatrixElement>();
 
@@ -24,25 +27,9 @@ function App() {
   }, [costMatrixProto]);
 
   useEffect(() => {
-    if (costMatrix != null) {
-      costMatrix.createImage().then((result) => {
-        setImage(result);
-      });
-    }
-  }, [costMatrix]);
-
-  useEffect(() => {
-    if (zoomParams == null || costMatrix == null) {
-      return;
-    }
-    setZoomedCostMatrix(
-      costMatrix.getSubMatrix(
-        zoomParams.topLeftX,
-        zoomParams.topLeftY,
-        zoomParams.windowHeightPx
-      )
-    );
-  }, [zoomParams, costMatrix]);
+    console.log("Matching result proto changed", matchingResultProto);
+    setMatchingResult(readMatchingResultFromProto(matchingResultProto));
+  }, [matchingResultProto]);
 
   return (
     <div className="App">
@@ -57,35 +44,21 @@ function App() {
       >
         <h1 style={{ textAlign: "center" }}>Cost Matrix Viewer</h1>
         <div className="costMatrix" style={{ backgroundColor: "ghostwhite" }}>
-          <ProtoLoader onLoad={setCostMatrixProto} />
-          <div
-            style={{
-              display: "flex",
-              flexDirection: "row",
-              flexWrap: "wrap",
-              justifyContent: "center",
-            }}
-          >
-            <div>
-              {image && (
-                <ImageCostMatrix
-                  image={image}
-                  setZoomParams={setZoomParams}
-                  width="100%"
-                  height="100%"
-                />
-              )}
-            </div>
-            <div>
-              {zoomParams && zoomedCostMatrix && (
-                <InteractiveCostMatrix
-                  costMatrix={zoomedCostMatrix}
-                  zoomBlock={zoomParams}
-                  setSelectedElement={setSelectedCostMatrixElement}
-                />
-              )}
-            </div>
-          </div>
+          <ProtoLoader
+            onLoad={setCostMatrixProto}
+            messageType={MessageType.CostMatrix}
+          />
+          <ProtoLoader
+            onLoad={setMatchingResultProto}
+            messageType={MessageType.MatchingResult}
+          />
+          {costMatrix && (
+            <CostMatrixComponent
+              costMatrix={costMatrix}
+              matchingResult={matchingResult}
+              setSelectedCostMatrixElement={setSelectedCostMatrixElement}
+            />
+          )}
         </div>
 
         <div

src/viewer/src/CostMatrixComponent.tsx

@@ -0,0 +1,128 @@
+// Created by O. Vysotska in 2023
+import { useState, useEffect } from "react";
+import { CostMatrix, CostMatrixElement } from "./costMatrix";
+import { ImageCostMatrix, ZoomBlockParams } from "./ImageCostMatrix";
+import { MatchingResultElement } from "./matchingResult";
+import InteractiveCostMatrix from "./InteractiveCostMatrix";
+
+function getMatchingResultInZoomBlock(
+  results: MatchingResultElement[],
+  zoomParams: ZoomBlockParams
+) {
+  return results.filter(
+    (result) =>
+      result.queryId >= zoomParams.topLeftY &&
+      result.queryId < zoomParams.topLeftY + zoomParams.windowHeightPx &&
+      result.refId >= zoomParams.topLeftX &&
+      result.refId < zoomParams.topLeftX + zoomParams.windowWidthPx
+  );
+}
+
+type CostMatrixProps = {
+  costMatrix: CostMatrix;
+  matchingResult?: MatchingResultElement[];
+  setSelectedCostMatrixElement: (element: CostMatrixElement) => void;
+};
+
+function CostMatrixComponent(props: CostMatrixProps): React.ReactElement {
+  const [image, setImage] = useState<ImageBitmap>();
+  const [matchingResult, setMatchingResult] =
+    useState<MatchingResultElement[]>();
+  const [zoomParams, setZoomParams] = useState<ZoomBlockParams>();
+  const [zoomedCostMatrix, setZoomedCostMatrix] = useState<CostMatrix>();
+  const [matchingResultVisible, setMatchingResultVisible] =
+    useState<boolean>(false);
+  const [selectedElement, setSelectedElement] = useState<CostMatrixElement>();
+
+  const { setSelectedCostMatrixElement } = props;
+  useEffect(() => {
+    if (selectedElement == null) {
+      return;
+    }
+    setSelectedCostMatrixElement(selectedElement);
+  }, [selectedElement, setSelectedCostMatrixElement]);
+
+  useEffect(() => {
+    if (props.costMatrix != null) {
+      props.costMatrix.createImage().then((result) => {
+        setImage(result);
+      });
+    }
+  }, [props.costMatrix]);
+
+  useEffect(() => {
+    setMatchingResult(props.matchingResult);
+  }, [props.matchingResult]);
+
+  useEffect(() => {
+    if (zoomParams == null || props.costMatrix == null) {
+      return;
+    }
+    setZoomedCostMatrix(
+      props.costMatrix.getSubMatrix(
+        zoomParams.topLeftX,
+        zoomParams.topLeftY,
+        zoomParams.windowHeightPx
+      )
+    );
+  }, [zoomParams, props.costMatrix]);
+
+  function showMatchingResult(event: any) {
+    if (event.target.checked) {
+      setMatchingResultVisible(true);
+    } else {
+      setMatchingResultVisible(false);
+    }
+  }
+
+  return (
+    <div
+      style={{
+        display: "flex",
+        flexDirection: "column",
+      }}
+    >
+      <div>
+        <input id="checkbox" type="checkbox" onChange={showMatchingResult} />
+        <label htmlFor="checkbox">Show matching result</label>
+      </div>
+      <div
+        style={{
+          display: "flex",
+          flexDirection: "row",
+          flexWrap: "wrap",
+          justifyContent: "center",
+        }}
+      >
+        <div>
+          {image && (
+            <ImageCostMatrix
+              image={image}
+              setZoomParams={setZoomParams}
+              width="100%"
+              height="100%"
+              matches={matchingResult}
+              showMatches={matchingResultVisible}
+            />
+          )}
+        </div>
+        <div>
+          {zoomParams && zoomedCostMatrix && (
+            <InteractiveCostMatrix
+              costMatrix={zoomedCostMatrix}
+              zoomBlock={zoomParams}
+              setSelectedElement={setSelectedElement}
+              showMatches={matchingResultVisible}
+              matches={
+                matchingResult &&
+                getMatchingResultInZoomBlock(matchingResult, zoomParams)
+              }
+            />
+          )}
+        </div>
+      </div>
+    </div>
+  );
+}
+
+export default CostMatrixComponent;

src/viewer/src/ImageCostMatrix.tsx

@@ -1,6 +1,6 @@
 import React, { forwardRef } from "react";
 import { useState, useRef, useEffect } from "react";
-import { NumberLiteralType } from "typescript";
+import { MatchingResultElement } from "./matchingResult";
 
 const kZoomWindowPx = 30;
 
@@ -73,11 +73,43 @@ function drawZoomBoxInContext(
   }
 }
 
+function overlayMatchingResult(
+  imageData: Uint8ClampedArray,
+  imageWidth: number,
+  matchingResult?: MatchingResultElement[]
+) {
+  if (matchingResult == null) {
+    return;
+  }
+  matchingResult.forEach((element) => {
+    const pixelIdx = (element.queryId * imageWidth + element.refId) * 4;
+    if (pixelIdx < 0 || pixelIdx + 4 >= imageData.length) {
+      console.log("Error: Estimated image element is outside bounds");
+      return;
+    }
+    if (element.real) {
+      // Assign red pixel
+      imageData[pixelIdx] = 255; // red
+      imageData[pixelIdx + 1] = 0; // green
+      imageData[pixelIdx + 2] = 0; // blue
+      imageData[pixelIdx + 3] = 255; // alpha
+    } else {
+      // Assign blue pixel
+      imageData[pixelIdx] = 0; // red
+      imageData[pixelIdx + 1] = 0; // green
+      imageData[pixelIdx + 2] = 255; // blue
+      imageData[pixelIdx + 3] = 255; // alpha
+    }
+  });
+}
+
 type ImageCostMatrixProps = {
   image: ImageBitmap;
   setZoomParams: (zoomParams: ZoomBlockParams) => void;
   width: string;
   height: string;
+  matches?: MatchingResultElement[];
+  showMatches?: boolean;
 };
 
 type ImageSize = {
@@ -89,24 +121,35 @@ type HoverCoords = {
   screenY: number;
 };
 
+// TODO(olga): Maybe separate canvas for image and matching path is a better solution. Then need to make sure it gets propagated to the zoomTooltip
+
 function ImageCostMatrix(props: ImageCostMatrixProps): React.ReactElement {
   const imageCanvasRef = useRef<HTMLCanvasElement>(null);
   const pixelZoomRef = useRef<HTMLCanvasElement>(null);
+  const [image, setImage] = useState<ImageBitmap>();
   const [imageSize, setImageSize] = useState<ImageSize>({
     width: 0,
     height: 0,
   });
+
   const [hoverCoords, setHoverCoords] = useState<HoverCoords>();
 
+  useEffect(() => {
+    if (props.image == null) {
+      console.log("Image is empty");
+      return;
+    }
+    setImage(props.image);
+  }, [props.image]);
+
   // Assumptions: Canvas size is adapted to image size.
   // Zoom image is always rescaled to 300px x 300px
   useEffect(() => {
     // Default for Firefox area:
     // canvasRef.current.width = 300;
     // canvasRef.current.height = 150;
 
-    if (props.image == null) {
-      console.error("Image is empty.");
+    if (image == null) {
       return;
     }
 
@@ -115,21 +158,20 @@ function ImageCostMatrix(props: ImageCostMatrixProps): React.ReactElement {
       console.error("Canvas is not set");
       return;
     }
-
     const imageContext = imageCanvas.getContext("2d");
     if (imageContext == null) {
       console.error("Image Context is null");
       return;
     }
 
-    console.log("Image size", props.image.width, props.image.height);
-    setImageSize({ width: props.image.width, height: props.image.height });
-    imageCanvas.width = props.image.width;
-    imageCanvas.height = props.image.height;
+    console.log("Image size", image.width, image.height);
+    setImageSize({ width: image.width, height: image.height });
+    imageCanvas.width = image.width;
+    imageCanvas.height = image.height;
 
     // Draw image of exact size.
-    imageContext.drawImage(props.image, 0, 0);
-  }, [props.image]);
+    imageContext.drawImage(image, 0, 0);
+  }, [image]);
 
   function onHoverOverImage(event: any) {
     const xCenterPx = event.nativeEvent.layerX;
@@ -173,6 +215,42 @@ function ImageCostMatrix(props: ImageCostMatrixProps): React.ReactElement {
     });
   }
 
+  useEffect(() => {
+    const imageCanvas = imageCanvasRef.current;
+    if (imageCanvas == null) {
+      console.error("Canvas is not set");
+      return;
+    }
+    const imageContext = imageCanvas.getContext("2d");
+    if (imageContext == null) {
+      console.error("Image Context is null");
+      return;
+    }
+
+    if (props.showMatches) {
+      if (props.matches) {
+        // show matching result
+        const imageData = imageContext.getImageData(
+          0,
+          0,
+          imageCanvas.width,
+          imageCanvas.height
+        );
+        overlayMatchingResult(imageData.data, imageCanvas.width, props.matches);
+        imageContext.putImageData(imageData, 0, 0);
+      } else {
+        console.error("Can't display matches. Matches are not set");
+        return;
+      }
+    } else {
+      console.log("Hide matches");
+      // hide matches
+      if (image) {
+        imageContext.drawImage(image, 0, 0);
+      }
+    }
+  }, [props.matches, props.showMatches, image]);
+
   return (
     <div>
       <div