Trouble with answering the call

[188931]

Hello,
Here is my code:
A)javascript:

import * as sdk from 'matrix-js-sdk';

const homeserverUrl = "https://matrix.org";
const username = "@username:matrix.org";
const password = "password";
const roomId = "!roomID:matrix.org";

const client = sdk.createClient({
    baseUrl: homeserverUrl,
    encryption: true
});

const originalGetUserMedia = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices);
navigator.mediaDevices.getUserMedia = async (constraints) => {
    // Ensure at least one of audio or video is requested
    if (!constraints.audio && !constraints.video) {
        console.warn("Forcing at least one media type to true as audio and video are both false.");
        constraints.audio = true;
        constraints.video = true; // or set constraints.video = true depending on your requirements
    }
    console.log("getUserMedia called with adjusted constraints:", constraints);

    try {
        const localStream = await originalGetUserMedia(constraints);
        console.log("getUserMedia successful:", localStream);
        console.log("Local Stream Tracks:", localStream.getTracks());
        return localStream;
    } catch (error) {
        console.error("getUserMedia failed:", error);
        throw error; // Re-throw to ensure the original error handling works as expected
    }
};

async function login() {
    try {
        const loginResponse = await client.login("m.login.password", {
            user: username,
            password: password
        });
        console.log("Login response:", loginResponse);
        console.log("Assigned Device ID:", loginResponse.device_id);

        client.deviceId = loginResponse.device_id;

        await client.startClient();
        console.log("Sync complete. Device ID:", client.getDeviceId());

    } catch (error) {
        console.error("Login failed:", error)
    }
}

async function joinRoom() {
    try {
        console.log(`Joining room ${roomId}...`);
        await client.joinRoom(roomId);
        console.log(`Successfully joined room: ${roomId}`);
    } catch (error) {
        console.error("Failed to join room:", error);
    }
}

async function checkMediaPermissions() {
    try {
        const audioStream = await navigator.mediaDevices.getUserMedia({ audio: true });
        console.log("Audio devices are accessible.");
        const videoStream = await navigator.mediaDevices.getUserMedia({ video: true });
        console.log("Video devices are accessible.");
    } catch (e) {
        console.error("Media devices access error:", e);
    }
}

async function startVideoCall() {
    try {
        // Ensure media permissions
        await checkMediaPermissions();

        const configuration = {
            iceServers: [{ urls: "stun:stun.l.google.com:19302" }]
        };
        
        const peerConnection = new RTCPeerConnection(configuration);

        let iceCandidates = []; // Store ICE candidates in a batch array

        const constraints = { audio: true, video: true };
        const localStream = await navigator.mediaDevices.getUserMedia(constraints);
        console.log("Local video id: ",localStream.id)
        localStream.getTracks().forEach(track => peerConnection.addTrack(track, localStream));

        const localVideoElement = document.getElementById("localVideo");
        if (localVideoElement) {
            localVideoElement.srcObject = localStream;
           localVideoElement.muted = true;
            await localVideoElement.play();
        }

        const call = client.createCall(roomId);
        call.placeVideoCall(localStream);

        call.on("state", (state) => {
            console.log("Call state changed:", state);
        });


        // Await setting local description before handling the answer
        const offer = await peerConnection.createOffer();
        await peerConnection.setLocalDescription(offer);
        console.log("Local SDP offer set successfully");

        let assignedStreamId = null;


        peerConnection.ontrack = (event) => {
            const remoteVideoElement = document.getElementById("remoteVideo");
            const stream = event.streams[0];

            if (!stream) {
                console.warn("No stream received in ontrack event.");
                return;
            }

            console.log("Received track with stream ID:", stream.id);

            // Check if this stream is already assigned by comparing stream IDs
            if (assignedStreamId !== stream.id) {
                remoteVideoElement.srcObject = stream;
                remoteVideoElement.muted = false;
                setTimeout(() => {
                    remoteVideoElement.play().catch((error) => {
                        console.error("Error auto-playing remote video:", error);
                    });
                }, 500);
                assignedStreamId = stream.id;
                console.log("Remote stream assigned based on SDP answer.");
            } else {
                console.log("Duplicate remote stream detected, skipping re-assignment.");
            }

            // Log track information
            stream.getTracks().forEach(track => {
                console.log(`Track kind: ${track.kind}, enabled: ${track.enabled}, id: ${track.id}`);
            });
        };
        
        // After setting up ontrack, proceed to process the SDP answer
        client.on("event", async (event) => {
            if (event.getType() === "m.call.answer") {
                const answer = event.getContent().answer;
                console.log("Received SDP Answer:", answer);
        
                try {
                    // Set the remote description from Client Two's SDP answer
                    await peerConnection.setRemoteDescription(new RTCSessionDescription(answer));
                    console.log("Remote SDP answer set successfully.");
        
                    // Analyze the SDP answer for media types
                    const hasAudio = answer.sdp.includes("m=audio");
                    const hasVideo = answer.sdp.includes("m=video");
        
                    console.log(`Answer SDP indicates presence of - Audio: ${hasAudio}, Video: ${hasVideo}`);
                } catch (error) {
                    console.error("Failed to set remote SDP:", error);
                }
            }
            if (event.getType() === "m.call.candidate") {
                const candidate = event.getContent().candidate;
                try {
                    await peerConnection.addIceCandidate(new RTCIceCandidate(candidate));
                    console.log("Added received ICE candidate:", candidate);
                } catch (error) {
                    console.error("Failed to add received ICE candidate:", error);
                }
            }
        });
        peerConnection.onicecandidate = (event) => {
            if (event.candidate) {
                console.log("ICE candidate:", event.candidate);

                // Store the ICE candidate in the batch (for now)
                iceCandidates.push({
                    candidate: event.candidate.candidate,
                    sdpMid: event.candidate.sdpMid,
                    sdpMLineIndex: event.candidate.sdpMLineIndex
                });

                // Check the state of the call
                if (call.state === "connecting") {
                    console.log("Call is in connecting state. Sending ICE candidate immediately.");
                    // Send the ICE candidates when the call is in the connecting state
                    sendIceCandidates();
                }
            }
        };

        // Function to send ICE candidates to the remote peer
        async function sendIceCandidates() {
            if (iceCandidates.length > 0) {
                console.log("Sending ICE candidates:", iceCandidates);
                await client.sendEvent(roomId, "m.call.candidates", {
                    call_id: call.callId,
                    candidates: iceCandidates.map(candidate => ({
                        candidate: candidate.candidate,
                        sdpMid: candidate.sdpMid,
                        sdpMLineIndex: candidate.sdpMLineIndex
                    }))
                });
                // Clear the candidates buffer after sending
                iceCandidates = [];
            }
        }

        console.log("Video call initiated.");
    } catch (error) {
        console.error("Error in startVideoCall:", error);
    }
}



let callInvites = [];  // Store all incoming invites

// Function to listen for incoming call invites
function listeningForIncomingCalls() {
    console.log("Listening for incoming calls...");

    client.on("event", (event) => {
        if (event.getType() === "m.call.invite") {
            console.log("Manual call invite received:", event);

            const callId = event.getContent().call_id;
            const call = client.createCall(roomId);
            call.callId = callId; // Attach the call ID to the call object

            // Store the invite in the array, ensuring the array only contains the most recent invite
            callInvites.push(event);  // Add the invite to the list
            console.log(`Stored invite with callId: ${callId}.`);
        }
    });
    setInterval(processLatestInvite, 10000);
}

// Function to process the latest invite from the stored list
async function processLatestInvite() {
    if (callInvites.length === 0) {
        console.log("No pending invites.");
        return;
    }

    const latestInvite = callInvites[callInvites.length - 1];
    const callId = latestInvite.getContent().call_id;
    console.log("Processing the latest invite with callId:", callId);

    const offer = latestInvite.getContent().offer.sdp;
    const hasAudio = offer && offer.includes("m=audio");
    const hasVideo = offer && offer.includes("m=video");

    const peerConnection = createPeerConnection(callId);

    try {
        await peerConnection.setRemoteDescription(new RTCSessionDescription({
            type: "offer",
            sdp: offer
        }));
        console.log("Remote SDP offer set successfully.");

        const constraints = { audio: hasAudio, video: hasVideo };
        const localStream = await navigator.mediaDevices.getUserMedia(constraints);
        //localStream.getTracks().forEach(track => peerConnection.addTrack(track, localStream));
        peerConnection.addStream(localStream);
        console.log("Added local media tracks to peer connection.");
        console.log("Stream", localStream.id)

        const localVideoElement = document.getElementById("localVideo");
        if (localVideoElement) {
            localVideoElement.srcObject = localStream;
            await localVideoElement.play();
        }

        const answer = await peerConnection.createAnswer();
        //let sdpWithMetadata = answer.sdp;

        //const ssrc = Math.floor(Math.random() * 1000000); // Generate a unique SSRC for the stream
        //const cname = "clientTwoUnifiedStream";
        //sdpWithMetadata += `a=ssrc:${ssrc} cname:${cname}\r\n`;
        //console.log(`Added SSRC metadata for unified stream - SSRC: ${ssrc}, cname: ${cname}`);

        // Update the SDP answer with the metadata-included string
        //answer.sdp = sdpWithMetadata;
        await peerConnection.setLocalDescription(answer);

        console.log("SDP answer:", answer)

        client.sendEvent(roomId, "m.call.answer", {
            call_id: callId,
            answer: {
                sdp: answer.sdp,
                type: answer.type,
            }
        });
        console.log("SDP answer sent to Client One.");

    } catch (error) {
        console.error("Failed to set up peer connection:", error);
    }

    callInvites = [];
}
function createPeerConnection(call) {
    const configuration = {
        iceServers: [
            { urls: "stun:stun.l.google.com:19302" },
            { urls: "stun:stun1.l.google.com:19302" },
            { urls: "stun:stun2.l.google.com:19302" },
            { urls: "stun:stun3.l.google.com:19302" }
            // Add TURN servers here if available
        ]
    };
    const peerConnection = new RTCPeerConnection(configuration);
    
    peerConnection.onicecandidate = (event) => {
        if (event.candidate) {
            console.log("ICE candidate:", event.candidate);
            // Send the candidate to the remote peer (Client One)
            client.sendEvent(roomId, "m.call.candidates", {
                call_id: call.callId,
                candidates: [
                    {
                        candidate: event.candidate.candidate,
                        sdpMid: event.candidate.sdpMid,
                        sdpMLineIndex: event.candidate.sdpMLineIndex
                    }
                ]
            });
        }
    };
    peerConnection.onconnectionstatechange = () => {
        console.log("Connection state change:", peerConnection.connectionState);
    };
    // Log ICE connection state changes
    peerConnection.oniceconnectionstatechange = () => {
        console.log("ICE connection state change:", peerConnection.iceConnectionState);
        if (peerConnection.iceConnectionState === "failed") {
            console.warn("ICE connection failed. Restarting ICE...");
            peerConnection.restartIce();
        }
    };


    peerConnection.ontrack = (event) => {
        const remoteVideo = document.getElementById("remoteVideo");
        const stream = event.streams[0];
    
        if (!stream.active) {
            console.warn("Stream is inactive");
        } else {
            console.log("Stream is active:", stream);
        }
    
        const videoTracks = stream.getVideoTracks();
        if (videoTracks.length > 0 && !videoTracks[0].enabled) {
            console.warn("Video track is disabled");
        } else {
            console.log("Video track is active:", videoTracks[0]);
        }

        const audioTracks = stream.getAudioTracks();
        if (audioTracks.length > 0 && !audioTracks[0].enabled) {
            console.warn("Audio track is disabled");
        } else {
            console.log("Audio track is active:", videoTracks[0]);
        }
    
        // Only set the video source if not already set
        if (remoteVideo && remoteVideo.srcObject !== stream) {
            remoteVideo.srcObject = stream;
            remoteVideo.play().catch(error => {
                console.error("Error playing remote video:", error);
            });
            console.log("Remote stream set to video element:", stream);
        } else {
            console.log("Remote video is already playing this stream, skipping re-assigning.");
        }
    };

    peerConnection.onnegotiationneeded = async () => {
        console.log("Negotiation needed...");
        try {
            const offer = await peerConnection.createOffer();
            await peerConnection.setLocalDescription(offer);
            console.log("Local SDP offer set for negotiation.");

            // Send the SDP offer to the remote peer
            client.sendEvent(roomId, "m.call.invite", {
                call_id: call.callId,
                offer: { sdp: offer.sdp, type: offer.type }
            });
            console.log("SDP offer sent to remote peer.");
        } catch (error) {
            console.error("Negotiation error:", error);
        }
    };

    return peerConnection;
}

(async function () {    
    await login();
    await joinRoom();
    startVideoCall();
    //startVoiceCall();
    //listeningForIncomingCalls();
})();

b)html:

<!DOCTYPE html>
<html lang="en">

<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Matrix Video Call</title>
</head>

<body>
    <h1>Matrix Video Call</h1>
    <video id="localVideo" autoplay muted style="width: 100%; height: auto;"></video>
    <video id="remoteVideo" autoplay style="width: 100%; height: auto;"></video>
    <script type="module" src="./src/client-client_Voip.js"></script>
</body>

</html>

I have two clients in protocol matrix and both of them are in the same room. One client is in my PC with build in microphone and camera, second one is in virtualBox with camera nad microphone from droidCAM. First client is starting the call, second is listening for call and answering it. I have an issue with answering this call. I tried two method:
a)call.anwer():
When i tried this method i got this error:
Uncaught (in promise) TypeError: Cannot read properties of undefined (reading 'addTrack')
It was weird bacause the audio and media are both true so i did not know why it could not attach the tracks from my localStream. After that i thought to manually answer the code which you see in the code.
b)Manually answering th call:
You see that i manually sending the messages 'm.call.answer' but i got this warning: onAnswerReceived() did not get any SDPStreamMetadata! Can not send/receive multiple streams. I do not know what is the problem. IS it the SDP answer? Is it the media?

Please someone help me because i am tring to do this for 3 weeks and i do not now what is wrong.

[dbkr]

We're unable to offer code-level support here, this is for identified defects with the js-sdk. You could try https://matrix.to/#/#matrix-dev:matrix.org