Skip to content

Commit

Permalink
Merged PR 15: Fix mouse input latency.
Browse files Browse the repository at this point in the history
  • Loading branch information
Jared Goodwin authored and Jared Goodwin committed Aug 28, 2024
1 parent 70e8328 commit 007238a
Show file tree
Hide file tree
Showing 22 changed files with 160 additions and 263 deletions.
2 changes: 1 addition & 1 deletion ControlR.Agent/ControlR.Agent.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
<ItemGroup>
<PackageReference Include="Bitbound.SimpleIpc" Version="2.0.0" />
<PackageReference Include="Bitbound.SimpleMessenger" Version="2.2.1" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.7" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.8" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="8.0.0" />
Expand Down
10 changes: 5 additions & 5 deletions ControlR.Server/ControlR.Server.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@
<ItemGroup>
<PackageReference Include="Azure.Monitor.OpenTelemetry.AspNetCore" Version="1.2.0" />
<PackageReference Include="Bitbound.WebSocketBridge.Common" Version="1.0.4" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.7" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="8.0.7" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="8.0.7" />
<PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="8.0.7" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.8" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="8.0.8" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="8.0.8" />
<PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="8.0.8" />
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="8.0.0" />
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
<PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.9.0" />
Expand All @@ -25,7 +25,7 @@
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.9.0" />
<PackageReference Include="Serilog.AspNetCore" Version="8.0.2" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.7.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.7.3" />
</ItemGroup>

<ItemGroup>
Expand Down
1 change: 0 additions & 1 deletion ControlR.Streamer/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,6 @@
services.AddSingleton<IDelayer, Delayer>();
services.AddScreenCapturer();
services.AddTransient<IHubConnectionBuilder, HubConnectionBuilder>();
services.AddTransient<IStreamingClient, StreamingClient>();
services.AddHostedService<SystemEventHandler>();
services.AddHostedService<HostLifetimeEventResponder>();
services.AddHostedService<InputDesktopReporter>();
Expand Down
74 changes: 24 additions & 50 deletions ControlR.Streamer/Services/DisplayManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ internal class DisplayManager : IDisplayManager
private readonly IMessenger _messenger;
private readonly Stopwatch _metricsBroadcastTimer = Stopwatch.StartNew();
private readonly IScreenCapturer _screenCapturer;
private readonly SemaphoreSlim _selectedDisplayLock = new(1, 1);
private readonly ConcurrentQueue<SentFrame> _sentRegions = new();
private readonly ISystemTime _systemTime;
private readonly IWin32Interop _win32Interop;
Expand Down Expand Up @@ -91,47 +90,32 @@ public DisplayManager(
_displays.FirstOrDefault();
}

public async Task ChangeDisplays(string displayId)
public Task ChangeDisplays(string displayId)
{
if (_displays.FirstOrDefault(x => x.DeviceName == displayId) is not { } newDisplay)
{
_logger.LogWarning("Could not find display with ID {DisplayId} when changing displays.", displayId);
return;
return Task.CompletedTask;
}

await _selectedDisplayLock.WaitAsync();
try
{
_lastCpuBitmap?.Dispose();
_lastCpuBitmap = null;
_selectedDisplay = newDisplay;
}
finally
{
_selectedDisplayLock.Release();
}
_lastCpuBitmap?.Dispose();
_lastCpuBitmap = null;
_selectedDisplay = newDisplay;

return Task.CompletedTask;
}

public async Task<Point> ConvertPercentageLocationToAbsolute(double percentX, double percentY)
public Task<Point> ConvertPercentageLocationToAbsolute(double percentX, double percentY)
{
await _selectedDisplayLock.WaitAsync();
try
if (_selectedDisplay?.MonitorArea is not { } bounds)
{
if (_selectedDisplay is null)
{
return Point.Empty;
}

var bounds = _selectedDisplay.MonitorArea;
var absoluteX = bounds.Width * percentX + bounds.Left;
var absoluteY = bounds.Height * percentY + bounds.Top;
return new Point((int)absoluteX, (int)absoluteY);
}
finally
{
_selectedDisplayLock.Release();
return Point.Empty.AsTaskResult();
}

var absoluteX = bounds.Width * percentX + bounds.Left;
var absoluteY = bounds.Height * percentY + bounds.Top;
return new Point((int)absoluteX, (int)absoluteY).AsTaskResult();

}

public async IAsyncEnumerable<ScreenRegionDto> GetChangedRegions()
Expand Down Expand Up @@ -165,23 +149,15 @@ public IEnumerable<DisplayDto> GetDisplays()
});
}

public async void ResetDisplays()
public void ResetDisplays()
{
_displays = _screenCapturer.GetDisplays().ToArray();
await _selectedDisplayLock.WaitAsync();
try
{
_selectedDisplay =
_displays.FirstOrDefault(x => x.IsPrimary) ??
_displays.FirstOrDefault();
_lastCpuBitmap?.Dispose();
_lastCpuBitmap = null;
_forceKeyFrame = true;
}
finally
{
_selectedDisplayLock.Release();
}
_selectedDisplay =
_displays.FirstOrDefault(x => x.IsPrimary) ??
_displays.FirstOrDefault();
_lastCpuBitmap?.Dispose();
_lastCpuBitmap = null;
_forceKeyFrame = true;
}

public Task StartCapturingChanges()
Expand Down Expand Up @@ -295,7 +271,6 @@ private async Task EncodeScreenCaptures(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
await _selectedDisplayLock.WaitAsync(stoppingToken);
try
{
await _frameRequestedSignal.Wait(stoppingToken);
Expand All @@ -311,7 +286,7 @@ private async Task EncodeScreenCaptures(CancellationToken stoppingToken)

_win32Interop.SwitchToInputDesktop();

using var captureResult = _screenCapturer.Capture(_selectedDisplay);
using var captureResult = _screenCapturer.Capture(selectedDisplay);

if (captureResult.HadNoChanges)
{
Expand Down Expand Up @@ -339,8 +314,8 @@ private async Task EncodeScreenCaptures(CancellationToken stoppingToken)
_needsKeyFrame = false;
_lastCpuBitmap?.Dispose();
_lastCpuBitmap = null;
_lastDisplayId = _selectedDisplay?.DeviceName;
_lastMonitorArea = _selectedDisplay?.MonitorArea;
_lastDisplayId = selectedDisplay.DeviceName;
_lastMonitorArea = selectedDisplay.MonitorArea;
continue;
}

Expand Down Expand Up @@ -374,7 +349,6 @@ private async Task EncodeScreenCaptures(CancellationToken stoppingToken)
{
_frameRequestedSignal.Set();
}
_selectedDisplayLock.Release();
}
}
}
Expand Down
8 changes: 2 additions & 6 deletions ControlR.Streamer/Services/DtoHandler.cs
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
using Bitbound.SimpleMessenger;
using ControlR.Libraries.DevicesCommon.Extensions;
using ControlR.Libraries.DevicesCommon.Messages;
using ControlR.Libraries.Shared.Dtos.SidecarDtos;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Options;
using System.Text.Json;
using Windows.ApplicationModel.DataTransfer;

namespace ControlR.Streamer.Services;
internal class DtoHandler(
Expand All @@ -18,7 +15,6 @@ internal class DtoHandler(
IOptions<StartupOptions> _startupOptions,
ILogger<DtoHandler> _logger) : IHostedService
{
private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNameCaseInsensitive = true };
public Task StartAsync(CancellationToken cancellationToken)
{
_messenger.Register<DtoReceivedMessage<SignedPayloadDto>>(this, HandleSignedDtoReceivedMessage);
Expand All @@ -40,13 +36,13 @@ private async Task HandleSignedDtoReceivedMessage(object subscriber, DtoReceived

if (!_keyProvider.Verify(wrapper))
{
_logger.LogCritical("Key verification failed for public key: {key}", wrapper.PublicKeyBase64);
_logger.LogCritical("Key verification failed for public key: {PublicKey}", wrapper.PublicKeyBase64);
return;
}

if (_startupOptions.Value.AuthorizedKey != wrapper.PublicKeyBase64)
{
_logger.LogCritical("Public key does not exist in authorized keys: {key}", wrapper.PublicKeyBase64);
_logger.LogCritical("Public key does not exist in authorized keys: {PublicKey}", wrapper.PublicKeyBase64);
return;
}

Expand Down
83 changes: 22 additions & 61 deletions ControlR.Streamer/Services/InputSimulatorWindows.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,50 +14,10 @@ public interface IInputSimulator
void TypeText(string text);
}

internal class InputSimulatorWindows : IInputSimulator
internal class InputSimulatorWindows(
IWin32Interop _win32Interop,
ILogger<InputSimulatorWindows> _logger) : IInputSimulator
{
private readonly ConcurrentQueue<Action> _actionQueue = new();
private readonly AutoResetEvent _queueSignal = new(false);
private readonly IWin32Interop _win32Interop;
private readonly IHostApplicationLifetime _appLifetime;
private readonly ILogger<InputSimulatorWindows> _logger;
private readonly Thread _processorThread;

public InputSimulatorWindows(
IWin32Interop win32Interop,
IHostApplicationLifetime appLifetime,
ILogger<InputSimulatorWindows> logger)
{
_win32Interop = win32Interop;
_appLifetime = appLifetime;
_logger = logger;
_processorThread = new Thread(() =>
{
_logger.LogInformation("Input simulator processor thread started.");
ProcessActions();
});
_processorThread.Start();
}

private void ProcessActions()
{
while (!_appLifetime.ApplicationStopping.IsCancellationRequested)
{
_queueSignal.WaitOne();
while (_actionQueue.TryDequeue(out var action))
{
try
{
action.Invoke();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error processing input simulator action.");
}
}
}
}

public void InvokeKeyEvent(string key, bool isPressed)
{
if (string.IsNullOrEmpty(key))
Expand All @@ -66,65 +26,66 @@ public void InvokeKeyEvent(string key, bool isPressed)
return;
}

_actionQueue.Enqueue(() =>
TryOnInputDesktop(() =>
{
_win32Interop.SwitchToInputDesktop();
var result = _win32Interop.InvokeKeyEvent(key, isPressed);
if (!result.IsSuccess)
{
_logger.LogWarning("Failed to invoke key event. Key: {Key}, IsPressed: {IsPressed}, Reason: {Reason}", key, isPressed, result.Reason);
}
});
_queueSignal.Set();
}

public void InvokeMouseButtonEvent(int x, int y, int button, bool isPressed)
{
_actionQueue.Enqueue(() =>
TryOnInputDesktop(() =>
{
_win32Interop.SwitchToInputDesktop();
_win32Interop.InvokeMouseButtonEvent(x, y, button, isPressed);
});
_queueSignal.Set();
}

public void MovePointer(int x, int y, MovePointerType moveType)
{
_actionQueue.Enqueue(() =>
TryOnInputDesktop(() =>
{
_win32Interop.SwitchToInputDesktop();
_win32Interop.MovePointer(x, y, moveType);
});
_queueSignal.Set();
}

public void ResetKeyboardState()
{
_actionQueue.Enqueue(() =>
TryOnInputDesktop(() =>
{
_win32Interop.SwitchToInputDesktop();
_win32Interop.ResetKeyboardState();
});
_queueSignal.Set();
}

public void ScrollWheel(int x, int y, int scrollY, int scrollX)
{
_actionQueue.Enqueue(() =>
TryOnInputDesktop(() =>
{
_win32Interop.SwitchToInputDesktop();
_win32Interop.InvokeWheelScroll(x, y, scrollY, scrollX);
});
_queueSignal.Set();
}

public void TypeText(string text)
{
_actionQueue.Enqueue(() =>
TryOnInputDesktop(() =>
{
_win32Interop.SwitchToInputDesktop();
_win32Interop.TypeText(text);
});
_queueSignal.Set();
}

private void TryOnInputDesktop(Action action)
{
try
{
_win32Interop.SwitchToInputDesktop();
action.Invoke();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error processing input simulator action.");
}
}
}
Loading

0 comments on commit 007238a

Please sign in to comment.