diff --git a/osu.Server.Spectator/Hubs/Spectator/SpectatorClientState.cs b/osu.Server.Spectator/Hubs/Spectator/SpectatorClientState.cs
index ce6f200a..2e6b4ea0 100644
--- a/osu.Server.Spectator/Hubs/Spectator/SpectatorClientState.cs
+++ b/osu.Server.Spectator/Hubs/Spectator/SpectatorClientState.cs
@@ -2,6 +2,7 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System;
+using System.Collections.Generic;
 using Newtonsoft.Json;
 using osu.Game.Online.Spectator;
 using osu.Game.Scoring;
@@ -26,6 +27,11 @@ public class SpectatorClientState : ClientState
         /// </summary>
         public long? ScoreToken;
 
+        /// <summary>
+        /// The list of IDs of users that this client is currently watching.
+        /// </summary>
+        public HashSet<int> WatchedUsers = new HashSet<int>();
+
         [JsonConstructor]
         public SpectatorClientState(in string connectionId, in int userId)
             : base(connectionId, userId)
diff --git a/osu.Server.Spectator/Hubs/Spectator/SpectatorHub.cs b/osu.Server.Spectator/Hubs/Spectator/SpectatorHub.cs
index c913e164..892c5e81 100644
--- a/osu.Server.Spectator/Hubs/Spectator/SpectatorHub.cs
+++ b/osu.Server.Spectator/Hubs/Spectator/SpectatorHub.cs
@@ -154,7 +154,12 @@ public async Task EndPlaySession(SpectatorState state)
                 }
                 finally
                 {
-                    usage.Destroy();
+                    if (usage.Item != null)
+                    {
+                        usage.Item.State = null;
+                        usage.Item.Score = null;
+                        usage.Item.ScoreToken = null;
+                    }
                 }
             }
 
@@ -202,6 +207,12 @@ public async Task StartWatchingUser(int userId)
                 // user isn't tracked.
             }
 
+            using (var state = await GetOrCreateLocalUserState())
+            {
+                var clientState = state.Item ??= new SpectatorClientState(Context.ConnectionId, Context.GetUserId());
+                clientState.WatchedUsers.Add(userId);
+            }
+
             await Groups.AddToGroupAsync(Context.ConnectionId, GetGroupId(userId));
 
             int watcherId = Context.GetUserId();
@@ -225,6 +236,12 @@ public async Task EndWatchingUser(int userId)
         {
             await Groups.RemoveFromGroupAsync(Context.ConnectionId, GetGroupId(userId));
 
+            using (var state = await GetOrCreateLocalUserState())
+            {
+                var clientState = state.Item ??= new SpectatorClientState(Context.ConnectionId, Context.GetUserId());
+                clientState.WatchedUsers.Remove(userId);
+            }
+
             int watcherId = Context.GetUserId();
 
             await Clients.User(userId.ToString()).UserEndedWatching(watcherId);
@@ -245,6 +262,9 @@ protected override async Task CleanUpState(SpectatorClientState state)
             if (state.State != null)
                 await endPlaySession(state.UserId, state.State);
 
+            foreach (int watchedUserId in state.WatchedUsers)
+                await Clients.User(watchedUserId.ToString()).UserEndedWatching(state.UserId);
+
             await base.CleanUpState(state);
         }