Improve history restart reliability a bit (#610)

I couldn't reproduce the actual issue here, but I noticed potential for some racyness and the logging was a bit thin. I also don't trust AutoResetEvent, so I'm swapping it for ManualResetEvent.
cognitedata · Feb 22, 2024 · c85d67e · c85d67e
1 parent c9ce092
commit c85d67e
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 21 deletions.
diff --git a/Extractor/History/HistoryReader.cs b/Extractor/History/HistoryReader.cs
@@ -96,7 +96,7 @@ public void MaxNodeParallelismChanged()
             continuationPoints.SetCapacity(config.History.Throttling.MaxNodeParallelism > 0 ? config.History.Throttling.MaxNodeParallelism : 1_000);
         }
 
-        private readonly AutoResetEvent stateChangedEvent = new AutoResetEvent(true);
+        private readonly ManualResetEvent stateChangedEvent = new ManualResetEvent(true);
 
         private Task? activeTask;
 
@@ -109,6 +109,7 @@ public async Task Run(CancellationToken token)
             {
                 if (activeTask != null)
                 {
+                    log.LogDebug("History is running, waiting for waker");
                     var res = await Task.WhenAny(activeTask, waitTask);
 
                     if (runTaskSource.IsCancellationRequested) break;
@@ -130,6 +131,7 @@ public async Task Run(CancellationToken token)
                     if (runTaskSource.IsCancellationRequested) break;
                 }
 
+                stateChangedEvent.Reset();
                 waitTask = CommonUtils.WaitAsync(stateChangedEvent, Timeout.InfiniteTimeSpan, runTaskSource.Token);
 
                 if (activeTask != null)
@@ -276,31 +278,39 @@ private void RestartHistoryInStates()
 
         public void RemoveIssue(StateIssue issue)
         {
-            bool stateOk = state.IsGood;
-
-            if (state.RemoveIssue(issue))
+            lock (statesLock)
             {
-                log.LogDebug("Removed history issue: {Issue}. New state: {State}", issue, state);
-            }
+                bool stateOk = state.IsGood;
 
-            if (stateOk != state.IsGood)
-            {
-                stateChangedEvent.Set();
+                if (state.RemoveIssue(issue))
+                {
+                    log.LogDebug("Removed history issue: {Issue}. New state: {State}", issue, state);
+                }
+
+                if (stateOk != state.IsGood)
+                {
+                    log.LogDebug("History state changed to good, history will restart");
+                    stateChangedEvent.Set();
+                }
             }
         }
 
         public void AddIssue(StateIssue issue)
         {
-            bool stateOk = state.IsGood;
-
-            if (state.AddIssue(issue))
+            lock (statesLock)
             {
-                log.LogDebug("Added history issue: {Issue}. New state: {State}", issue, state);
-            }
+                bool stateOk = state.IsGood;
 
-            if (stateOk != state.IsGood)
-            {
-                stateChangedEvent.Set();
+                if (state.AddIssue(issue))
+                {
+                    log.LogDebug("Added history issue: {Issue}. New state: {State}", issue, state);
+                }
+
+                if (stateOk != state.IsGood)
+                {
+                    log.LogDebug("History state changed to bad, history will stop");
+                    stateChangedEvent.Set();
+                }
             }
         }
 
@@ -317,10 +327,10 @@ public void AddStates(IEnumerable<VariableExtractionState> varStates, IEnumerabl
                 {
                     activeEventStates[state.SourceId] = state;
                 }
-            }
-            if (state.IsGood)
-            {
-                stateChangedEvent.Set();
+                if (state.IsGood)
+                {
+                    stateChangedEvent.Set();
+                }
             }
         }
 

diff --git a/manifest.yml b/manifest.yml
@@ -60,6 +60,8 @@ schema:
    - "https://raw.githubusercontent.com/"
 
 versions:
+  "2.27.2":
+    description: Improve reliability of history restart triggers.
   "2.27.1":
     description: Enforce start-time < end-time when reading history, fix subscription recreation bug.
     changelog: