Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Try to find deadlock #265

Closed
wants to merge 20 commits into from
Closed

Try to find deadlock #265

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
37b23bd
Wrap SpinWait to try to find where it's getting stuck in CI.
timcassell Sep 17, 2023
d9275f0
Reset the thread pool when it times out.
timcassell Sep 20, 2023
138a503
Don't allow further threads to run.
timcassell Sep 22, 2023
f1485c2
Try to capture stacktrace of deadlocked threads.
timcassell Sep 22, 2023
b9231f7
Fix compilation errors
timcassell Sep 22, 2023
3604e0c
Added never completed check in TestHelper.Setup.
timcassell Sep 22, 2023
c865122
Fix compile error
timcassell Sep 22, 2023
3fb065b
Use Assert.Inconclusive instead of throw.
timcassell Sep 22, 2023
cd00a47
Only test standalone.
timcassell Sep 23, 2023
e1f04eb
Try writing progress directly to console.
timcassell Sep 23, 2023
6959f6b
Quit.
timcassell Sep 24, 2023
4ab8330
Force kill.
timcassell Sep 24, 2023
beef356
Switch Unity version to 2021.3.30
timcassell Sep 25, 2023
1e1d532
Update actions
timcassell Sep 25, 2023
ad0bba2
Switch Unity to 2021.3.26
timcassell Sep 26, 2023
6cfce33
Comment out Unity tests.
timcassell Sep 26, 2023
6e2db6c
Add check for tests running too long.
timcassell Oct 10, 2023
39b35aa
Increase SpinWait timeout
timcassell Oct 11, 2023
5e9b09b
Background thread timeout.
timcassell Oct 15, 2023
1433a0b
Increased timeout.
timcassell Oct 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions .github/workflows/unity-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ jobs:
fail-fast: false
matrix:
testMode:
- {
name: Editor,
value: PlayMode,
buildTargetId: 1
}
# - {
# name: Editor,
# value: PlayMode,
# buildTargetId: 1
# }
- {
name: Standalone,
value: Standalone,
Expand Down Expand Up @@ -71,7 +71,7 @@ jobs:
}
# Editor uses 2018.4 to test Net3.5 and Net4.x.
# Standalone uses 2019.4 and 2021.3 to test IL2CPP with netstandard 2.0 and netstandard2.1.
unityVersion: [2018.4.36f1, 2019.4.40f1, 2021.3.29f1]
unityVersion: [2018.4.36f1, 2019.4.40f1, 2021.3.26f1]
devMode:
- {
name: devMode,
Expand All @@ -95,7 +95,7 @@ jobs:
}
- {
testMode: { name: Editor },
unityVersion: 2021.3.29f1
unityVersion: 2021.3.26f1
}
# Standalone with IL2CPP can only be built with 2019.4+ (unity-builder docker images constraint), which doesn't support Net3.5.
- {
Expand All @@ -119,7 +119,7 @@ jobs:

steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3

- name: Rewrite ProjectSettings
run: |
Expand All @@ -145,7 +145,7 @@ jobs:
projectPath: ProtoPromise_Unity
testMode: ${{ matrix.testMode.value }}
unityVersion: ${{ matrix.unityVersion }}
timeout-minutes: 120
timeout-minutes: 180

# Workaround for NUnit XML (see https://github.com/dorny/test-reporter/issues/98#issuecomment-867106931)
- name: Install NUnit
Expand Down
4 changes: 2 additions & 2 deletions Package/Core/Cancelations/Internal/CancelationInternal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -869,7 +869,7 @@ internal void UnhookAndDispose()
}

// Spin until this has been disposed.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
while (_parent != null)
{
spinner.SpinOnce();
Expand Down Expand Up @@ -1075,7 +1075,7 @@ internal static void TryUnregisterOrWaitForCallbackToComplete(CancelationRef par
if (idsMatch & parentIsCanceling
& parent._executingThread != Thread.CurrentThread)
{
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
// _this._nodeId will be incremented when the callback is complete and this is disposed.
// parent.TokenId will be incremented when all callbacks are complete and it is disposed.
// We really only need to compare the nodeId, the tokenId comparison is just for a little extra safety in case of thread starvation and node re-use.
Expand Down
36 changes: 36 additions & 0 deletions Package/Core/InternalShared/SpinWaitWithTimeout.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
using System;
using System.Threading;

namespace Proto.Promises
{
internal static partial class Internal
{
internal struct SpinWaitWithTimeout
{
private SpinWait _spinWait;
private ValueStopwatch _stopwatch;
private TimeSpan _timeout;

internal bool NextSpinWillYield
{
get { return _spinWait.NextSpinWillYield; }
}

internal SpinWaitWithTimeout(TimeSpan timeout)
{
_timeout = timeout;
_spinWait = new SpinWait();
_stopwatch = ValueStopwatch.StartNew();
}

internal void SpinOnce()
{
if (NextSpinWillYield && _stopwatch.GetElapsedTime() > _timeout)
{
throw new TimeoutException("SpinWait exceeded timeout " + _timeout);
}
_spinWait.SpinOnce();
}
}
}
}
11 changes: 11 additions & 0 deletions Package/Core/InternalShared/SpinWaitWithTimeout.cs.meta
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Package/Core/InternalShared/ValueCollectionsInternal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ internal void Enter()
private void EnterCore()
{
// Spin until we successfully get lock.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
do
{
spinner.SpinOnce();
Expand Down
2 changes: 2 additions & 0 deletions Package/Core/Promises/Config.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ public enum PoolType : byte
#endif
public static class Config
{
internal static readonly TimeSpan SpinTimeout = TimeSpan.FromSeconds(10);

[Obsolete("Use ProgressPrecision to get the precision of progress reports."), EditorBrowsable(EditorBrowsableState.Never)]
public static readonly int ProgressDecimalBits = 32;

Expand Down
2 changes: 1 addition & 1 deletion Package/Core/Promises/Internal/Progress/ProgressMergeInternal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ internal void Handle(float oldProgress, float maxProgress, PromiseRefBase handle
[MethodImpl(MethodImplOptions.NoInlining)]
private void WaitForHookup()
{
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
while (_retainCounter == 0)
{
spinner.SpinOnce();
Expand Down
2 changes: 1 addition & 1 deletion Package/Core/Promises/Internal/Progress/ProgressSingleAwaitInternal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ private void WaitForSecondPreviousAssignment()
[MethodImpl(MethodImplOptions.NoInlining)]
private void WaitForSecondPreviousAssignmentCore()
{
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
while (_waitState == WaitState.SettingSecond)
{
spinner.SpinOnce();
Expand Down
6 changes: 3 additions & 3 deletions Package/Core/Promises/Internal/PromiseInternal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ private static bool TryWaitForCompletion(PromiseRefBase promise, short promiseId
private bool TryWaitForCompletion(PromiseRefBase promise, TimeSpan timeout, ValueStopwatch stopwatch)
{
// We do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
if (timeout.Milliseconds == Timeout.Infinite)
{
while (promise.State == Promise.State.Pending & !spinner.NextSpinWillYield)
Expand Down Expand Up @@ -259,7 +259,7 @@ internal override void Handle(PromiseRefBase handler, object rejectContainer, Pr
}

// Wait until we're sure the other thread has continued.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
while (waitState <= CompletedState)
{
spinner.SpinOnce();
Expand Down Expand Up @@ -512,7 +512,7 @@ internal void WaitUntilStateIsNotPending()

private void WaitUntilStateIsNotPendingCore()
{
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
while (State == Promise.State.Pending)
{
spinner.SpinOnce();
Expand Down
4 changes: 2 additions & 2 deletions Package/Core/Threading/Internal/AsyncAutoResetEventInternal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ internal Promise<bool> TryWaitAsync(CancelationToken cancelationToken)
internal void Wait()
{
// Because this is a synchronous wait, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
while (!_isSet & !spinner.NextSpinWillYield)
{
spinner.SpinOnce();
Expand All @@ -183,7 +183,7 @@ internal void Wait()
internal bool TryWait(CancelationToken cancelationToken)
{
// Because this is a synchronous wait, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isCanceled = cancelationToken.IsCancelationRequested;
while (!_isSet & !isCanceled & !spinner.NextSpinWillYield)
{
Expand Down
4 changes: 2 additions & 2 deletions Package/Core/Threading/Internal/AsyncCountdownEventInternal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ internal Promise<bool> TryWaitAsync(CancelationToken cancelationToken)
internal void Wait()
{
// Because this is a synchronous wait, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isSet = _currentCount == 0;
while (!isSet & !spinner.NextSpinWillYield)
{
Expand Down Expand Up @@ -197,7 +197,7 @@ internal void Wait()
internal bool TryWait(CancelationToken cancelationToken)
{
// Because this is a synchronous wait, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isSet = _currentCount == 0;
bool isCanceled = cancelationToken.IsCancelationRequested;
while (!isSet & !isCanceled & !spinner.NextSpinWillYield)
Expand Down
6 changes: 3 additions & 3 deletions Package/Core/Threading/Internal/AsyncLockInternal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ private void SetNextKey()
internal AsyncLock.Key Lock()
{
// Since this is a synchronous lock, we do a short spinwait before entering the full lock.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
while (Volatile.Read(ref _currentKey) != 0 & !spinner.NextSpinWillYield)
{
spinner.SpinOnce();
Expand Down Expand Up @@ -315,7 +315,7 @@ internal AsyncLock.Key Lock()
internal AsyncLock.Key Lock(CancelationToken cancelationToken)
{
// Because this is a synchronous wait, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isCanceled = cancelationToken.IsCancelationRequested;
while (Volatile.Read(ref _currentKey) != 0 & !isCanceled & !spinner.NextSpinWillYield)
{
Expand Down Expand Up @@ -402,7 +402,7 @@ internal bool TryEnter(out AsyncLock.Key key)
internal bool TryEnter(out AsyncLock.Key key, CancelationToken cancelationToken)
{
// Because this is a synchronous wait, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isCanceled = cancelationToken.IsCancelationRequested;
while (Volatile.Read(ref _currentKey) != 0 & !isCanceled & !spinner.NextSpinWillYield)
{
Expand Down
4 changes: 2 additions & 2 deletions Package/Core/Threading/Internal/AsyncManualResetEventInternal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ internal Promise<bool> TryWaitAsync(CancelationToken cancelationToken)
internal void Wait()
{
// Because this is a synchronous wait, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isSet = _isSet;
while (!isSet & !spinner.NextSpinWillYield)
{
Expand Down Expand Up @@ -194,7 +194,7 @@ internal void Wait()
internal bool TryWait(CancelationToken cancelationToken)
{
// Because this is a synchronous wait, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isSet = _isSet;
bool isCanceled = cancelationToken.IsCancelationRequested;
while (!isSet & !isCanceled & !spinner.NextSpinWillYield)
Expand Down
24 changes: 12 additions & 12 deletions Package/Core/Threading/Internal/AsyncReaderWriterLockInternal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@ private bool CanEnterReaderLock(AsyncReaderWriterLockType currentLockType)
internal AsyncReaderWriterLock.ReaderKey ReaderLock()
{
// Since this is a synchronous lock, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
while (!CanEnterReaderLock(_lockType) & !spinner.NextSpinWillYield)
{
spinner.SpinOnce();
Expand Down Expand Up @@ -451,7 +451,7 @@ internal AsyncReaderWriterLock.ReaderKey ReaderLock()
internal AsyncReaderWriterLock.ReaderKey ReaderLock(CancelationToken cancelationToken)
{
// Since this is a synchronous lock, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isCanceled = cancelationToken.IsCancelationRequested;
while (!CanEnterReaderLock(_lockType) & !isCanceled & !spinner.NextSpinWillYield)
{
Expand Down Expand Up @@ -577,7 +577,7 @@ internal bool TryEnterReaderLock(out AsyncReaderWriterLock.ReaderKey readerKey)
internal bool TryEnterReaderLock(out AsyncReaderWriterLock.ReaderKey readerKey, CancelationToken cancelationToken)
{
// Since this is a synchronous lock, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isCanceled = cancelationToken.IsCancelationRequested;
while (!CanEnterReaderLock(_lockType) & !isCanceled & !spinner.NextSpinWillYield)
{
Expand Down Expand Up @@ -688,7 +688,7 @@ internal bool TryEnterReaderLock(out AsyncReaderWriterLock.ReaderKey readerKey,
internal AsyncReaderWriterLock.WriterKey WriterLock()
{
// Since this is a synchronous lock, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
while (_lockType != AsyncReaderWriterLockType.None & !spinner.NextSpinWillYield)
{
spinner.SpinOnce();
Expand Down Expand Up @@ -720,7 +720,7 @@ internal AsyncReaderWriterLock.WriterKey WriterLock()
internal AsyncReaderWriterLock.WriterKey WriterLock(CancelationToken cancelationToken)
{
// Since this is a synchronous lock, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isCanceled = cancelationToken.IsCancelationRequested;
while (_lockType != AsyncReaderWriterLockType.None & !isCanceled & !spinner.NextSpinWillYield)
{
Expand Down Expand Up @@ -821,7 +821,7 @@ internal bool TryEnterWriterLock(out AsyncReaderWriterLock.WriterKey writerKey)
internal bool TryEnterWriterLock(out AsyncReaderWriterLock.WriterKey writerKey, CancelationToken cancelationToken)
{
// Since this is a synchronous lock, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isCanceled = cancelationToken.IsCancelationRequested;
while (_lockType != AsyncReaderWriterLockType.None & !isCanceled & !spinner.NextSpinWillYield)
{
Expand Down Expand Up @@ -945,7 +945,7 @@ private bool CanEnterUpgradeableReaderLock(AsyncReaderWriterLockType currentLock
internal AsyncReaderWriterLock.UpgradeableReaderKey UpgradeableReaderLock()
{
// Since this is a synchronous lock, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
while (!CanEnterUpgradeableReaderLock(_lockType) & !spinner.NextSpinWillYield)
{
spinner.SpinOnce();
Expand Down Expand Up @@ -981,7 +981,7 @@ internal AsyncReaderWriterLock.UpgradeableReaderKey UpgradeableReaderLock()
internal AsyncReaderWriterLock.UpgradeableReaderKey UpgradeableReaderLock(CancelationToken cancelationToken)
{
// Since this is a synchronous lock, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isCanceled = cancelationToken.IsCancelationRequested;
while (!CanEnterUpgradeableReaderLock(_lockType) & !isCanceled & !spinner.NextSpinWillYield)
{
Expand Down Expand Up @@ -1093,7 +1093,7 @@ internal bool TryEnterUpgradeableReaderLock(out AsyncReaderWriterLock.Upgradeabl
internal bool TryEnterUpgradeableReaderLock(out AsyncReaderWriterLock.UpgradeableReaderKey readerKey, CancelationToken cancelationToken)
{
// Since this is a synchronous lock, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isCanceled = cancelationToken.IsCancelationRequested;
while (!CanEnterUpgradeableReaderLock(_lockType) & !isCanceled & !spinner.NextSpinWillYield)
{
Expand Down Expand Up @@ -1208,7 +1208,7 @@ internal bool TryEnterUpgradeableReaderLock(out AsyncReaderWriterLock.Upgradeabl
internal AsyncReaderWriterLock.WriterKey UpgradeToWriterLock(AsyncReaderWriterLock.UpgradeableReaderKey readerKey)
{
// Since this is a synchronous lock, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
while (_readerLockCount != 1 & !spinner.NextSpinWillYield)
{
spinner.SpinOnce();
Expand Down Expand Up @@ -1245,7 +1245,7 @@ internal AsyncReaderWriterLock.WriterKey UpgradeToWriterLock(AsyncReaderWriterLo
internal AsyncReaderWriterLock.WriterKey UpgradeToWriterLock(AsyncReaderWriterLock.UpgradeableReaderKey readerKey, CancelationToken cancelationToken)
{
// Since this is a synchronous lock, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isCanceled = cancelationToken.IsCancelationRequested;
while (_readerLockCount != 1 & !isCanceled & !spinner.NextSpinWillYield)
{
Expand Down Expand Up @@ -1362,7 +1362,7 @@ internal bool TryUpgradeToWriterLock(AsyncReaderWriterLock.UpgradeableReaderKey
internal bool TryUpgradeToWriterLock(AsyncReaderWriterLock.UpgradeableReaderKey readerKey, out AsyncReaderWriterLock.WriterKey writerKey, CancelationToken cancelationToken)
{
// Since this is a synchronous lock, we do a short spinwait before yielding the thread.
var spinner = new SpinWait();
var spinner = new SpinWaitWithTimeout(Promise.Config.SpinTimeout);
bool isCanceled = cancelationToken.IsCancelationRequested;
while (_readerLockCount != 1 & !isCanceled & !spinner.NextSpinWillYield)
{
Expand Down
Loading
Loading