From 665fb3de69499df53970285c5bba461faad96877 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Wed, 1 Feb 2023 09:10:28 -0600 Subject: [PATCH] Maybe a little better resiliency in the node reassignment handler --- .../IncrementBHandler1993886962.cs.cs | 6 +++--- .../Durability/DurabilityAgent.cs | 2 +- .../Durability/NodeReassignment.cs | 20 +++++++++++++++++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/Persistence/PersistenceTests/Internal/Generated/WolverineHandlers/IncrementBHandler1993886962.cs.cs b/src/Persistence/PersistenceTests/Internal/Generated/WolverineHandlers/IncrementBHandler1993886962.cs.cs index 9d893a9fb..030e03f9e 100644 --- a/src/Persistence/PersistenceTests/Internal/Generated/WolverineHandlers/IncrementBHandler1993886962.cs.cs +++ b/src/Persistence/PersistenceTests/Internal/Generated/WolverineHandlers/IncrementBHandler1993886962.cs.cs @@ -8,13 +8,13 @@ namespace Internal.Generated.WolverineHandlers // START: IncrementBHandler1993886962 public class IncrementBHandler1993886962 : Wolverine.Runtime.Handlers.MessageHandler { - private readonly Microsoft.Extensions.Logging.ILogger _logger; private readonly Wolverine.Marten.Publishing.OutboxedSessionFactory _outboxedSessionFactory; + private readonly Microsoft.Extensions.Logging.ILogger _logger; - public IncrementBHandler1993886962(Microsoft.Extensions.Logging.ILogger logger, Wolverine.Marten.Publishing.OutboxedSessionFactory outboxedSessionFactory) + public IncrementBHandler1993886962(Wolverine.Marten.Publishing.OutboxedSessionFactory outboxedSessionFactory, Microsoft.Extensions.Logging.ILogger logger) { - _logger = logger; _outboxedSessionFactory = outboxedSessionFactory; + _logger = logger; } diff --git a/src/Persistence/Wolverine.RDBMS/Durability/DurabilityAgent.cs b/src/Persistence/Wolverine.RDBMS/Durability/DurabilityAgent.cs index 5657f5fa3..509ff4b3b 100644 --- a/src/Persistence/Wolverine.RDBMS/Durability/DurabilityAgent.cs +++ b/src/Persistence/Wolverine.RDBMS/Durability/DurabilityAgent.cs @@ -61,7 +61,7 @@ internal DurabilityAgent(IWolverineRuntime runtime, ILogger logger, _incomingMessages = new RecoverIncomingMessages(logger, runtime.Endpoints); _outgoingMessages = new RecoverOutgoingMessages(runtime, logger); - _nodeReassignment = new NodeReassignment(); + _nodeReassignment = new NodeReassignment(logger); _deleteExpired = new DeleteExpiredHandledEnvelopes(); _scheduledJobs = new RunScheduledJobs(settings, logger); _moveReplayable = new MoveReplayableErrorMessagesToIncoming(); diff --git a/src/Persistence/Wolverine.RDBMS/Durability/NodeReassignment.cs b/src/Persistence/Wolverine.RDBMS/Durability/NodeReassignment.cs index 959400836..f0f844c83 100644 --- a/src/Persistence/Wolverine.RDBMS/Durability/NodeReassignment.cs +++ b/src/Persistence/Wolverine.RDBMS/Durability/NodeReassignment.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Weasel.Core; using Wolverine.Persistence.Durability; using Wolverine.Transports; @@ -9,6 +10,13 @@ namespace Wolverine.RDBMS.Durability; internal class NodeReassignment : IDurabilityAction { + private readonly ILogger _logger; + + public NodeReassignment(ILogger logger) + { + _logger = logger; + } + public string Description { get; } = "Dormant node reassignment"; public async Task ExecuteAsync(IMessageDatabase database, IDurabilityAgent agent, @@ -18,7 +26,7 @@ await session.WithinTransactionalGlobalLockAsync(TransportConstants.Reassignment () => ReassignNodesAsync(session, database.Node, database.Settings)); } - public static async Task ReassignNodesAsync(IDurableStorageSession session, NodeSettings nodeSettings, + public async Task ReassignNodesAsync(IDurableStorageSession session, NodeSettings nodeSettings, DatabaseSettings databaseSettings) { var owners = await FindUniqueOwnersAsync(session, nodeSettings, databaseSettings); @@ -33,7 +41,15 @@ public static async Task ReassignNodesAsync(IDurableStorageSession session, Node if (await session.TryGetGlobalTxLockAsync(owner)) { await ReassignDormantNodeToAnyNodeAsync(session, owner, databaseSettings); - await session.ReleaseGlobalLockAsync(owner); + try + { + await session.ReleaseGlobalLockAsync(owner); + } + catch (Exception e) + { + // Need to swallow the exception on releasing the tx here + _logger.LogError(e, "Error trying to release global transaction lock for '{Owner}'", owner); + } } } }