Merge pull request #214 from Concordium/support-p7

Support Concordium Protocol Version 7

.github/workflows/ci.yml

@@ -21,12 +21,16 @@ jobs:
       uses: actions/setup-dotnet@v3
       with:
         dotnet-version: 6.0.x
-    
+
     - name: Restore dependencies
       run: dotnet restore ./backend/CcScan.Backend.sln
-    
+
     - name: Build
       run: dotnet build ./backend/CcScan.Backend.sln -c Release --no-restore
 
     - name: Test
-      run: dotnet test ./backend/CcScan.Backend.sln --filter Category!=IntegrationTests -c Release --no-build --verbosity normal
+      run: |
+        # Tests depend on docker-compose being available due to this issue https://github.com/mariotoffia/FluentDocker/issues/312.
+        # The soft linking should be remove when a fix is released.
+        ln -s /usr/libexec/docker/cli-plugins/docker-compose /usr/local/bin/docker-compose
+        dotnet test ./backend/CcScan.Backend.sln --filter Category!=IntegrationTests -c Release --no-build --verbosity normal

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "backend/concordium-net-sdk"]
+	path = backend/concordium-net-sdk
+	url = ../concordium-net-sdk.git

backend/Application/Api/GraphQL/Import/AccountWriter.cs

@@ -8,6 +8,7 @@
 using Dapper;
 using Microsoft.EntityFrameworkCore;
 using Npgsql;
+using Concordium.Sdk.Types;
 
 namespace Application.Api.GraphQL.Import;
 
@@ -157,20 +158,23 @@ private static IEnumerable<T> IterateBatchDbDataReader<T>(DbDataReader reader, F
     public async Task UpdateAccount<TSource>(TSource item, Func<TSource, ulong> delegatorIdSelector, Action<TSource, Account> updateAction)
     {
         using var counter = _metrics.MeasureDuration(nameof(AccountWriter), nameof(UpdateAccount));
-        
+
         await using var context = await _dbContextFactory.CreateDbContextAsync();
         var delegatorId = (long)delegatorIdSelector(item);
 
         var account = await context.Accounts.SingleAsync(x => x.Id == delegatorId);
         updateAction(item, account);
-        
+
         await context.SaveChangesAsync();
     }
-
+
+    /// <summary>
+    /// Update using <paramref name="updateAction"/> on each account with a pending change for delegation that is effective before <paramref name="effectiveTimeEqualOrBefore"/>.
+    /// </summary>
     public async Task UpdateAccountsWithPendingDelegationChange(DateTimeOffset effectiveTimeEqualOrBefore, Action<Account> updateAction)
     {
         using var counter = _metrics.MeasureDuration(nameof(AccountWriter), nameof(UpdateAccountsWithPendingDelegationChange));
-        
+
         await using var context = await _dbContextFactory.CreateDbContextAsync();
 
         var sql = $"select * from graphql_accounts where delegation_pending_change->'data'->>'EffectiveTime' <= '{effectiveTimeEqualOrBefore:O}'";
@@ -202,7 +206,31 @@ public async Task UpdateAccounts(Expression<Func<Account, bool>> whereClause, Ac
             await context.SaveChangesAsync();
         }
     }
-
+
+    /// <summary>
+    /// Remove baker and move its delegators to the passive pool.
+    /// Returns the number of delegators which were moved.
+    /// </summary>
+    public async Task<int> RemoveBaker(BakerId bakerId, DateTimeOffset effectiveTime) {
+        using var counter = _metrics.MeasureDuration(nameof(AccountWriter), nameof(RemoveBaker));
+        await using var context = await _dbContextFactory.CreateDbContextAsync();
+
+        var baker = await context.Bakers.SingleAsync(x => x.Id == (long)bakerId.Id.Index);
+        baker.State = new Bakers.RemovedBakerState(effectiveTime);
+
+        var target = new BakerDelegationTarget((long) bakerId.Id.Index);
+        var delegatorAccounts = await context.Accounts
+             .Where(account => account.Delegation != null && account.Delegation.DelegationTarget == target)
+             .ToArrayAsync();
+        foreach (var delegatorAccount in delegatorAccounts) {
+            var delegation = delegatorAccount.Delegation ?? throw new InvalidOperationException("Account delegating to baker target being removed has no delegation attached.");
+            delegation.DelegationTarget = new PassiveDelegationTarget();
+        }
+
+        await context.SaveChangesAsync();
+        return delegatorAccounts.Length;
+    }
+
     public async Task UpdateDelegationStakeIfRestakingEarnings(AccountRewardSummary[] stakeUpdates)
     {
         using var counter = _metrics.MeasureDuration(nameof(AccountWriter), nameof(UpdateDelegationStakeIfRestakingEarnings));

backend/Application/Api/GraphQL/Import/BakerChangeStrategy.cs

@@ -9,24 +9,27 @@ internal interface IBakerChangeStrategy
     Task UpdateBakersFromTransactionEvents(
         IEnumerable<AccountTransactionDetails> transactionEvents,
         ImportState importState,
-        BakerImportHandler.BakerUpdateResultsBuilder resultBuilder);
-
+        BakerImportHandler.BakerUpdateResultsBuilder resultBuilder,
+        DateTimeOffset blockSlotTime);
+
     bool MustApplyPendingChangesDue(DateTimeOffset? nextPendingBakerChangeTime);
     DateTimeOffset GetEffectiveTime();
+    /// <summary>Whether the protocol supports pending changes. Starting from Protocol version 7 this is not the case.</summary>
+    bool SupportsPendingChanges();
 }
 
 internal static class BakerChangeStrategyFactory
 {
     internal static IBakerChangeStrategy Create(
         BlockInfo blockInfo,
-        ChainParameters chainParameters, 
+        ChainParameters chainParameters,
         BlockImportPaydayStatus importPaydayStatus,
         BakerWriter writer,
         AccountInfo[] bakersWithNewPendingChanges)
     {
         if (blockInfo.ProtocolVersion.AsInt() < 4)
             return new PreProtocol4Strategy(bakersWithNewPendingChanges, blockInfo, writer);
-        
+
         ChainParameters.TryGetPoolOwnerCooldown(chainParameters, out var poolOwnerCooldown);
         return new PostProtocol4Strategy(blockInfo, poolOwnerCooldown!.Value, importPaydayStatus, writer);
 
@@ -46,13 +49,17 @@ public PreProtocol4Strategy(AccountInfo[] accountInfos, BlockInfo blockInfo, Bak
         _accountInfos = accountInfos;
     }
 
+    public bool SupportsPendingChanges() => true;
+
     /// <summary>
     /// Prior to protocol 4 <see cref="Concordium.Sdk.Types.BakerConfigured"/> isn't used.
     /// </summary>
     public async Task UpdateBakersFromTransactionEvents(
         IEnumerable<AccountTransactionDetails> transactionEvents,
         ImportState importState,
-        BakerImportHandler.BakerUpdateResultsBuilder resultBuilder)
+        BakerImportHandler.BakerUpdateResultsBuilder resultBuilder,
+        DateTimeOffset blockSlotTime
+    )
     {
         foreach (var txEvent in transactionEvents)
         {
@@ -184,6 +191,8 @@ public bool MustApplyPendingChangesDue(DateTimeOffset? nextPendingBakerChangeTim
         return false;
     }
 
+    public bool SupportsPendingChanges() => _blockInfo.ProtocolVersion < ProtocolVersion.P7;
+
     public DateTimeOffset GetEffectiveTime()
     {
         if (_importPaydayStatus is FirstBlockAfterPayday firstBlockAfterPayday)
@@ -195,8 +204,9 @@ public DateTimeOffset GetEffectiveTime()
     /// <see cref="Concordium.Sdk.Types.BakerConfigured"/> are used from protocol 4.
     /// </summary>
     public async Task UpdateBakersFromTransactionEvents(IEnumerable<AccountTransactionDetails> transactionEvents, ImportState importState,
-        BakerImportHandler.BakerUpdateResultsBuilder resultBuilder)
+        BakerImportHandler.BakerUpdateResultsBuilder resultBuilder, DateTimeOffset blockSlotTime)
     {
+        bool supportsPendingChanges = SupportsPendingChanges();
         foreach (var txEvent in transactionEvents)
         {
             switch (txEvent.Effects)
@@ -219,8 +229,21 @@ await _writer.AddOrUpdateBaker(bakerAdded,
                                 resultBuilder.IncrementBakersAdded();
                                 break;
                             case BakerRemovedEvent bakerRemovedEvent:
-                                var pendingChange = await SetPendingChangeOnBaker(bakerRemovedEvent.BakerId, bakerRemovedEvent);
-                                importState.UpdateNextPendingBakerChangeTimeIfLower(pendingChange.EffectiveTime);
+                                if (supportsPendingChanges) {
+                                    // If the protocol version (prior to 7) supports pending changes, then store a pending change.
+                                    var pendingChange = await SetPendingChangeOnBaker(bakerRemovedEvent.BakerId, bakerRemovedEvent);
+                                    importState.UpdateNextPendingBakerChangeTimeIfLower(pendingChange.EffectiveTime);
+                                } else {
+                                    // Otherwise update the stake immediately.
+                                    await _writer.UpdateBaker(bakerRemovedEvent,
+                                    src => src.BakerId.Id.Index,
+                                    (src, dst) =>
+                                    {
+                                        var activeState = dst.State as ActiveBakerState ?? throw new InvalidOperationException("Cannot remove a baker that is not active!");
+                                        dst.State = new RemovedBakerState(blockSlotTime);
+                                        resultBuilder.AddBakerRemoved((long) bakerRemovedEvent.BakerId.Id.Index);
+                                    });
+                                }
                                 break;
                             case BakerRestakeEarningsUpdatedEvent bakerRestakeEarningsUpdatedEvent:
                                 await _writer.UpdateBaker(bakerRestakeEarningsUpdatedEvent,
@@ -280,18 +303,36 @@ await _writer.UpdateBaker(bakerSetOpenStatusEvent,
                                     resultBuilder.AddBakerClosedForAll((long)bakerSetOpenStatusEvent.BakerId.Id.Index);
                                 break;
                             case BakerStakeDecreasedEvent bakerStakeDecreasedEvent:
-                                var pendingChangeStakeDecreased = await SetPendingChangeOnBaker(bakerStakeDecreasedEvent.BakerId, bakerStakeDecreasedEvent);
-                                importState.UpdateNextPendingBakerChangeTimeIfLower(pendingChangeStakeDecreased.EffectiveTime);
+                                if (supportsPendingChanges) {
+                                    // If the protocol version (prior to 7) supports pending changes, then store a pending change.
+                                    var pendingChangeStakeDecreased = await SetPendingChangeOnBaker(bakerStakeDecreasedEvent.BakerId, bakerStakeDecreasedEvent);
+                                    importState.UpdateNextPendingBakerChangeTimeIfLower(pendingChangeStakeDecreased.EffectiveTime);
+                                } else {
+                                    // From protocol version 7 and onwards stake changes are immediate.
+                                    await _writer.UpdateBaker(bakerStakeDecreasedEvent,
+                                    src => src.BakerId.Id.Index,
+                                    (src, dst) =>
+                                    {
+                                        var activeState = dst.State as ActiveBakerState ?? throw new InvalidOperationException("Cannot decrease stake for a baker that is not active!");
+                                        activeState.StakedAmount = src.NewStake.Value;
+                                    });
+                                }
                                 break;
                             case BakerStakeIncreasedEvent bakerStakeIncreasedEvent:
                                 await _writer.UpdateBaker(bakerStakeIncreasedEvent,
                                     src => src.BakerId.Id.Index,
                                     (src, dst) =>
                                     {
-                                        var activeState = dst.State as ActiveBakerState ?? throw new InvalidOperationException("Cannot set restake earnings for a baker that is not active!");
+                                        var activeState = dst.State as ActiveBakerState ?? throw new InvalidOperationException("Cannot increase stake for a baker that is not active!");
                                         activeState.StakedAmount = src.NewStake.Value;
                                     });
                                 break;
+                            case BakerEventDelegationRemoved delegationRemoved:
+                                // This event was introduced as part of Concordium Protocol Version 7,
+                                // which also removes the logic around pending changes, meaning we can
+                                // just update the state immediately.
+                                await _writer.RemoveDelegator(delegationRemoved.DelegatorId);
+                                break;
                             case BakerKeysUpdatedEvent:
                             default:
                                 break;

backend/Application/Api/GraphQL/Import/BakerImportHandler.cs

@@ -25,11 +25,14 @@ public BakerImportHandler(IDbContextFactory<GraphQlDbContext> dbContextFactory,
         _logger = Log.ForContext(GetType());
     }
 
+    /// <summary>
+    /// Process block data related to changes for bakers/validators.
+    /// </summary>
     public async Task<BakerUpdateResults> HandleBakerUpdates(BlockDataPayload payload, RewardsSummary rewardsSummary,
         ChainParametersState chainParameters, BlockImportPaydayStatus importPaydayStatus, ImportState importState)
     {
         using var counter = _metrics.MeasureDuration(nameof(BakerImportHandler), nameof(HandleBakerUpdates));
-        
+
         var changeStrategy = BakerChangeStrategyFactory.Create(payload.BlockInfo, chainParameters.Current, importPaydayStatus, _writer,
             payload.AccountInfos.BakersWithNewPendingChanges);;
 
@@ -164,8 +167,8 @@ or BakerRestakeEarningsUpdated
                 or BakerConfigured
                 or BakerKeysUpdated
             );
-        
-        await bakerChangeStrategy.UpdateBakersFromTransactionEvents(txEvents, importState, resultBuilder);
+
+        await bakerChangeStrategy.UpdateBakersFromTransactionEvents(txEvents, importState, resultBuilder, payload.BlockInfo.BlockSlotTime);
 
         // This should happen after the bakers from current block has been added to the database
         if (isFirstBlockAfterPayday)
@@ -224,10 +227,10 @@ await _writer.UpdateBakers(
                         FinalizationCommission = source.PoolInfo.CommissionRates.FinalizationCommission.AsDecimal(),
                         BakingCommission = source.PoolInfo.CommissionRates.BakingCommission.AsDecimal()
                     },
-                    DelegatedStake = source.DelegatedCapital.Value,
+                    DelegatedStake = source.DelegatedCapital!.Value.Value,
                     DelegatorCount = 0,
-                    DelegatedStakeCap = source.DelegatedCapitalCap.Value,
-                    TotalStake = source.BakerEquityCapital.Value + source.DelegatedCapital.Value
+                    DelegatedStakeCap = source.DelegatedCapitalCap!.Value.Value,
+                    TotalStake = source.BakerEquityCapital!.Value.Value + source.DelegatedCapital.Value.Value
                 };
                 pool.ApplyPaydayStatus(source.CurrentPaydayStatus, source.PoolInfo.CommissionRates);
 
@@ -262,8 +265,8 @@ await _writer.UpdateBakers(baker =>
                 {
                     var rates = baker.ActiveState!.Pool!.CommissionRates;
                     rates.FinalizationCommission = AdjustValueToRange(rates.FinalizationCommission, currentFinalizationCommissionRange);
-                    rates.BakingCommission = AdjustValueToRange(rates.BakingCommission, currentBakingCommissionRange);
-                    rates.TransactionCommission = AdjustValueToRange(rates.TransactionCommission, currentTransactionCommissionRange);
+                    rates.BakingCommission = AdjustValueToRange(rates.BakingCommission, currentBakingCommissionRange!);
+                    rates.TransactionCommission = AdjustValueToRange(rates.TransactionCommission, currentTransactionCommissionRange!);
                 },
                 baker => baker.ActiveState!.Pool != null);
 
@@ -309,16 +312,22 @@ await _writer.UpdateBaker(1900UL, bakerId => bakerId, (bakerId, baker) =>
         }
     }
 
-    private async Task UpdateBakersWithPendingChangesDue(IBakerChangeStrategy bakerChangeStrategy, 
+    private async Task UpdateBakersWithPendingChangesDue(IBakerChangeStrategy bakerChangeStrategy,
         ImportState importState, BakerUpdateResultsBuilder resultBuilder)
     {
-        if (bakerChangeStrategy.MustApplyPendingChangesDue(importState.NextPendingBakerChangeTime))
-        {
-            var effectiveTime = bakerChangeStrategy.GetEffectiveTime();
-            await _writer.UpdateBakersWithPendingChange(effectiveTime, baker => ApplyPendingChange(baker, resultBuilder));
+        // Check if this protocol supports pending changes.
+        if (bakerChangeStrategy.SupportsPendingChanges()) {
+            if (bakerChangeStrategy.MustApplyPendingChangesDue(importState.NextPendingBakerChangeTime))
+            {
+                var effectiveTime = bakerChangeStrategy.GetEffectiveTime();
+                await _writer.UpdateBakersWithPendingChange(effectiveTime, baker => ApplyPendingChange(baker, resultBuilder));
 
-            importState.NextPendingBakerChangeTime = await _writer.GetMinPendingChangeTime();
-            _logger.Information("NextPendingBakerChangeTime set to {value}", importState.NextPendingBakerChangeTime);
+                importState.NextPendingBakerChangeTime = await _writer.GetMinPendingChangeTime();
+                _logger.Information("NextPendingBakerChangeTime set to {value}", importState.NextPendingBakerChangeTime);
+            }
+        } else {
+            // Starting from protocol version 7 and onwards stake changes are immediate, so we apply all of them in the first block of P7 and this is a no-op for future blocks.
+            await _writer.UpdateBakersWithPendingChange(DateTimeOffset.MaxValue, baker => ApplyPendingChange(baker, resultBuilder));
         }
     }
 

backend/Application/Api/GraphQL/Import/BakerWriter.cs

@@ -164,6 +164,33 @@ public async Task AddBakerTransactionRelations(IEnumerable<BakerTransactionRelat
         await context.SaveChangesAsync();
     }
 
+    /// <summary>
+    /// Removes delegator information tracked for an account.
+    /// Throws for accounts with no delegation information.
+    /// </summary>
+    public async Task RemoveDelegator(DelegatorId delegatorId) {
+        using var counter = _metrics.MeasureDuration(nameof(BakerWriter), nameof(RemoveDelegator));
+        await using var context = await _dbContextFactory.CreateDbContextAsync();
+        var account = await context.Accounts.SingleAsync(x => x.Id == (long) delegatorId.Id.Index);
+        if (account.Delegation == null) throw new InvalidOperationException("Trying to remove delegator, but account is not delegating.");
+        // Update the delegation counter on the target.
+        switch (account.Delegation.DelegationTarget) {
+            case PassiveDelegationTarget passiveTarget:
+                var passive = await context.PassiveDelegations.SingleAsync();
+                passive.DelegatorCount -= 1;
+                break;
+            case BakerDelegationTarget target:
+                var baker = await context.Bakers.SingleAsync(baker => baker.BakerId == target.BakerId);
+                var activeState = baker.State as ActiveBakerState ?? throw new InvalidOperationException("Trying to remove delegator targeting a baker pool, but the baker state is not active.");
+                var pool = activeState.Pool ?? throw new InvalidOperationException("Trying to remove delegator targeting a baker pool, but the baker state had no pool information.");
+                pool.DelegatorCount -= 1;
+                break;
+        };
+        // Delete the delegation information
+        account.Delegation = null;
+        await context.SaveChangesAsync();
+    }
+
     public async Task UpdateDelegatedStake()
     {
         using var counter = _metrics.MeasureDuration(nameof(BakerWriter), nameof(UpdateDelegatedStake));