From 8c2d7d7c1aedb95ee67875cff853df1e674a8f57 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 22 Apr 2024 17:12:59 -0700 Subject: [PATCH] Fixes #236 - Restore not working corectly came up with a solution to forcefully release the lock on the sqlite database file. ensured all contexts are being disposed properly. --- .../Binner.Common/IO/BackupProvider.MySql.cs | 2 +- .../IO/BackupProvider.Postgresql.cs | 2 +- .../IO/BackupProvider.SqlServer.cs | 4 +- .../Binner.Common/IO/BackupProvider.Sqlite.cs | 62 +++++++++++++++++-- .../Binner.Common/IO/BackupProvider.cs | 13 +++- .../Binner.Common/Services/AccountService.cs | 6 +- .../Binner.Common/Services/AdminService.cs | 2 +- .../Services/AuthenticationService.cs | 12 ++-- .../Services/IntegrationApiFactory.cs | 4 +- .../Services/IntegrationService.cs | 2 +- .../Binner.Common/Services/PrintService.cs | 2 +- .../Binner.Common/Services/ProjectService.cs | 10 +-- .../Binner.Common/Services/UserService.cs | 4 +- 13 files changed, 94 insertions(+), 31 deletions(-) diff --git a/Binner/Library/Binner.Common/IO/BackupProvider.MySql.cs b/Binner/Library/Binner.Common/IO/BackupProvider.MySql.cs index 958ca6c3..f5526917 100644 --- a/Binner/Library/Binner.Common/IO/BackupProvider.MySql.cs +++ b/Binner/Library/Binner.Common/IO/BackupProvider.MySql.cs @@ -67,7 +67,7 @@ private async Task RestoreMySqlAsync(DbInfo dbInfo) }; // drop the database if it exists - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); await using var conn = context.Database.GetDbConnection(); await using var dropCmd = conn.CreateCommand(); await conn.OpenAsync(); diff --git a/Binner/Library/Binner.Common/IO/BackupProvider.Postgresql.cs b/Binner/Library/Binner.Common/IO/BackupProvider.Postgresql.cs index 1a70c7f6..39053342 100644 --- a/Binner/Library/Binner.Common/IO/BackupProvider.Postgresql.cs +++ b/Binner/Library/Binner.Common/IO/BackupProvider.Postgresql.cs @@ -66,7 +66,7 @@ private async Task RestorePostgresqlAsync(DbInfo dbInfo) }; // drop the database if it exists - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); await using var conn = context.Database.GetDbConnection(); await conn.OpenAsync(); await using var cmdDrop = conn.CreateCommand(); diff --git a/Binner/Library/Binner.Common/IO/BackupProvider.SqlServer.cs b/Binner/Library/Binner.Common/IO/BackupProvider.SqlServer.cs index e5649a41..1fb0e557 100644 --- a/Binner/Library/Binner.Common/IO/BackupProvider.SqlServer.cs +++ b/Binner/Library/Binner.Common/IO/BackupProvider.SqlServer.cs @@ -17,7 +17,7 @@ private async Task BackupSqlServerAsync() }; var dbName = builder["Database"]; - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); await using var conn = context.Database.GetDbConnection(); await using var cmd = conn.CreateCommand(); cmd.CommandText = $"BACKUP DATABASE [{dbName}] TO DISK = '{filename}' WITH FORMAT"; @@ -54,7 +54,7 @@ private async Task RestoreSqlServerAsync(DbInfo dbInfo) }; var dbName = builder["Database"]; - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); await using var conn = context.Database.GetDbConnection(); await using var cmd = conn.CreateCommand(); cmd.CommandText = $@"USE [master]; diff --git a/Binner/Library/Binner.Common/IO/BackupProvider.Sqlite.cs b/Binner/Library/Binner.Common/IO/BackupProvider.Sqlite.cs index 4e684d49..e10f5bc3 100644 --- a/Binner/Library/Binner.Common/IO/BackupProvider.Sqlite.cs +++ b/Binner/Library/Binner.Common/IO/BackupProvider.Sqlite.cs @@ -1,4 +1,9 @@ -using System.IO; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using System; +using System.IO; +using System.Threading; using System.Threading.Tasks; namespace Binner.Common.IO @@ -21,9 +26,58 @@ private async Task BackupSqliteAsync() private async Task RestoreSqliteAsync(DbInfo dbInfo) { var filename = _configuration.ProviderConfiguration["Filename"] ?? throw new BinnerConfigurationException("Error: no Filename specified in StorageProviderConfiguration"); - await using var fileRef = File.Open(filename, FileMode.Create, FileAccess.Write, FileShare.ReadWrite); - dbInfo!.Database!.Position = 0; - await dbInfo.Database.CopyToAsync(fileRef); + + // get a connection so we can use the handle to forcefully close the Sqlite database + await using var context = await _contextFactory.CreateDbContextAsync(); + var conn = context.Database.GetDbConnection() as SqliteConnection; + conn.Open(); + // forcefully close the sqlite database using it's handle + var result = SQLitePCL.raw.sqlite3_close_v2(conn.Handle); + conn.Handle.Close(); + conn.Handle.Dispose(); + + // required GC collection + try + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + } + catch (Exception ex) + { + _logger.LogError($"Failed to garbage collect while closing existing Sqlite database at '{filename}' for restore operation."); + } + + var deleteSuccess = false; + var attempts = 1; + while ((attempts < 10) && (!deleteSuccess)) + { + try + { + Thread.Sleep(attempts * 100); + File.Delete(filename); + _logger.LogInformation($"Deleted existing Sqlite database at '{filename}' for restore operation."); + deleteSuccess = true; + } + catch (IOException e) // delete only throws this on locking + { + _logger.LogError($"Failed to delete existing Sqlite database at '{filename}' for restore operation. Retrying {attempts} of 10..."); + attempts++; + } + } + + if (deleteSuccess) + { + await using var fileRef = File.Open(filename, FileMode.Create, FileAccess.Write, FileShare.None); + //dbInfo!.Database!.Position = 0; + dbInfo.Database.Seek(0, SeekOrigin.Begin); + await dbInfo.Database.CopyToAsync(fileRef); + _logger.LogInformation($"Restored Sqlite database at '{filename}'."); + } + else + { + _logger.LogError($"Failed to overwrite existing Sqlite database at '{filename}' for restore operation."); + throw new InvalidOperationException($"Unable to overwrite the current Sqlite database '{filename}'!"); + } } } } diff --git a/Binner/Library/Binner.Common/IO/BackupProvider.cs b/Binner/Library/Binner.Common/IO/BackupProvider.cs index 85a4f66b..504a39ce 100644 --- a/Binner/Library/Binner.Common/IO/BackupProvider.cs +++ b/Binner/Library/Binner.Common/IO/BackupProvider.cs @@ -68,7 +68,14 @@ public async Task RestoreAsync(UploadFile backupFile) { case "sqlite": case "binner": - await RestoreSqliteAsync(dbInfo); + try + { + await RestoreSqliteAsync(dbInfo); + } + catch (ObjectDisposedException) + { + // expected, unavoidable + } await ProcessFileOperationsAsync(dbInfo); return; case "sqlserver": @@ -261,7 +268,7 @@ public class DbInfo { public BackupInfo? BackupInfo { get; set; } public MemoryStream? Database { get; set; } - public List FileOperations { get; set; } = new (); + public List FileOperations { get; set; } = new(); } public class FileOperation @@ -275,5 +282,7 @@ public FileOperation(string destinationFilename, byte[] data) DestinationFilename = destinationFilename; Data = data; } + + public override string ToString() => DestinationFilename; } } diff --git a/Binner/Library/Binner.Common/Services/AccountService.cs b/Binner/Library/Binner.Common/Services/AccountService.cs index 92a20aca..ae2b7f75 100644 --- a/Binner/Library/Binner.Common/Services/AccountService.cs +++ b/Binner/Library/Binner.Common/Services/AccountService.cs @@ -41,7 +41,7 @@ public AccountService(IDbContextFactory contextFactory, IMapper m public async Task GetAccountAsync() { var userContext = _requestContext.GetUserContext(); - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var entity = await GetAccountQueryable(context) .Where(x => x.UserId == userContext.UserId && x.OrganizationId == userContext.OrganizationId) .AsSplitQuery() @@ -60,7 +60,7 @@ public async Task GetAccountAsync() public async Task UpdateAccountAsync(Account account) { var userContext = _requestContext.GetUserContext(); - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var entity = await GetAccountQueryable(context) .Where(x => x.UserId == userContext.UserId) .AsSplitQuery() @@ -114,7 +114,7 @@ public async Task UpdateAccountAsync(Account account) public async Task UploadProfileImageAsync(MemoryStream stream, string originalFilename, string contentType, long length) { var userContext = _requestContext.GetUserContext(); - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var entity = await GetAccountQueryable(context) .Where(x => x.UserId == userContext.UserId && x.OrganizationId == userContext.OrganizationId) .AsSplitQuery() diff --git a/Binner/Library/Binner.Common/Services/AdminService.cs b/Binner/Library/Binner.Common/Services/AdminService.cs index c2da525c..0ad094c6 100644 --- a/Binner/Library/Binner.Common/Services/AdminService.cs +++ b/Binner/Library/Binner.Common/Services/AdminService.cs @@ -40,7 +40,7 @@ public async Task GetSystemInfoAsync() { var model = new SystemInfoResponse(); var userContext = _requestContext.GetUserContext(); - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); Assembly? entryAssembly = null; try { diff --git a/Binner/Library/Binner.Common/Services/AuthenticationService.cs b/Binner/Library/Binner.Common/Services/AuthenticationService.cs index 3bee3364..be736661 100644 --- a/Binner/Library/Binner.Common/Services/AuthenticationService.cs +++ b/Binner/Library/Binner.Common/Services/AuthenticationService.cs @@ -44,7 +44,7 @@ public async Task AuthenticateAsync(AuthenticationReques { if (model == null) throw new ArgumentNullException(nameof(model)); - using var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); using var transaction = await context.Database.BeginTransactionAsync(System.Data.IsolationLevel.Serializable); try { @@ -154,7 +154,7 @@ private async Task CreateAuthenticationLoginAsync(Binner public async Task RefreshTokenAsync(string token) { if (string.IsNullOrEmpty(token)) throw new ArgumentNullException(nameof(token)); - using var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); // todo: seems to be causing deadlocks, need to investigate //using var transaction = await context.Database.BeginTransactionAsync(System.Data.IsolationLevel.Serializable); @@ -254,7 +254,7 @@ public async Task RevokeTokenAsync(string token) public async Task GetUserAsync(int userId) { if (userId == 0) throw new ArgumentNullException(nameof(userId)); - using var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var userContext = await context.Users.FirstOrDefaultAsync(x => x.UserId == userId); return userContext != null ? Map(userContext) : null; } @@ -351,7 +351,7 @@ public async Task SendPasswordResetRequest(PasswordRec { if (request == null) throw new ArgumentNullException(nameof(request)); - using var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var user = await context.Users .FirstOrDefaultAsync(x => x.EmailAddress == request.EmailAddress); if (user == null) @@ -395,7 +395,7 @@ public async Task SendPasswordResetRequest(PasswordRec public async Task ValidatePasswordResetTokenAsync(ConfirmPasswordRecoveryRequest request) { if (request == null) throw new ArgumentNullException(nameof(request)); - using var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var user = await context.Users .FirstOrDefaultAsync(x => x.EmailAddress == request.EmailAddress); if (user == null) @@ -435,7 +435,7 @@ public async Task ValidatePasswordResetTokenAsync(Conf public async Task ResetPasswordUsingTokenAsync(PasswordRecoverySetNewPasswordRequest request) { if (request == null) throw new ArgumentNullException(nameof(request)); - using var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var user = await context.Users .FirstOrDefaultAsync(x => x.EmailAddress == request.EmailAddress); if (user == null) diff --git a/Binner/Library/Binner.Common/Services/IntegrationApiFactory.cs b/Binner/Library/Binner.Common/Services/IntegrationApiFactory.cs index e3f8ffaa..9fab2dc9 100644 --- a/Binner/Library/Binner.Common/Services/IntegrationApiFactory.cs +++ b/Binner/Library/Binner.Common/Services/IntegrationApiFactory.cs @@ -100,7 +100,7 @@ public async Task CreateAsync(int userId) #pragma warning restore CS1998 { // create a db context - //using var context = await _contextFactory.CreateDbContextAsync(); + //await using var context = await _contextFactory.CreateDbContextAsync(); /*var userIntegrationConfiguration = await context.UserIntegrationConfigurations .Where(x => x.UserId.Equals(userId)) .FirstOrDefaultAsync() @@ -221,7 +221,7 @@ public async Task CreateAsync(Type apiType, int userId) #pragma warning restore CS1998 { // create a db context - //using var context = await _contextFactory.CreateDbContextAsync(); + //await using var context = await _contextFactory.CreateDbContextAsync(); /*var userIntegrationConfiguration = await context.UserIntegrationConfigurations .Where(x => x.UserId.Equals(userId)) .FirstOrDefaultAsync() diff --git a/Binner/Library/Binner.Common/Services/IntegrationService.cs b/Binner/Library/Binner.Common/Services/IntegrationService.cs index 588a9f89..884804c5 100644 --- a/Binner/Library/Binner.Common/Services/IntegrationService.cs +++ b/Binner/Library/Binner.Common/Services/IntegrationService.cs @@ -35,7 +35,7 @@ public async Task TestApiAsync(TestApiRequest request) #pragma warning restore CS1998 { // create a db context - //using var context = await _contextFactory.CreateDbContextAsync(); + //await using var context = await _contextFactory.CreateDbContextAsync(); /*var userIntegrationConfiguration = await context.UserIntegrationConfigurations .Where(x => x.UserId.Equals(userId)) .FirstOrDefaultAsync() diff --git a/Binner/Library/Binner.Common/Services/PrintService.cs b/Binner/Library/Binner.Common/Services/PrintService.cs index a0508eed..651a0710 100644 --- a/Binner/Library/Binner.Common/Services/PrintService.cs +++ b/Binner/Library/Binner.Common/Services/PrintService.cs @@ -104,7 +104,7 @@ public async Task DeleteLabelTemplateAsync(LabelTemplate model) public async Task> GetLabelTemplatesAsync() { var user = _requestContext.GetUserContext(); - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var entities = await context.LabelTemplates .Where(x => x.OrganizationId == user.OrganizationId) .ToListAsync(); diff --git a/Binner/Library/Binner.Common/Services/ProjectService.cs b/Binner/Library/Binner.Common/Services/ProjectService.cs index 6fc2a1f0..f1b4e8fe 100644 --- a/Binner/Library/Binner.Common/Services/ProjectService.cs +++ b/Binner/Library/Binner.Common/Services/ProjectService.cs @@ -317,7 +317,7 @@ public async Task> GetProduceHistoryAsync(Get var user = _requestContext.GetUserContext(); if (user == null) throw new ArgumentNullException(nameof(user)); - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var pageRecords = (request.Page - 1) * request.Results; var entities = await context.ProjectProduceHistory .Include(x => x.ProjectPcbProduceHistory) @@ -347,7 +347,7 @@ public async Task UpdateProduceHistoryAsync(ProjectProduc var user = _requestContext.GetUserContext(); if (user == null) throw new ArgumentNullException(nameof(user)); - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var entity = await context.ProjectProduceHistory .Include(x => x.ProjectPcbProduceHistory) @@ -379,7 +379,7 @@ public async Task DeleteProduceHistoryAsync(ProjectProduceHistory request) var user = _requestContext.GetUserContext(); if (user == null) throw new ArgumentNullException(nameof(user)); - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var entity = await context.ProjectProduceHistory .Include(x => x.Project) .ThenInclude(x => x.ProjectPartAssignments) @@ -406,7 +406,7 @@ public async Task DeletePcbProduceHistoryAsync(ProjectPcbProduceHistory re var user = _requestContext.GetUserContext(); if (user == null) throw new ArgumentNullException(nameof(user)); - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var entity = await context.ProjectPcbProduceHistory .Include(x => x.Pcb) .Include(x => x.ProjectProduceHistory) @@ -478,7 +478,7 @@ public async Task ProducePcbAsync(ProduceBomPcbRequest request) { // get all the parts in the project var user = _requestContext.GetUserContext(); - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var numberOfPcbsProduced = request.Quantity; var project = await GetProjectAsync(request.ProjectId); diff --git a/Binner/Library/Binner.Common/Services/UserService.cs b/Binner/Library/Binner.Common/Services/UserService.cs index 31536800..cc5f7479 100644 --- a/Binner/Library/Binner.Common/Services/UserService.cs +++ b/Binner/Library/Binner.Common/Services/UserService.cs @@ -60,7 +60,7 @@ public async Task DeleteUserAsync(int userId) if (userId == 1) throw new SecurityException($"The root admin user cannot be deleted."); var userContext = _requestContext.GetUserContext(); - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var entity = await GetUserQueryable(context, userContext) .Where(x => x.UserId == userId) .AsSplitQuery() @@ -83,7 +83,7 @@ public async Task DeleteUserAsync(int userId) public async Task ValidateUserImageToken(string token) { - var context = await _contextFactory.CreateDbContextAsync(); + await using var context = await _contextFactory.CreateDbContextAsync(); var userToken = await context.UserTokens .Include(x => x.User) .FirstOrDefaultAsync(x =>