From 88d57b877d5386b51c77f4b3a76b0afa4668d116 Mon Sep 17 00:00:00 2001 From: Dmitriy Benyuk Date: Thu, 16 Nov 2023 10:54:22 +0200 Subject: [PATCH] Log: How many record were imported --- ...taIntegration.Providers.SqlProvider.csproj | 4 +- src/SQLDestinationWriter.cs | 115 +++--------------- src/SQLProvider.cs | 8 +- 3 files changed, 22 insertions(+), 105 deletions(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.SqlProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.SqlProvider.csproj index 4c3f27c..cfec729 100644 --- a/src/Dynamicweb.DataIntegration.Providers.SqlProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.SqlProvider.csproj @@ -1,6 +1,6 @@  - 10.0.8 + 10.0.9 1.0.0.0 SQL Provider SQL Provider @@ -23,7 +23,7 @@ snupkg - + diff --git a/src/SQLDestinationWriter.cs b/src/SQLDestinationWriter.cs index 5d227cc..5febfa6 100644 --- a/src/SQLDestinationWriter.cs +++ b/src/SQLDestinationWriter.cs @@ -1,5 +1,4 @@ -using Dynamicweb.Core; -using Dynamicweb.DataIntegration.Integration; +using Dynamicweb.DataIntegration.Integration; using Dynamicweb.DataIntegration.Integration.Interfaces; using Dynamicweb.DataIntegration.ProviderHelpers; using Dynamicweb.Logging; @@ -8,7 +7,6 @@ using System.Data; using System.Data.SqlClient; using System.Linq; -using System.Text; namespace Dynamicweb.DataIntegration.Providers.SqlProvider { @@ -238,11 +236,21 @@ public SqlDestinationWriter(Mapping mapping, SqlCommand mockSqlCommand, bool rem /// /// Where condition to filter data for deletion public virtual void DeleteExcessFromMainTable(string extraConditions) + { + DeleteRowsNotInSourceFromMainTable(extraConditions); + } + + /// + /// Deletes rows not present in the import source + /// + /// Where condition to filter data for deletion + public virtual long DeleteRowsNotInSourceFromMainTable(string extraConditions) { if (removeMissingAfterImport || removeMissingAfterImportDestinationTablesOnly) { - DeleteExcessFromMainTable(Mapping, extraConditions, SqlCommand, tempTablePrefix, removeMissingAfterImportDestinationTablesOnly); + return DeleteExcessFromMainTable(SqlCommand, Mapping, extraConditions, tempTablePrefix, removeMissingAfterImportDestinationTablesOnly); } + return 0; } /// @@ -273,104 +281,9 @@ public virtual void FinishWriting() } } - /// - /// Move data to main table - /// - /// Transaction - internal void MoveDataToMainTable(SqlTransaction sqlTransaction) - { - MoveDataToMainTable(sqlTransaction, false); - } - - /// - /// Move data to main table - /// - /// Transaction - /// Update only - private void MoveDataToMainTable(SqlTransaction sqlTransaction, bool updateOnly) - { - MoveDataToMainTable(sqlTransaction, updateOnly, false); - } - - /// - /// Move data to main table - /// - /// Transaction - /// Update only - /// Insert only - private void MoveDataToMainTable(SqlTransaction sqlTransaction, bool updateOnly, bool insertOnly) + protected internal int MoveDataToMainTable(SqlTransaction sqlTransaction) { - SqlCommand.Transaction = sqlTransaction; - List insertColumns = new List(); - //Get columnList for current Table - try - { - string sqlConditions = ""; - string firstKey = ""; - var columnMappings = Mapping.GetColumnMappings().Where(cm => cm.Active).DistinctBy(obj => obj.DestinationColumn.Name); - bool isPrimaryKeyColumnExists = columnMappings.IsKeyColumnExists(); - - foreach (ColumnMapping columnMapping in columnMappings) - { - SqlColumn column = (SqlColumn)columnMapping.DestinationColumn; - if (column.IsKeyColumn(columnMappings) || (!isPrimaryKeyColumnExists && !columnMapping.ScriptValueForInsert)) - { - sqlConditions = sqlConditions + "[" + Mapping.DestinationTable.SqlSchema + "].[" + - Mapping.DestinationTable.Name + "].[" + columnMapping.DestinationColumn.Name + "]=[" + - Mapping.DestinationTable.SqlSchema + "].[" + - Mapping.DestinationTable.Name + tempTablePrefix + "].[" + columnMapping.DestinationColumn.Name + "] and "; - if (firstKey == "") - firstKey = columnMapping.DestinationColumn.Name; - } - } - sqlConditions = sqlConditions.Substring(0, sqlConditions.Length - 4); - - string selectColumns = ""; - string updateColumns = ""; - foreach (var columnMapping in columnMappings) - { - insertColumns.Add("[" + columnMapping.DestinationColumn.Name + "]"); - selectColumns = selectColumns + "[" + Mapping.DestinationTable.SqlSchema + "].[" + Mapping.DestinationTable.Name + tempTablePrefix + "].[" + columnMapping.DestinationColumn.Name + "], "; - if (!((SqlColumn)columnMapping.DestinationColumn).IsIdentity && !((SqlColumn)columnMapping.DestinationColumn).IsKeyColumn(columnMappings) && !columnMapping.ScriptValueForInsert) - updateColumns = updateColumns + "[" + columnMapping.DestinationColumn.Name + "]=[" + Mapping.DestinationTable.SqlSchema + "].[" + columnMapping.DestinationColumn.Table.Name + tempTablePrefix + "].[" + columnMapping.DestinationColumn.Name + "], "; - } - - string sqlUpdateInsert = ""; - if (!string.IsNullOrEmpty(updateColumns) && !insertOnly) - { - updateColumns = updateColumns.Substring(0, updateColumns.Length - 2); - sqlUpdateInsert = sqlUpdateInsert + "update [" + Mapping.DestinationTable.SqlSchema + "].[" + Mapping.DestinationTable.Name + "] set " + updateColumns + " from [" + Mapping.DestinationTable.SqlSchema + "].[" + Mapping.DestinationTable.Name + tempTablePrefix + "] where " + sqlConditions + ";"; - } - if (!string.IsNullOrEmpty(selectColumns)) - { - selectColumns = selectColumns.Substring(0, selectColumns.Length - 2); - if (!updateOnly) - { - if (HasIdentity(Mapping)) - { - sqlUpdateInsert = sqlUpdateInsert + "set identity_insert [" + Mapping.DestinationTable.SqlSchema + "].[" + - Mapping.DestinationTable.Name + "] ON;"; - } - sqlUpdateInsert = sqlUpdateInsert + " insert into [" + Mapping.DestinationTable.SqlSchema + "].[" + Mapping.DestinationTable.Name + "] (" + string.Join(",", insertColumns) + ") (" + - "select " + selectColumns + " from [" + Mapping.DestinationTable.SqlSchema + "].[" + Mapping.DestinationTable.Name + tempTablePrefix + "] left outer join [" + Mapping.DestinationTable.SqlSchema + "].[" + Mapping.DestinationTable.Name + "] on " + sqlConditions + " where [" + Mapping.DestinationTable.SqlSchema + "].[" + Mapping.DestinationTable.Name + "].[" + firstKey + "] is null);"; - if (HasIdentity(Mapping)) - { - sqlUpdateInsert = sqlUpdateInsert + "set identity_insert [" + Mapping.DestinationTable.SqlSchema + "].[" + - Mapping.DestinationTable.Name + "] OFF;"; - } - } - } - SqlCommand.CommandText = sqlUpdateInsert; - if (SqlCommand.Connection.State != ConnectionState.Open) - { - SqlCommand.Connection.Open(); - } - SqlCommand.ExecuteNonQuery(); - } - catch (Exception ex) - { - throw GetMoveDataToMainTableException(ex, SqlCommand, Mapping, tempTablePrefix, insertColumns); - } + return MoveDataToMainTable(SqlCommand, Mapping, sqlTransaction, tempTablePrefix); } #region IDisposable Implementation diff --git a/src/SQLProvider.cs b/src/SQLProvider.cs index 4eb94d9..f07bcdf 100644 --- a/src/SQLProvider.cs +++ b/src/SQLProvider.cs @@ -493,7 +493,9 @@ public override bool RunJob(Job job) if (writer.RowsToWriteCount > 0) { System.Diagnostics.Debug.WriteLine(DateTime.Now + ": Moving data to main table: " + writer.Mapping.DestinationTable.Name); - writer.MoveDataToMainTable(Transaction); + int rowsAffected = writer.MoveDataToMainTable(Transaction); + if (rowsAffected > 0) + Logger.Log($"The number of rows affected: {rowsAffected} in the {writer.Mapping.DestinationTable.Name} table"); } else { @@ -505,8 +507,10 @@ public override bool RunJob(Job job) if (writer.RowsToWriteCount > 0) { System.Diagnostics.Debug.WriteLine(DateTime.Now + ": Removing excess data from table: " + writer.Mapping.DestinationTable.Name); - writer.DeleteExcessFromMainTable(""); + long rowsAffected = writer.DeleteRowsNotInSourceFromMainTable(""); System.Diagnostics.Debug.WriteLine(DateTime.Now + ": excess data Removed from table: " + writer.Mapping.DestinationTable.Name); + if (rowsAffected > 0) + Logger.Log($"The number of deleted rows: {rowsAffected} for the destination {writer.Mapping.DestinationTable.Name} table mapping"); } } CommitTransaction();