From f0184ee4de7f4cbdc254ba700de180c6342a66e4 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Tue, 14 May 2024 14:05:34 +0200 Subject: [PATCH 1/5] Changed DestinationWriter.GetStringToWrite to no longer converts values itself bur uses general function in columnMapping. --- src/CSVDestinationWriter.cs | 58 ++----------------- ...taIntegration.Providers.CsvProvider.csproj | 2 +- 2 files changed, 7 insertions(+), 53 deletions(-) diff --git a/src/CSVDestinationWriter.cs b/src/CSVDestinationWriter.cs index b00d3b7..182e8f7 100644 --- a/src/CSVDestinationWriter.cs +++ b/src/CSVDestinationWriter.cs @@ -92,71 +92,25 @@ public virtual void Write(Dictionary row) private string GetStringToWrite(Dictionary row, ColumnMapping columnMapping) { - string stringToWrite; - if (columnMapping.SourceColumn == null && columnMapping.HasScriptWithValue) + if (columnMapping.HasScriptWithValue) { - stringToWrite = quoteChar + columnMapping.GetScriptValue() + quoteChar + fieldDelimiter; + return quoteChar + columnMapping.GetScriptValue() + quoteChar + fieldDelimiter; } - else if (row.ContainsKey(columnMapping.SourceColumn.Name)) + else if (row.TryGetValue(columnMapping.SourceColumn?.Name ?? "", out object rowValue)) { - if (row[columnMapping.SourceColumn.Name] == DBNull.Value) + if (rowValue == DBNull.Value) { - if (columnMapping.HasScriptWithValue) - { - stringToWrite = quoteChar + columnMapping.GetScriptValue() + quoteChar + fieldDelimiter; - } - else - { - stringToWrite = "NULL" + fieldDelimiter; - } + return "NULL" + fieldDelimiter; } else { - if (columnMapping.SourceColumn.Type == typeof(string)) - { - stringToWrite = row[columnMapping.SourceColumn.Name].ToString().Replace(quoteChar.ToString(CultureInfo.CurrentCulture), - quoteChar.ToString(CultureInfo.CurrentCulture) + quoteChar.ToString(CultureInfo.CurrentCulture)); - } - else if (columnMapping.SourceColumn.Type == typeof(DateTime)) - { - stringToWrite = DateTime.Parse(row[columnMapping.SourceColumn.Name].ToString()).ToString("dd-MM-yyyy HH:mm:ss:fff", CultureInfo.InvariantCulture); - } - else if (cultureInfo != null && (columnMapping.SourceColumn.Type == typeof(int) || - columnMapping.SourceColumn.Type == typeof(decimal) || - columnMapping.SourceColumn.Type == typeof(double) || - columnMapping.SourceColumn.Type == typeof(float))) - { - stringToWrite = ValueFormatter.GetFormattedValue(row[columnMapping.SourceColumn.Name], cultureInfo, columnMapping.ScriptType, columnMapping.ScriptValue); - } - else - { - stringToWrite = row[columnMapping.SourceColumn.Name].ToString(); - } - switch (columnMapping.ScriptType) - { - case ScriptType.Append: - stringToWrite = quoteChar + stringToWrite + columnMapping.ScriptValue + quoteChar + fieldDelimiter; - break; - case ScriptType.Constant: - stringToWrite = quoteChar + columnMapping.ScriptValue + quoteChar + fieldDelimiter; - break; - case ScriptType.Prepend: - stringToWrite = quoteChar + columnMapping.ScriptValue + stringToWrite + quoteChar + fieldDelimiter; - break; - case ScriptType.None: - stringToWrite = quoteChar + stringToWrite + quoteChar + fieldDelimiter; - break; - case ScriptType.NewGuid: - stringToWrite = quoteChar + columnMapping.GetScriptValue() + quoteChar + fieldDelimiter; - break; - } + return quoteChar + columnMapping.ConvertInputValueToOutputValue(rowValue)?.ToString()?.ToString(cultureInfo) + quoteChar + fieldDelimiter ?? "NULL" + fieldDelimiter; } } else { throw new Exception(BaseDestinationWriter.GetRowValueNotFoundMessage(row, columnMapping.SourceColumn.Table.Name, columnMapping.SourceColumn.Name)); } - return stringToWrite; } diff --git a/src/Dynamicweb.DataIntegration.Providers.CsvProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.CsvProvider.csproj index 6416adf..9b99746 100644 --- a/src/Dynamicweb.DataIntegration.Providers.CsvProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.CsvProvider.csproj @@ -1,6 +1,6 @@  - 10.0.9 + 10.0.10 1.0.0.0 CSV Provider CSV Provider From ae68e9b1cec8db85ecb89374d4d659f66b71f219 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Tue, 14 May 2024 14:49:10 +0200 Subject: [PATCH 2/5] Changed Number format culture to Destination format culture as we also use the culture to convert DateTimes plus added logic to convert DateTimes correctly in the Writer --- src/CSVDestinationWriter.cs | 21 +++++++++++++++++++-- src/CSVProvider.cs | 10 +++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/CSVDestinationWriter.cs b/src/CSVDestinationWriter.cs index 182e8f7..be8608b 100644 --- a/src/CSVDestinationWriter.cs +++ b/src/CSVDestinationWriter.cs @@ -1,7 +1,6 @@ using Dynamicweb.Core; using Dynamicweb.DataIntegration.Integration; using Dynamicweb.DataIntegration.Integration.Interfaces; -using Dynamicweb.DataIntegration.ProviderHelpers; using System; using System.Collections.Generic; using System.Globalization; @@ -98,9 +97,27 @@ private string GetStringToWrite(Dictionary row, ColumnMapping co } else if (row.TryGetValue(columnMapping.SourceColumn?.Name ?? "", out object rowValue)) { + if (columnMapping.SourceColumn.Type == typeof(DateTime)) + { + if (DateTime.TryParse(columnMapping.ConvertInputValueToOutputValue(rowValue)?.ToString(), out var theDateTime)) + { + if (cultureInfo != null) + { + return quoteChar + theDateTime.ToString("dd-MM-yyyy HH:mm:ss:fff", cultureInfo) + quoteChar + fieldDelimiter; + } + else + { + return quoteChar + theDateTime.ToString("dd-MM-yyyy HH:mm:ss:fff", CultureInfo.InvariantCulture) + quoteChar + fieldDelimiter; + } + } + else + { + return quoteChar + DateTime.MinValue.ToString("dd-MM-yyyy HH:mm:ss:fff", CultureInfo.InvariantCulture) + quoteChar + fieldDelimiter; + } + } if (rowValue == DBNull.Value) { - return "NULL" + fieldDelimiter; + return "NULL" + fieldDelimiter; } else { diff --git a/src/CSVProvider.cs b/src/CSVProvider.cs index b29fe67..9798e23 100644 --- a/src/CSVProvider.cs +++ b/src/CSVProvider.cs @@ -115,8 +115,8 @@ public string DestinationQuoteCharacter [AddInParameter("Destination encoding"), AddInParameterEditor(typeof(DropDownParameterEditor), "none=true"), AddInParameterGroup("Destination")] public string DestinationEncoding { get; set; } - [AddInParameter("Number format culture"), AddInParameterEditor(typeof(DropDownParameterEditor), "none=true"), AddInParameterGroup("Destination")] - public string ExportCultureInfo { get; set; } + [AddInParameter("Destination format culture"), AddInParameterEditor(typeof(DropDownParameterEditor), "none=true"), AddInParameterGroup("Destination")] + public string ExportCultureInfo { get; set; } = CultureInfo.CurrentCulture.Name; [AddInParameter("Source decimal separator"), AddInParameterEditor(typeof(DropDownParameterEditor), "none=false"), AddInParameterGroup("Source")] public string SourceDecimalSeparator @@ -495,7 +495,7 @@ public override string Serialize() root.Add(CreateParameterNode(GetType(), "Output string delimiter", DestinationQuoteCharacter.ToString(CultureInfo.CurrentCulture))); root.Add(CreateParameterNode(GetType(), "Destination encoding", DestinationEncoding)); root.Add(CreateParameterNode(GetType(), "Source decimal separator", _sourceDecimalSeparator)); - root.Add(CreateParameterNode(GetType(), "Number format culture", ExportCultureInfo)); + root.Add(CreateParameterNode(GetType(), "Destination format culture", ExportCultureInfo)); root.Add(CreateParameterNode(GetType(), "Delete source files", DeleteSourceFiles.ToString())); root.Add(CreateParameterNode(GetType(), "Include timestamp in filename", IncludeTimestampInFileName.ToString(CultureInfo.CurrentCulture))); root.Add(CreateParameterNode(GetType(), "Ignore defective rows", IgnoreDefectiveRows.ToString(CultureInfo.CurrentCulture))); @@ -600,7 +600,7 @@ private Encoding GetEncoding(string encoding) private CultureInfo GetCultureInfo() { - CultureInfo result = null; + CultureInfo result = CultureInfo.CurrentCulture; if (!string.IsNullOrEmpty(ExportCultureInfo)) { @@ -714,7 +714,7 @@ public void WriteToSourceFile(string InputXML) new(".", "."), new(",", ",") }, - "Number format culture" => Ecommerce.Services.Countries.GetCountries() + "Destination format culture" => Ecommerce.Services.Countries.GetCountries() .Where(c => !string.IsNullOrEmpty(c.CultureInfo)) .DistinctBy(c => c.CultureInfo) .Select(c => new ParameterOption(c.Code2, c.CultureInfo)), From bbf9ea38c1606390f4c7229a0769b5a269ee44b4 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Wed, 15 May 2024 07:48:59 +0200 Subject: [PATCH 3/5] Changed culture logic, so now it uses CultureInfo.GetCultures() instead of Ecommerce.Services.Countries.GetCountries() --- src/CSVProvider.cs | 91 +++++++++++++++++++--------------------------- 1 file changed, 38 insertions(+), 53 deletions(-) diff --git a/src/CSVProvider.cs b/src/CSVProvider.cs index 9798e23..aa22efa 100644 --- a/src/CSVProvider.cs +++ b/src/CSVProvider.cs @@ -111,11 +111,11 @@ public string DestinationQuoteCharacter get { return _quoteChar; } set { _quoteChar = value; } } - + [AddInParameter("Destination encoding"), AddInParameterEditor(typeof(DropDownParameterEditor), "none=true"), AddInParameterGroup("Destination")] public string DestinationEncoding { get; set; } - [AddInParameter("Destination format culture"), AddInParameterEditor(typeof(DropDownParameterEditor), "none=true"), AddInParameterGroup("Destination")] + [AddInParameter("Destination format culture"), AddInParameterEditor(typeof(DropDownParameterEditor), "none=false"), AddInParameterGroup("Destination")] public string ExportCultureInfo { get; set; } = CultureInfo.CurrentCulture.Name; [AddInParameter("Source decimal separator"), AddInParameterEditor(typeof(DropDownParameterEditor), "none=false"), AddInParameterGroup("Source")] @@ -162,13 +162,13 @@ public override Schema GetOriginalSourceSchema() } else { - Dictionary csvReaders = new Dictionary(); + Dictionary csvReaders = new Dictionary(); var config = new CsvConfiguration(CultureInfo.CurrentCulture) { Comment = '¤', - Delimiter = _fieldDelimiter + "", + Delimiter = _fieldDelimiter + "", HasHeaderRecord = SourceFirstRowContainsColumnNames, - Escape = '\\', + Escape = '\\', TrimOptions = TrimOptions.None, DetectColumnCountChanges = true }; @@ -318,7 +318,7 @@ private void GetSchemaForTableFromFile(Schema schema, Dictionary(); + foreach (var mapping in job.Mappings) { - _destinationWriters = new List(); - foreach (var mapping in job.Mappings) + if (mapping.Active && mapping.GetColumnMappings().Count > 0) { - if (mapping.Active && mapping.GetColumnMappings().Count > 0) - { - _destinationWriters.Add(new CsvDestinationWriter((WorkingDirectory.CombinePaths(_path)).Replace("\\", "/"), mapping, DestinationFirstRowContainsColumnNames, Convert.ToChar(_fieldDelimiter, CultureInfo.CurrentCulture), Convert.ToChar(_quoteChar, CultureInfo.CurrentCulture), GetEncoding(DestinationEncoding), ci, IncludeTimestampInFileName)); - } + _destinationWriters.Add(new CsvDestinationWriter((WorkingDirectory.CombinePaths(_path)).Replace("\\", "/"), mapping, DestinationFirstRowContainsColumnNames, Convert.ToChar(_fieldDelimiter, CultureInfo.CurrentCulture), Convert.ToChar(_quoteChar, CultureInfo.CurrentCulture), GetEncoding(DestinationEncoding), ci, IncludeTimestampInFileName)); } } + } - foreach (var writer in _destinationWriters) + foreach (var writer in _destinationWriters) + { + using (ISourceReader sourceReader = writer.Mapping.Source.GetReader(writer.Mapping)) { - using (ISourceReader sourceReader = writer.Mapping.Source.GetReader(writer.Mapping)) + while (!sourceReader.IsDone()) { - while (!sourceReader.IsDone()) - { - sourceRow = sourceReader.GetNext(); - ProcessInputRow(writer.Mapping, sourceRow); - writer.Write(sourceRow); - } + sourceRow = sourceReader.GetNext(); + ProcessInputRow(writer.Mapping, sourceRow); + writer.Write(sourceRow); } + } } sourceRow = null; @@ -570,18 +570,18 @@ public override bool RunJob(Job job) if (sourceRow != null) msg += GetFailedSourceRowMessage(sourceRow); - Logger?.Log("Job failed: " + msg); - return false; - } - finally + Logger?.Log("Job failed: " + msg); + return false; + } + finally + { + foreach (var writer in _destinationWriters) { - foreach (var writer in _destinationWriters) - { - writer.Close(); - } + writer.Close(); } - return true; } + return true; + } private Encoding GetEncoding(string encoding) { @@ -600,21 +600,7 @@ private Encoding GetEncoding(string encoding) private CultureInfo GetCultureInfo() { - CultureInfo result = CultureInfo.CurrentCulture; - - if (!string.IsNullOrEmpty(ExportCultureInfo)) - { - try - { - result = CultureInfo.GetCultureInfo(ExportCultureInfo); - } - catch (Exception ex) - { - Logger?.Log(string.Format("Error getting culture: {0}.", ex.Message)); - } - } - - return result; + return CultureInfo.GetCultureInfo(ExportCultureInfo) ?? CultureInfo.CurrentCulture; } private string GetSourceFile() @@ -714,10 +700,9 @@ public void WriteToSourceFile(string InputXML) new(".", "."), new(",", ",") }, - "Destination format culture" => Ecommerce.Services.Countries.GetCountries() - .Where(c => !string.IsNullOrEmpty(c.CultureInfo)) - .DistinctBy(c => c.CultureInfo) - .Select(c => new ParameterOption(c.Code2, c.CultureInfo)), + "Destination format culture" => CultureInfo.GetCultures(CultureTypes.SpecificCultures) + .Where(obj => !string.IsNullOrEmpty(obj.Name)) + .Select(c => new ParameterOption(c.DisplayName, c.Name)), _ => new List() { new("Unicode (UTF-8)", "UTF8"), From 0856da13ecd2496889d59c08abccee6d281485a0 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Wed, 15 May 2024 10:00:07 +0200 Subject: [PATCH 4/5] split up the source and destination schema, so now it does not trigger an error when you have documents open in the destination folder. added string.format in the GetStringToWrite function so it can convert numbers correctly. --- src/CSVDestinationWriter.cs | 2 +- src/CSVProvider.cs | 52 +++++++++++++++++++++++++++---------- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/CSVDestinationWriter.cs b/src/CSVDestinationWriter.cs index be8608b..84d96a3 100644 --- a/src/CSVDestinationWriter.cs +++ b/src/CSVDestinationWriter.cs @@ -121,7 +121,7 @@ private string GetStringToWrite(Dictionary row, ColumnMapping co } else { - return quoteChar + columnMapping.ConvertInputValueToOutputValue(rowValue)?.ToString()?.ToString(cultureInfo) + quoteChar + fieldDelimiter ?? "NULL" + fieldDelimiter; + return quoteChar + string.Format(cultureInfo, "{0}", columnMapping.ConvertInputValueToOutputValue(rowValue)) + quoteChar + fieldDelimiter ?? "NULL" + fieldDelimiter; } } else diff --git a/src/CSVProvider.cs b/src/CSVProvider.cs index aa22efa..a904f22 100644 --- a/src/CSVProvider.cs +++ b/src/CSVProvider.cs @@ -25,6 +25,7 @@ public class CsvProvider : BaseProvider, ISource, IDestination, IParameterOption private bool _destinationFirstRowContainsColumnNames = true; private string _path = ""; private Schema _schema; + private Schema _destinationSchema; private readonly Dictionary _csvReadersForTest; private List _destinationWriters; private readonly string _detectAutomaticallySeparator = "Detect automatically"; @@ -199,7 +200,7 @@ public override Schema GetOriginalSourceSchema() public override Schema GetOriginalDestinationSchema() { - return GetSchema(); + return _destinationSchema = new Schema(); } public override void OverwriteSourceSchemaToOriginal() @@ -209,9 +210,16 @@ public override void OverwriteSourceSchemaToOriginal() public override void OverwriteDestinationSchemaToOriginal() { + _destinationSchema = new Schema(); } - public override Schema GetSchema() + Schema IDestination.GetSchema() + { + _destinationSchema ??= new Schema(); + return _destinationSchema; + } + + Schema ISource.GetSchema() { _schema ??= GetOriginalSourceSchema(); return _schema; @@ -307,7 +315,7 @@ private void GetSchemaForTableFromFile(Schema schema, Dictionary Date: Wed, 15 May 2024 10:03:45 +0200 Subject: [PATCH 5/5] Removed _destinationSchema and used _schema instead. --- src/CSVProvider.cs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/CSVProvider.cs b/src/CSVProvider.cs index a904f22..c1daf8c 100644 --- a/src/CSVProvider.cs +++ b/src/CSVProvider.cs @@ -25,7 +25,6 @@ public class CsvProvider : BaseProvider, ISource, IDestination, IParameterOption private bool _destinationFirstRowContainsColumnNames = true; private string _path = ""; private Schema _schema; - private Schema _destinationSchema; private readonly Dictionary _csvReadersForTest; private List _destinationWriters; private readonly string _detectAutomaticallySeparator = "Detect automatically"; @@ -200,7 +199,7 @@ public override Schema GetOriginalSourceSchema() public override Schema GetOriginalDestinationSchema() { - return _destinationSchema = new Schema(); + return _schema = new Schema(); } public override void OverwriteSourceSchemaToOriginal() @@ -210,13 +209,13 @@ public override void OverwriteSourceSchemaToOriginal() public override void OverwriteDestinationSchemaToOriginal() { - _destinationSchema = new Schema(); + _schema = new Schema(); } Schema IDestination.GetSchema() { - _destinationSchema ??= new Schema(); - return _destinationSchema; + _schema ??= new Schema(); + return _schema; } Schema ISource.GetSchema() @@ -390,7 +389,6 @@ public CsvProvider(XmlNode xmlNode) break; case "Schema": _schema = new Schema(node); - _destinationSchema = new Schema(node); break; case "SourcePath": if (node.HasChildNodes)