Skip to content

Commit

Permalink
Merge pull request #22 from dynamicweb/mss/16537
Browse files Browse the repository at this point in the history 
Mss/16537
  • Loading branch information
@frederik5480
frederik5480 authored May 15, 2024
2 parents 3fd7b2b + 12e753f commit 97fb0ee
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 116 deletions.
67 changes: 19 additions & 48 deletions src/CSVDestinationWriter.cs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
using Dynamicweb.Core;
using Dynamicweb.DataIntegration.Integration;
using Dynamicweb.DataIntegration.Integration.Interfaces;
using Dynamicweb.DataIntegration.ProviderHelpers;
using System;
using System.Collections.Generic;
using System.Globalization;
Expand Down Expand Up @@ -92,71 +91,43 @@ public virtual void Write(Dictionary<string, object> row)

private string GetStringToWrite(Dictionary<string, object> row, ColumnMapping columnMapping)
{
string stringToWrite;
if (columnMapping.SourceColumn == null && columnMapping.HasScriptWithValue)
if (columnMapping.HasScriptWithValue)
{
stringToWrite = quoteChar + columnMapping.GetScriptValue() + quoteChar + fieldDelimiter;
return quoteChar + columnMapping.GetScriptValue() + quoteChar + fieldDelimiter;
}
else if (row.ContainsKey(columnMapping.SourceColumn.Name))
else if (row.TryGetValue(columnMapping.SourceColumn?.Name ?? "", out object rowValue))
{
if (row[columnMapping.SourceColumn.Name] == DBNull.Value)
if (columnMapping.SourceColumn.Type == typeof(DateTime))
{
if (columnMapping.HasScriptWithValue)
if (DateTime.TryParse(columnMapping.ConvertInputValueToOutputValue(rowValue)?.ToString(), out var theDateTime))
{
stringToWrite = quoteChar + columnMapping.GetScriptValue() + quoteChar + fieldDelimiter;
if (cultureInfo != null)
{
return quoteChar + theDateTime.ToString("dd-MM-yyyy HH:mm:ss:fff", cultureInfo) + quoteChar + fieldDelimiter;
}
else
{
return quoteChar + theDateTime.ToString("dd-MM-yyyy HH:mm:ss:fff", CultureInfo.InvariantCulture) + quoteChar + fieldDelimiter;
}
}
else
{
stringToWrite = "NULL" + fieldDelimiter;
return quoteChar + DateTime.MinValue.ToString("dd-MM-yyyy HH:mm:ss:fff", CultureInfo.InvariantCulture) + quoteChar + fieldDelimiter;
}
}
if (rowValue == DBNull.Value)
{
return "NULL" + fieldDelimiter;
}
else
{
if (columnMapping.SourceColumn.Type == typeof(string))
{
stringToWrite = row[columnMapping.SourceColumn.Name].ToString().Replace(quoteChar.ToString(CultureInfo.CurrentCulture),
quoteChar.ToString(CultureInfo.CurrentCulture) + quoteChar.ToString(CultureInfo.CurrentCulture));
}
else if (columnMapping.SourceColumn.Type == typeof(DateTime))
{
stringToWrite = DateTime.Parse(row[columnMapping.SourceColumn.Name].ToString()).ToString("dd-MM-yyyy HH:mm:ss:fff", CultureInfo.InvariantCulture);
}
else if (cultureInfo != null && (columnMapping.SourceColumn.Type == typeof(int) ||
columnMapping.SourceColumn.Type == typeof(decimal) ||
columnMapping.SourceColumn.Type == typeof(double) ||
columnMapping.SourceColumn.Type == typeof(float)))
{
stringToWrite = ValueFormatter.GetFormattedValue(row[columnMapping.SourceColumn.Name], cultureInfo, columnMapping.ScriptType, columnMapping.ScriptValue);
}
else
{
stringToWrite = row[columnMapping.SourceColumn.Name].ToString();
}
switch (columnMapping.ScriptType)
{
case ScriptType.Append:
stringToWrite = quoteChar + stringToWrite + columnMapping.ScriptValue + quoteChar + fieldDelimiter;
break;
case ScriptType.Constant:
stringToWrite = quoteChar + columnMapping.ScriptValue + quoteChar + fieldDelimiter;
break;
case ScriptType.Prepend:
stringToWrite = quoteChar + columnMapping.ScriptValue + stringToWrite + quoteChar + fieldDelimiter;
break;
case ScriptType.None:
stringToWrite = quoteChar + stringToWrite + quoteChar + fieldDelimiter;
break;
case ScriptType.NewGuid:
stringToWrite = quoteChar + columnMapping.GetScriptValue() + quoteChar + fieldDelimiter;
break;
}
return quoteChar + string.Format(cultureInfo, "{0}", columnMapping.ConvertInputValueToOutputValue(rowValue)) + quoteChar + fieldDelimiter ?? "NULL" + fieldDelimiter;
}
}
else
{
throw new Exception(BaseDestinationWriter.GetRowValueNotFoundMessage(row, columnMapping.SourceColumn.Table.Name, columnMapping.SourceColumn.Name));
}
return stringToWrite;
}


Expand Down
143 changes: 76 additions & 67 deletions src/CSVProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,12 +111,12 @@ public string DestinationQuoteCharacter
get { return _quoteChar; }
set { _quoteChar = value; }
}

[AddInParameter("Destination encoding"), AddInParameterEditor(typeof(DropDownParameterEditor), "none=true"), AddInParameterGroup("Destination")]
public string DestinationEncoding { get; set; }

[AddInParameter("Number format culture"), AddInParameterEditor(typeof(DropDownParameterEditor), "none=true"), AddInParameterGroup("Destination")]
public string ExportCultureInfo { get; set; }
[AddInParameter("Destination format culture"), AddInParameterEditor(typeof(DropDownParameterEditor), "none=false"), AddInParameterGroup("Destination")]
public string ExportCultureInfo { get; set; } = CultureInfo.CurrentCulture.Name;

[AddInParameter("Source decimal separator"), AddInParameterEditor(typeof(DropDownParameterEditor), "none=false"), AddInParameterGroup("Source")]
public string SourceDecimalSeparator
Expand Down Expand Up @@ -162,13 +162,13 @@ public override Schema GetOriginalSourceSchema()
}
else
{
Dictionary<string, CsvReader> csvReaders = new Dictionary<string, CsvReader>();
Dictionary<string, CsvReader> csvReaders = new Dictionary<string, CsvReader>();
var config = new CsvConfiguration(CultureInfo.CurrentCulture)
{
Comment = '¤',
Delimiter = _fieldDelimiter + "",
Delimiter = _fieldDelimiter + "",
HasHeaderRecord = SourceFirstRowContainsColumnNames,
Escape = '\\',
Escape = '\\',
TrimOptions = TrimOptions.None,
DetectColumnCountChanges = true
};
Expand Down Expand Up @@ -199,7 +199,7 @@ public override Schema GetOriginalSourceSchema()

public override Schema GetOriginalDestinationSchema()
{
return GetSchema();
return _schema = new Schema();
}

public override void OverwriteSourceSchemaToOriginal()
Expand All @@ -209,9 +209,16 @@ public override void OverwriteSourceSchemaToOriginal()

public override void OverwriteDestinationSchemaToOriginal()
{
_schema = new Schema();
}

public override Schema GetSchema()
Schema IDestination.GetSchema()
{
_schema ??= new Schema();
return _schema;
}

Schema ISource.GetSchema()
{
_schema ??= GetOriginalSourceSchema();
return _schema;
Expand Down Expand Up @@ -307,7 +314,7 @@ private void GetSchemaForTableFromFile(Schema schema, Dictionary<string, CsvRead
}
}

public new virtual void SaveAsXml(XmlTextWriter xmlTextWriter)
void ISource.SaveAsXml(XmlTextWriter xmlTextWriter)
{
xmlTextWriter.WriteStartElement("SourceFieldDelimiter");
xmlTextWriter.WriteCData(_fieldDelimiter.ToString(CultureInfo.CurrentCulture));
Expand All @@ -316,17 +323,32 @@ private void GetSchemaForTableFromFile(Schema schema, Dictionary<string, CsvRead
xmlTextWriter.WriteCData(_quoteChar?.ToString(CultureInfo.CurrentCulture));
xmlTextWriter.WriteEndElement();
xmlTextWriter.WriteElementString("SourceFirstRowContainsColumnNames", SourceFirstRowContainsColumnNames.ToString(CultureInfo.CurrentCulture));
xmlTextWriter.WriteElementString("SourcePath", _path);
xmlTextWriter.WriteElementString("SourceFile", SourceFile);
xmlTextWriter.WriteElementString("SourceDecimalSeparator", _sourceDecimalSeparator);
xmlTextWriter.WriteElementString("DeleteSourceFiles", DeleteSourceFiles.ToString());
xmlTextWriter.WriteElementString("IgnoreDefectiveRows", IgnoreDefectiveRows.ToString(CultureInfo.CurrentCulture));
(this as ISource).GetSchema().SaveAsXml(xmlTextWriter);
}

void IDestination.SaveAsXml(XmlTextWriter xmlTextWriter)
{
xmlTextWriter.WriteStartElement("SourceFieldDelimiter");
xmlTextWriter.WriteCData(_fieldDelimiter.ToString(CultureInfo.CurrentCulture));
xmlTextWriter.WriteEndElement();
xmlTextWriter.WriteStartElement("QuoteChar");
xmlTextWriter.WriteCData(_quoteChar?.ToString(CultureInfo.CurrentCulture));
xmlTextWriter.WriteEndElement();
xmlTextWriter.WriteElementString("DestinationFirstRowContainsColumnNames", DestinationFirstRowContainsColumnNames.ToString(CultureInfo.CurrentCulture));
xmlTextWriter.WriteElementString("SourcePath", _path);
xmlTextWriter.WriteElementString("SourceFile", SourceFile);
if (!string.IsNullOrEmpty(DestinationEncoding))
{
xmlTextWriter.WriteElementString("DestinationEncoding", DestinationEncoding);
}
xmlTextWriter.WriteElementString("SourceDecimalSeparator", _sourceDecimalSeparator);
xmlTextWriter.WriteElementString("ExportCultureInfo", ExportCultureInfo);
xmlTextWriter.WriteElementString("DeleteSourceFiles", DeleteSourceFiles.ToString());
xmlTextWriter.WriteElementString("IncludeTimestampInFileName", IncludeTimestampInFileName.ToString(CultureInfo.CurrentCulture));
xmlTextWriter.WriteElementString("IgnoreDefectiveRows", IgnoreDefectiveRows.ToString(CultureInfo.CurrentCulture));
GetSchema().SaveAsXml(xmlTextWriter);
(this as IDestination).GetSchema().SaveAsXml(xmlTextWriter);
}

public CsvProvider() { }
Expand Down Expand Up @@ -397,7 +419,7 @@ public CsvProvider(XmlNode xmlNode)
{
DestinationFirstRowContainsColumnNames = node.FirstChild.Value == "True";
}
break;
break;
case "DestinationEncoding":
if (node.HasChildNodes)
{
Expand Down Expand Up @@ -470,7 +492,7 @@ public CsvProvider(string path)
}

return new CsvSourceReader(filePath, mapping, SourceFirstRowContainsColumnNames,
Convert.ToChar(_fieldDelimiter, CultureInfo.CurrentCulture), !string.IsNullOrEmpty(_quoteChar) ? Convert.ToChar(_quoteChar, CultureInfo.CurrentCulture) : char.MinValue,
Convert.ToChar(_fieldDelimiter, CultureInfo.CurrentCulture), !string.IsNullOrEmpty(_quoteChar) ? Convert.ToChar(_quoteChar, CultureInfo.CurrentCulture) : char.MinValue,
decimalSeparator, autoDetectDecimalSeparator, IgnoreDefectiveRows, Logger, this);
}

Expand All @@ -492,10 +514,10 @@ public override string Serialize()
root.Add(CreateParameterNode(GetType(), "Input Field delimiter", _fieldDelimiter.ToString(CultureInfo.CurrentCulture)));
root.Add(CreateParameterNode(GetType(), "Output Field delimiter", _fieldDelimiter.ToString(CultureInfo.CurrentCulture)));
root.Add(CreateParameterNode(GetType(), "Input string delimiter", SourceQuoteCharacter.ToString(CultureInfo.CurrentCulture)));
root.Add(CreateParameterNode(GetType(), "Output string delimiter", DestinationQuoteCharacter.ToString(CultureInfo.CurrentCulture)));
root.Add(CreateParameterNode(GetType(), "Output string delimiter", DestinationQuoteCharacter.ToString(CultureInfo.CurrentCulture)));
root.Add(CreateParameterNode(GetType(), "Destination encoding", DestinationEncoding));
root.Add(CreateParameterNode(GetType(), "Source decimal separator", _sourceDecimalSeparator));
root.Add(CreateParameterNode(GetType(), "Number format culture", ExportCultureInfo));
root.Add(CreateParameterNode(GetType(), "Destination format culture", ExportCultureInfo));
root.Add(CreateParameterNode(GetType(), "Delete source files", DeleteSourceFiles.ToString()));
root.Add(CreateParameterNode(GetType(), "Include timestamp in filename", IncludeTimestampInFileName.ToString(CultureInfo.CurrentCulture)));
root.Add(CreateParameterNode(GetType(), "Ignore defective rows", IgnoreDefectiveRows.ToString(CultureInfo.CurrentCulture)));
Expand All @@ -506,23 +528,25 @@ public override void UpdateSourceSettings(ISource source)
{
CsvProvider newProvider = (CsvProvider)source;
SourceFirstRowContainsColumnNames = newProvider.SourceFirstRowContainsColumnNames;
DestinationFirstRowContainsColumnNames = newProvider.DestinationFirstRowContainsColumnNames;
_path = newProvider._path;
_fieldDelimiter = newProvider._fieldDelimiter;
_quoteChar = newProvider._quoteChar;
DestinationEncoding = newProvider.DestinationEncoding;
_quoteChar = newProvider._quoteChar;
SourceDecimalSeparator = newProvider.SourceDecimalSeparator;
ExportCultureInfo = newProvider.ExportCultureInfo;
DeleteSourceFiles = newProvider.DeleteSourceFiles;
IncludeTimestampInFileName = newProvider.IncludeTimestampInFileName;
IgnoreDefectiveRows = newProvider.IgnoreDefectiveRows;
SourceFile = newProvider.SourceFile;
}

public override void UpdateDestinationSettings(IDestination destination)
{
ISource newProvider = (ISource)destination;
UpdateSourceSettings(newProvider);
CsvProvider newProvider = (CsvProvider)destination;
DestinationFirstRowContainsColumnNames = newProvider.DestinationFirstRowContainsColumnNames;
_path = newProvider._path;
_fieldDelimiter = newProvider._fieldDelimiter;
_quoteChar = newProvider._quoteChar;
DestinationEncoding = newProvider.DestinationEncoding;
ExportCultureInfo = newProvider.ExportCultureInfo;
IncludeTimestampInFileName = newProvider.IncludeTimestampInFileName;
}

public override bool RunJob(Job job)
Expand All @@ -533,29 +557,29 @@ public override bool RunJob(Job job)
{
CultureInfo ci = GetCultureInfo();

if (_destinationWriters == null)
if (_destinationWriters == null)
{
_destinationWriters = new List<CsvDestinationWriter>();
foreach (var mapping in job.Mappings)
{
_destinationWriters = new List<CsvDestinationWriter>();
foreach (var mapping in job.Mappings)
if (mapping.Active && mapping.GetColumnMappings().Count > 0)
{
if (mapping.Active && mapping.GetColumnMappings().Count > 0)
{
_destinationWriters.Add(new CsvDestinationWriter((WorkingDirectory.CombinePaths(_path)).Replace("\\", "/"), mapping, DestinationFirstRowContainsColumnNames, Convert.ToChar(_fieldDelimiter, CultureInfo.CurrentCulture), Convert.ToChar(_quoteChar, CultureInfo.CurrentCulture), GetEncoding(DestinationEncoding), ci, IncludeTimestampInFileName));
}
_destinationWriters.Add(new CsvDestinationWriter((WorkingDirectory.CombinePaths(_path)).Replace("\\", "/"), mapping, DestinationFirstRowContainsColumnNames, Convert.ToChar(_fieldDelimiter, CultureInfo.CurrentCulture), Convert.ToChar(_quoteChar, CultureInfo.CurrentCulture), GetEncoding(DestinationEncoding), ci, IncludeTimestampInFileName));
}
}
}

foreach (var writer in _destinationWriters)
foreach (var writer in _destinationWriters)
{
using (ISourceReader sourceReader = writer.Mapping.Source.GetReader(writer.Mapping))
{
using (ISourceReader sourceReader = writer.Mapping.Source.GetReader(writer.Mapping))
while (!sourceReader.IsDone())
{
while (!sourceReader.IsDone())
{
sourceRow = sourceReader.GetNext();
ProcessInputRow(writer.Mapping, sourceRow);
writer.Write(sourceRow);
}
sourceRow = sourceReader.GetNext();
ProcessInputRow(writer.Mapping, sourceRow);
writer.Write(sourceRow);
}
}

}
sourceRow = null;
Expand All @@ -570,18 +594,18 @@ public override bool RunJob(Job job)
if (sourceRow != null)
msg += GetFailedSourceRowMessage(sourceRow);

Logger?.Log("Job failed: " + msg);
return false;
}
finally
Logger?.Log("Job failed: " + msg);
return false;
}
finally
{
foreach (var writer in _destinationWriters)
{
foreach (var writer in _destinationWriters)
{
writer.Close();
}
writer.Close();
}
return true;
}
return true;
}

private Encoding GetEncoding(string encoding)
{
Expand All @@ -600,21 +624,7 @@ private Encoding GetEncoding(string encoding)

private CultureInfo GetCultureInfo()
{
CultureInfo result = null;

if (!string.IsNullOrEmpty(ExportCultureInfo))
{
try
{
result = CultureInfo.GetCultureInfo(ExportCultureInfo);
}
catch (Exception ex)
{
Logger?.Log(string.Format("Error getting culture: {0}.", ex.Message));
}
}

return result;
return CultureInfo.GetCultureInfo(ExportCultureInfo) ?? CultureInfo.CurrentCulture;
}

private string GetSourceFile()
Expand Down Expand Up @@ -714,10 +724,9 @@ public void WriteToSourceFile(string InputXML)
new(".", "."),
new(",", ",")
},
"Number format culture" => Ecommerce.Services.Countries.GetCountries()
.Where(c => !string.IsNullOrEmpty(c.CultureInfo))
.DistinctBy(c => c.CultureInfo)
.Select(c => new ParameterOption(c.Code2, c.CultureInfo)),
"Destination format culture" => CultureInfo.GetCultures(CultureTypes.SpecificCultures)
.Where(obj => !string.IsNullOrEmpty(obj.Name))
.Select(c => new ParameterOption(c.DisplayName, c.Name)),
_ => new List<ParameterOption>()
{
new("Unicode (UTF-8)", "UTF8"),
Expand Down
2 changes: 1 addition & 1 deletion src/Dynamicweb.DataIntegration.Providers.CsvProvider.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Version>10.0.9</Version>
<Version>10.0.10</Version>
<AssemblyVersion>1.0.0.0</AssemblyVersion>
<Title>CSV Provider</Title>
<Description>CSV Provider</Description>
Expand Down

0 comments on commit 97fb0ee

Please sign in to comment.