From 987de4f1236b8828c1a993bcddd551f22acb9cfc Mon Sep 17 00:00:00 2001 From: MatthiasSort Date: Mon, 19 Feb 2024 13:49:37 +0100 Subject: [PATCH] Removed use of folder=... in the add-in options. --- ...Integration.Providers.ExcelProvider.csproj | 2 +- src/ExcelProvider.cs | 621 +++++++++--------- 2 files changed, 311 insertions(+), 312 deletions(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.ExcelProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.ExcelProvider.csproj index 90ea042..505bee7 100644 --- a/src/Dynamicweb.DataIntegration.Providers.ExcelProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.ExcelProvider.csproj @@ -1,6 +1,6 @@  - 10.0.7 + 10.0.8 1.0.0 Excel Provider Excel Provider diff --git a/src/ExcelProvider.cs b/src/ExcelProvider.cs index 695e6b5..7ff29c6 100644 --- a/src/ExcelProvider.cs +++ b/src/ExcelProvider.cs @@ -12,435 +12,434 @@ using System.Xml; using System.Xml.Linq; -namespace Dynamicweb.DataIntegration.Providers.ExcelProvider +namespace Dynamicweb.DataIntegration.Providers.ExcelProvider; + +[AddInName("Dynamicweb.DataIntegration.Providers.Provider"), AddInLabel("Excel Provider"), AddInDescription("Excel Provider"), AddInIgnore(false)] +public class ExcelProvider : BaseProvider, ISource, IDestination, IParameterOptions { - [AddInName("Dynamicweb.DataIntegration.Providers.Provider"), AddInLabel("Excel Provider"), AddInDescription("Excel Provider"), AddInIgnore(false)] - public class ExcelProvider : BaseProvider, ISource, IDestination, IParameterOptions - { - internal static string GetOLEDB12ConnectionString(string fileName) => $"Provider=Microsoft.ACE.OLEDB.12.0;Data Source={fileName};Extended Properties=\"Excel 12.0;HDR=Yes;IMEX=1\""; - internal static string GetOLEDB4ConnectionString(string fileName) => $"Provider=Microsoft.Jet.OLEDB.4.0;Data Source={fileName};Extended Properties=\"Excel 8.0;HDR=Yes;IMEX=1\""; + internal static string GetOLEDB12ConnectionString(string fileName) => $"Provider=Microsoft.ACE.OLEDB.12.0;Data Source={fileName};Extended Properties=\"Excel 12.0;HDR=Yes;IMEX=1\""; + internal static string GetOLEDB4ConnectionString(string fileName) => $"Provider=Microsoft.Jet.OLEDB.4.0;Data Source={fileName};Extended Properties=\"Excel 8.0;HDR=Yes;IMEX=1\""; - private const string ExcelExtension = ".xlsx"; - //path should point to a folder - if it doesn't, write will fail. + private const string ExcelExtension = ".xlsx"; + //path should point to a folder - if it doesn't, write will fail. - [AddInParameter("Source file"), AddInParameterEditor(typeof(FileManagerEditor), "folder=/Files/;required"), AddInParameterGroup("Source")] - public string SourceFile + [AddInParameter("Source file"), AddInParameterEditor(typeof(FileManagerEditor), "required"), AddInParameterGroup("Source")] + public string SourceFile + { + get { - get - { - return _sourceFileName; - } - set - { - _sourceFileName = value; - } + return _sourceFileName; } + set + { + _sourceFileName = value; + } + } - [AddInParameter("Destination file"), AddInParameterEditor(typeof(TextParameterEditor), $"append={ExcelExtension};required"), AddInParameterGroup("Destination")] - public string DestinationFile + [AddInParameter("Destination file"), AddInParameterEditor(typeof(TextParameterEditor), $"append={ExcelExtension};required"), AddInParameterGroup("Destination")] + public string DestinationFile + { + get { - get - { - return Path.GetFileNameWithoutExtension(_destinationFileName); - } - set - { - _destinationFileName = Path.GetFileNameWithoutExtension(value); - } + return Path.GetFileNameWithoutExtension(_destinationFileName); + } + set + { + _destinationFileName = Path.GetFileNameWithoutExtension(value); } - private string _sourceFileName; - private string _destinationFileName; - private string _destinationFolder = "/Files"; + } + private string _sourceFileName; + private string _destinationFileName; + private string _destinationFolder = "/Files"; - [AddInParameter("Destination folder"), AddInParameterEditor(typeof(FolderSelectEditor), "folder=/Files/"), AddInParameterGroup("Destination")] - public string DestinationFolder + [AddInParameter("Destination folder"), AddInParameterEditor(typeof(FolderSelectEditor), ""), AddInParameterGroup("Destination")] + public string DestinationFolder + { + get { - get - { - return _destinationFolder; - } - set - { - _destinationFolder = value; - } + return _destinationFolder; + } + set + { + _destinationFolder = value; } + } + + private Schema schema; - private Schema schema; + private ExcelDestinationWriter destinationWriter; - private ExcelDestinationWriter destinationWriter; + public override Schema GetOriginalDestinationSchema() + { + return GetSchema(); + } - public override Schema GetOriginalDestinationSchema() + public override bool SchemaIsEditable + { + get { - return GetSchema(); + return true; } + } - public override bool SchemaIsEditable + public override Schema GetOriginalSourceSchema() + { + Schema result = new Schema(); + + string currentPath = SourceFile; + if (!string.IsNullOrEmpty(workingDirectory)) { - get - { - return true; - } + currentPath = workingDirectory.CombinePaths(SourceFile); } - public override Schema GetOriginalSourceSchema() + var sourceFilePath = GetSourceFilePath(); + if (File.Exists(sourceFilePath)) { - Schema result = new Schema(); - - string currentPath = SourceFile; - if (!string.IsNullOrEmpty(workingDirectory)) - { - currentPath = workingDirectory.CombinePaths(SourceFile); - } - - var sourceFilePath = GetSourceFilePath(); - if (File.Exists(sourceFilePath)) + try { - try + if (currentPath.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) || + currentPath.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || + currentPath.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase)) { - if (currentPath.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) || - currentPath.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || - currentPath.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase)) - { - Dictionary excelReaders = new Dictionary - { - { sourceFilePath, new ExcelReader(sourceFilePath) } - }; - GetSchemaForTableFromFile(result, excelReaders); - } - else + Dictionary excelReaders = new Dictionary { - Logger?.Error("File is not an Excel file"); - } + { sourceFilePath, new ExcelReader(sourceFilePath) } + }; + GetSchemaForTableFromFile(result, excelReaders); } - catch (Exception ex) + else { - Logger?.Error(string.Format("GetOriginalSourceSchema error reading file: {0} message: {1} stack: {2}", sourceFilePath, ex.Message, ex.StackTrace)); + Logger?.Error("File is not an Excel file"); } } - else + catch (Exception ex) { - Logger?.Error($"Source file {sourceFilePath} does not exist"); + Logger?.Error(string.Format("GetOriginalSourceSchema error reading file: {0} message: {1} stack: {2}", sourceFilePath, ex.Message, ex.StackTrace)); } - - return result; } - - private string workingDirectory; - public override string WorkingDirectory + else { - get - { - return workingDirectory; - } - set { workingDirectory = value.Replace("\\", "/"); } + Logger?.Error($"Source file {sourceFilePath} does not exist"); } - private string GetSourceFilePath() + return result; + } + + private string workingDirectory; + public override string WorkingDirectory + { + get { - string srcFilePath = string.Empty; + return workingDirectory; + } + set { workingDirectory = value.Replace("\\", "/"); } + } + + private string GetSourceFilePath() + { + string srcFilePath = string.Empty; - if (!string.IsNullOrEmpty(SourceFile)) + if (!string.IsNullOrEmpty(SourceFile)) + { + if (SourceFile.StartsWith("..")) { - if (SourceFile.StartsWith("..")) - { - srcFilePath = (this.WorkingDirectory.CombinePaths(SourceFile.TrimStart(new char[] { '.' })).Replace("\\", "/")); - } - else - { - srcFilePath = this.WorkingDirectory.CombinePaths(FilesFolderName, SourceFile).Replace("\\", "/"); - } + srcFilePath = (this.WorkingDirectory.CombinePaths(SourceFile.TrimStart(new char[] { '.' })).Replace("\\", "/")); + } + else + { + srcFilePath = this.WorkingDirectory.CombinePaths(FilesFolderName, SourceFile).Replace("\\", "/"); } - return srcFilePath; } + return srcFilePath; + } - public override void UpdateSourceSettings(ISource source) - { - ExcelProvider newProvider = (ExcelProvider)source; - SourceFile = newProvider.SourceFile; - _destinationFolder = newProvider._destinationFolder; - } + public override void UpdateSourceSettings(ISource source) + { + ExcelProvider newProvider = (ExcelProvider)source; + SourceFile = newProvider.SourceFile; + _destinationFolder = newProvider._destinationFolder; + } - public override string Serialize() - { - XDocument document = new XDocument(new XDeclaration("1.0", "utf-8", string.Empty)); + public override string Serialize() + { + XDocument document = new XDocument(new XDeclaration("1.0", "utf-8", string.Empty)); - XElement root = new XElement("Parameters"); - document.Add(root); + XElement root = new XElement("Parameters"); + document.Add(root); - root.Add(CreateParameterNode(GetType(), "Source file", SourceFile)); - root.Add(CreateParameterNode(GetType(), "Destination file", DestinationFile)); - root.Add(CreateParameterNode(GetType(), "Destination folder", DestinationFolder)); + root.Add(CreateParameterNode(GetType(), "Source file", SourceFile)); + root.Add(CreateParameterNode(GetType(), "Destination file", DestinationFile)); + root.Add(CreateParameterNode(GetType(), "Destination folder", DestinationFolder)); - return document.ToString(); - } + return document.ToString(); + } - public new virtual void SaveAsXml(XmlTextWriter xmlTextWriter) - { - xmlTextWriter.WriteElementString("SourcePath", SourceFile); - xmlTextWriter.WriteElementString("DestinationFile", DestinationFile); - xmlTextWriter.WriteElementString("DestinationFolder", DestinationFolder); - xmlTextWriter.WriteElementString("WorkingDirectory", WorkingDirectory); - GetSchema().SaveAsXml(xmlTextWriter); - } + public new virtual void SaveAsXml(XmlTextWriter xmlTextWriter) + { + xmlTextWriter.WriteElementString("SourcePath", SourceFile); + xmlTextWriter.WriteElementString("DestinationFile", DestinationFile); + xmlTextWriter.WriteElementString("DestinationFolder", DestinationFolder); + xmlTextWriter.WriteElementString("WorkingDirectory", WorkingDirectory); + GetSchema().SaveAsXml(xmlTextWriter); + } - public new ISourceReader GetReader(Mapping mapping) + public new ISourceReader GetReader(Mapping mapping) + { + if (SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || + SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) || + SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase)) { - if (SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || - SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) || - SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase)) + if (!string.IsNullOrEmpty(WorkingDirectory)) { - if (!string.IsNullOrEmpty(WorkingDirectory)) - { - return new ExcelSourceReader(GetSourceFilePath(), mapping, this); - } - else - { - return new ExcelSourceReader(SourceFile, mapping, this); - } + return new ExcelSourceReader(GetSourceFilePath(), mapping, this); } else { - Logger?.Error("The file is not a Excel file"); - return null; + return new ExcelSourceReader(SourceFile, mapping, this); } } - - public override void Close() + else { - + Logger?.Error("The file is not a Excel file"); + return null; } + } - public override void UpdateDestinationSettings(IDestination destination) - { - ISource newProvider = (ISource)destination; - UpdateSourceSettings(newProvider); - } + public override void Close() + { + + } - public override bool RunJob(Job job) + public override void UpdateDestinationSettings(IDestination destination) + { + ISource newProvider = (ISource)destination; + UpdateSourceSettings(newProvider); + } + + public override bool RunJob(Job job) + { + ReplaceMappingConditionalsWithValuesFromRequest(job); + Dictionary sourceRow = null; + try { - ReplaceMappingConditionalsWithValuesFromRequest(job); - Dictionary sourceRow = null; - try + if (destinationWriter == null) { - if (destinationWriter == null) + if (!string.IsNullOrEmpty(WorkingDirectory)) { - if (!string.IsNullOrEmpty(WorkingDirectory)) - { - destinationWriter = new ExcelDestinationWriter(workingDirectory.CombinePaths(_destinationFolder), $"{Path.GetFileNameWithoutExtension(DestinationFile)}{ExcelExtension}", job.Mappings, Logger); - } - else - { - destinationWriter = new ExcelDestinationWriter($"{Path.GetFileNameWithoutExtension(SourceFile)}{ExcelExtension}", "", job.Mappings, Logger); - } + destinationWriter = new ExcelDestinationWriter(workingDirectory.CombinePaths(_destinationFolder), $"{Path.GetFileNameWithoutExtension(DestinationFile)}{ExcelExtension}", job.Mappings, Logger); } - foreach (var mapping in destinationWriter.Mappings) + else { - destinationWriter.currentMapping = mapping; - using (ISourceReader sourceReader = mapping.Source.GetReader(mapping)) - { - while (!sourceReader.IsDone()) - { - sourceRow = sourceReader.GetNext(); - ProcessInputRow(mapping, sourceRow); - destinationWriter.Write(sourceRow); - } - destinationWriter.AddTableToSet(); - } + destinationWriter = new ExcelDestinationWriter($"{Path.GetFileNameWithoutExtension(SourceFile)}{ExcelExtension}", "", job.Mappings, Logger); } - destinationWriter.GenerateExcel(); - sourceRow = null; } - catch (Exception ex) + foreach (var mapping in destinationWriter.Mappings) { - string msg = ex.Message; - string stackTrace = ex.StackTrace; - - Logger?.Error($"Error: {msg.Replace(System.Environment.NewLine, " ")} Stack: {stackTrace.Replace(System.Environment.NewLine, " ")}", ex); - LogManager.System.GetLogger(LogCategory.Application, "Dataintegration").Error($"{GetType().Name} error: {msg} Stack: {stackTrace}", ex); - if (sourceRow != null) + destinationWriter.currentMapping = mapping; + using (ISourceReader sourceReader = mapping.Source.GetReader(mapping)) { - msg += GetFailedSourceRowMessage(sourceRow); + while (!sourceReader.IsDone()) + { + sourceRow = sourceReader.GetNext(); + ProcessInputRow(mapping, sourceRow); + destinationWriter.Write(sourceRow); + } + destinationWriter.AddTableToSet(); } - Logger.Log("Job failed " + msg); - return false; } - finally + destinationWriter.GenerateExcel(); + sourceRow = null; + } + catch (Exception ex) + { + string msg = ex.Message; + string stackTrace = ex.StackTrace; + + Logger?.Error($"Error: {msg.Replace(System.Environment.NewLine, " ")} Stack: {stackTrace.Replace(System.Environment.NewLine, " ")}", ex); + LogManager.System.GetLogger(LogCategory.Application, "Dataintegration").Error($"{GetType().Name} error: {msg} Stack: {stackTrace}", ex); + if (sourceRow != null) { - sourceRow = null; + msg += GetFailedSourceRowMessage(sourceRow); } - return true; + Logger.Log("Job failed " + msg); + return false; } + finally + { + sourceRow = null; + } + return true; + } - private void GetSchemaForTableFromFile(Schema schema, Dictionary excelReaders) + private void GetSchemaForTableFromFile(Schema schema, Dictionary excelReaders) + { + foreach (var reader in excelReaders) { - foreach (var reader in excelReaders) + foreach (DataTable dt in reader.Value.ExcelSet.Tables) { - foreach (DataTable dt in reader.Value.ExcelSet.Tables) + Table excelTable = schema.AddTable(dt.TableName); + try { - Table excelTable = schema.AddTable(dt.TableName); + int columnCount; try { - int columnCount; - try - { - columnCount = dt.Columns.Count; - } - catch (System.ArgumentException) - { - columnCount = dt.Columns.Count; - } - foreach (System.Data.DataColumn c in dt.Columns) - { - excelTable.AddColumn(new Column(c.ColumnName, c.DataType, excelTable)); - } - + columnCount = dt.Columns.Count; } - catch (System.ArgumentException ae) + catch (System.ArgumentException) { - string msg = string.Format("Error reading Excel file: {0} ", reader.Key); - Exception ex = new Exception(msg, ae); - throw ex; + columnCount = dt.Columns.Count; + } + foreach (System.Data.DataColumn c in dt.Columns) + { + excelTable.AddColumn(new Column(c.ColumnName, c.DataType, excelTable)); } - } + } + catch (System.ArgumentException ae) + { + string msg = string.Format("Error reading Excel file: {0} ", reader.Key); + Exception ex = new Exception(msg, ae); + throw ex; + } } + } + } - public override Schema GetSchema() + public override Schema GetSchema() + { + if (schema == null) { - if (schema == null) - { - schema = GetOriginalSourceSchema(); - } - return schema; + schema = GetOriginalSourceSchema(); } + return schema; + } - public ExcelProvider() + public ExcelProvider() + { + if (string.IsNullOrEmpty(FilesFolderName)) { - if (string.IsNullOrEmpty(FilesFolderName)) - { - FilesFolderName = "Files"; - } + FilesFolderName = "Files"; } + } - public ExcelProvider(XmlNode xmlNode) + public ExcelProvider(XmlNode xmlNode) + { + foreach (XmlNode node in xmlNode.ChildNodes) { - foreach (XmlNode node in xmlNode.ChildNodes) + switch (node.Name) { - switch (node.Name) - { - case "Schema": - schema = new Schema(node); - break; - case "SourcePath": - if (node.HasChildNodes) - { - SourceFile = node.FirstChild.Value; - } - break; - case "DestinationFile": - if (node.HasChildNodes) - { - DestinationFile = node.FirstChild.Value; - } - break; - case "DestinationFolder": - if (node.HasChildNodes) - { - DestinationFolder = node.FirstChild.Value; - } - break; - case "WorkingDirectory": - if (node.HasChildNodes) - { - WorkingDirectory = node.FirstChild.Value; - } - break; + case "Schema": + schema = new Schema(node); + break; + case "SourcePath": + if (node.HasChildNodes) + { + SourceFile = node.FirstChild.Value; + } + break; + case "DestinationFile": + if (node.HasChildNodes) + { + DestinationFile = node.FirstChild.Value; + } + break; + case "DestinationFolder": + if (node.HasChildNodes) + { + DestinationFolder = node.FirstChild.Value; + } + break; + case "WorkingDirectory": + if (node.HasChildNodes) + { + WorkingDirectory = node.FirstChild.Value; + } + break; - } } } - internal ExcelProvider(Dictionary excelReaders, Schema schema, ExcelDestinationWriter writer) - { - this.schema = schema; - destinationWriter = writer; - } + } + internal ExcelProvider(Dictionary excelReaders, Schema schema, ExcelDestinationWriter writer) + { + this.schema = schema; + destinationWriter = writer; + } - public ExcelProvider(string path) - { - SourceFile = path; - } + public ExcelProvider(string path) + { + SourceFile = path; + } - public override void OverwriteSourceSchemaToOriginal() - { - schema = GetOriginalSourceSchema(); - } + public override void OverwriteSourceSchemaToOriginal() + { + schema = GetOriginalSourceSchema(); + } + + public override void OverwriteDestinationSchemaToOriginal() + { + } - public override void OverwriteDestinationSchemaToOriginal() + public override string ValidateDestinationSettings() + { + string extension = Path.GetFileNameWithoutExtension(DestinationFile); + if (!string.Equals(extension, DestinationFile, StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(extension) && !(extension.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || extension.EndsWith(".xls", StringComparison.OrdinalIgnoreCase))) { + return "File has to end with .xlsx or .xls"; } + return ""; + } - public override string ValidateDestinationSettings() + public override string ValidateSourceSettings() + { + if (SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || + SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) || + SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase)) { - string extension = Path.GetFileNameWithoutExtension(DestinationFile); - if (!string.Equals(extension, DestinationFile, StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(extension) && !(extension.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || extension.EndsWith(".xls", StringComparison.OrdinalIgnoreCase))) + string filename = GetSourceFilePath(); + if (!File.Exists(filename)) { - return "File has to end with .xlsx or .xls"; + return "Excel file \"" + SourceFile + "\" does not exist. WorkingDirectory - " + WorkingDirectory; } - return ""; - } - public override string ValidateSourceSettings() - { - if (SourceFile.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) || - SourceFile.EndsWith(".xls", StringComparison.OrdinalIgnoreCase) || - SourceFile.EndsWith(".xlsm", StringComparison.OrdinalIgnoreCase)) + string strConn = GetOLEDB12ConnectionString(filename); + + if (filename.EndsWith(".xls")) + { + strConn = GetOLEDB4ConnectionString(filename); + } + + using (OleDbConnection conn = new OleDbConnection(strConn)) { - string filename = GetSourceFilePath(); - if (!File.Exists(filename)) + try { - return "Excel file \"" + SourceFile + "\" does not exist. WorkingDirectory - " + WorkingDirectory; + conn.Open(); } - - string strConn = GetOLEDB12ConnectionString(filename); - - if (filename.EndsWith(".xls")) + catch (Exception ex) { - strConn = GetOLEDB4ConnectionString(filename); + return string.Format("Could not open source file: {0} message: {1} stack: {2}", filename, ex.Message, ex.StackTrace); } - using (OleDbConnection conn = new OleDbConnection(strConn)) - { - try - { - conn.Open(); - } - catch (Exception ex) - { - return string.Format("Could not open source file: {0} message: {1} stack: {2}", filename, ex.Message, ex.StackTrace); - } + DataTable schemaTable = conn.GetOleDbSchemaTable(OleDbSchemaGuid.Tables, new object[] { null, null, null, "TABLE" }); - DataTable schemaTable = conn.GetOleDbSchemaTable(OleDbSchemaGuid.Tables, new object[] { null, null, null, "TABLE" }); + foreach (DataRow schemaRow in schemaTable.Rows) + { + string sheet = schemaRow["TABLE_NAME"].ToString(); - foreach (DataRow schemaRow in schemaTable.Rows) + if (sheet.Contains(" ")) { - string sheet = schemaRow["TABLE_NAME"].ToString(); - - if (sheet.Contains(" ")) - { - return sheet + " contains whitespaces"; - } + return sheet + " contains whitespaces"; } } } - else - { - return "The file is not an Excel file"; - } - return null; } - - public IEnumerable GetParameterOptions(string parameterName) => new List(); + else + { + return "The file is not an Excel file"; + } + return null; } + + public IEnumerable GetParameterOptions(string parameterName) => new List(); }