Merge branch 'release/4.1.1'

CluedIn-io · Dec 12, 2024 · e30ae84 · e30ae84
2 parents 86dac83 + c57b861
commit e30ae84
Show file tree

Hide file tree

Showing 8 changed files with 346 additions and 8 deletions.
diff --git a/docs/4.1.1-release-notes.md b/docs/4.1.1-release-notes.md
@@ -0,0 +1,5 @@
+### Features
+- Allow user to optionally configure connection pool size.
+
+### Fixes
+- Fix issue where entity type properties were not serialized correctly
diff --git a/src/Connector.SqlServer/Connector/ISqlClient.cs b/src/Connector.SqlServer/Connector/ISqlClient.cs
@@ -5,8 +5,12 @@
 
 namespace CluedIn.Connector.SqlServer.Connector
 {
+    public record ConnectionConfigurationError(string ErrorMessage);
+
     public interface ISqlClient
     {
+        bool VerifyConnectionProperties(IReadOnlyDictionary<string, object> config, out ConnectionConfigurationError configurationError);
+
         Task<SqlConnection> BeginConnection(IReadOnlyDictionary<string, object> config);
 
         Task<DataTable> GetTableColumns(SqlConnection connection, string tableName, string schema);

diff --git a/src/Connector.SqlServer/Connector/SqlClient.cs b/src/Connector.SqlServer/Connector/SqlClient.cs
@@ -11,6 +11,7 @@ namespace CluedIn.Connector.SqlServer.Connector
     public class SqlClient : ISqlClient
     {
         private readonly int _defaultPort = 1433;
+        private readonly int _defaultConnectionPoolSize = 200;
 
         public string BuildConnectionString(IReadOnlyDictionary<string, object> config)
         {
@@ -22,17 +23,76 @@ public string BuildConnectionString(IReadOnlyDictionary<string, object> config)
                 DataSource = (string)config[SqlServerConstants.KeyName.Host],
                 InitialCatalog = (string)config[SqlServerConstants.KeyName.DatabaseName],
                 Pooling = true,
-                MaxPoolSize = 200
             };
 
-            if (config.TryGetValue(SqlServerConstants.KeyName.PortNumber, out var portEntry) && int.TryParse(portEntry.ToString(), out var port))
+            // Configure port
+            {
+                var port = _defaultPort;
+                if (config.TryGetValue(SqlServerConstants.KeyName.PortNumber, out var portEntry) &&
+                    !string.IsNullOrEmpty(portEntry.ToString()) &&
+                    int.TryParse(portEntry.ToString(), out var parsedPort))
+                {
+                    port = parsedPort;
+                }
+
                 connectionStringBuilder.DataSource = $"{connectionStringBuilder.DataSource},{port}";
-            else
-                connectionStringBuilder.DataSource = $"{connectionStringBuilder.DataSource},{_defaultPort}";
+            }
+
+            // Configure connection pool size
+            {
+                var connectionPoolSize = _defaultConnectionPoolSize;
+                if (config.TryGetValue(SqlServerConstants.KeyName.ConnectionPoolSize, out var connectionPoolSizeEntry) &&
+                    !string.IsNullOrEmpty(connectionPoolSizeEntry.ToString()) &&
+                    int.TryParse(connectionPoolSizeEntry.ToString(), out var parsedConnectionPoolSize))
+                {
+                    connectionPoolSize = parsedConnectionPoolSize;
+                }
+
+                connectionStringBuilder.MaxPoolSize = connectionPoolSize;
+            }
+
 
             return connectionStringBuilder.ToString();
         }
 
+        public bool VerifyConnectionProperties(IReadOnlyDictionary<string, object> config, out ConnectionConfigurationError configurationError)
+        {
+            if (config.TryGetValue(SqlServerConstants.KeyName.PortNumber, out var portEntry) && !string.IsNullOrEmpty(portEntry.ToString()))
+            {
+                if (!int.TryParse(portEntry.ToString(), out _))
+                {
+                    configurationError = new ConnectionConfigurationError("Port number was set, but could not be read as a number");
+                    return false;
+                }
+            }
+
+            if (config.TryGetValue(SqlServerConstants.KeyName.ConnectionPoolSize, out var connectionPoolSizeEntry) && !string.IsNullOrEmpty(connectionPoolSizeEntry.ToString()))
+            {
+                if (int.TryParse(connectionPoolSizeEntry.ToString(), out var parsedPoolSize))
+                {
+                    if (parsedPoolSize < 1)
+                    {
+                        configurationError = new ConnectionConfigurationError("Connection pool size was set to a value smaller than 1");
+                        return false;
+                    }
+
+                    if (parsedPoolSize > _defaultConnectionPoolSize)
+                    {
+                        configurationError = new ConnectionConfigurationError("Connection pool size was set to a value higher than 200");
+                        return false;
+                    }
+                }
+                else
+                {
+                    configurationError = new ConnectionConfigurationError("Connection pool size was set, but could not be read as a number");
+                    return false;
+                }
+            }
+
+            configurationError = null;
+            return true;
+        }
+
         public async Task<SqlConnection> BeginConnection(IReadOnlyDictionary<string, object> config)
         {
             var connectionString = BuildConnectionString(config);

diff --git a/src/Connector.SqlServer/Connector/SqlServerConnector.cs b/src/Connector.SqlServer/Connector/SqlServerConnector.cs
@@ -308,6 +308,11 @@ public override async Task<ConnectionVerificationResult> VerifyConnection(Execut
         {
             try
             {
+                if (!_client.VerifyConnectionProperties(configurationData, out var configurationError))
+                {
+                    return new ConnectionVerificationResult(success: false, errorMessage: configurationError.ErrorMessage);
+                }
+
                 await using var connectionAndTransaction = await _client.BeginTransaction(configurationData);
                 var connectionIsOpen = connectionAndTransaction.Connection.State == ConnectionState.Open;
                 await connectionAndTransaction.DisposeAsync();
@@ -484,7 +489,7 @@ await ExecuteWithRetryAsync(async () =>
 
         public override Task<string> GetValidMappingDestinationPropertyName(ExecutionContext executionContext, Guid connectorProviderDefinitionId, string propertyName)
         {
-            return Task.FromResult(propertyName.ToSanitizedSqlName());
+            return Task.FromResult(propertyName);
         }
 
         public override async Task RemoveContainer(ExecutionContext executionContext, IReadOnlyStreamModel streamModel)

diff --git a/src/Connector.SqlServer/SqlServerConstants.cs b/src/Connector.SqlServer/SqlServerConstants.cs
@@ -17,6 +17,7 @@ public struct KeyName
             public const string Username = "username";
             public const string Password = "password";
             public const string PortNumber = "portNumber";
+            public const string ConnectionPoolSize = "connectionPoolSize";
         }
 
         public SqlServerConstants()
@@ -103,6 +104,13 @@ public SqlServerConstants()
                     displayName = "Schema",
                     type = "input",
                     isRequired = false
+                },
+                new Control
+                {
+                    name = KeyName.ConnectionPoolSize,
+                    displayName = "Connection pool size",
+                    type = "input",
+                    isRequired = false
                 }
             }
         };

diff --git a/src/Connector.SqlServer/Utils/TableDefinitions/MainTableDefinition.cs b/src/Connector.SqlServer/Utils/TableDefinitions/MainTableDefinition.cs
@@ -1,5 +1,6 @@
 using CluedIn.Connector.SqlServer.Connector;
 using CluedIn.Core.Connectors;
+using CluedIn.Core.Data;
 using CluedIn.Core.Streams.Models;
 using Microsoft.Data.SqlClient;
 using System;
@@ -49,15 +50,21 @@ public static MainTableColumnDefinition[] GetColumnDefinitions(StreamMode stream
 
             var defaultColumnNamesHashSet = defaultColumns.Select(x => x.Name).ToHashSet();
 
+            var alreadyUsedNames = new HashSet<string>();
+
             var propertyColumns = properties
                 // We need to filter out any properties, that are contained in the default columns.
                 .Where(property => !defaultColumnNamesHashSet.Contains(property.name.ToSanitizedSqlName()))
+                .OrderBy(property => property.dataType is VocabularyKeyConnectorPropertyDataType x
+                    ? $"{x.VocabularyKey.Vocabulary.KeyPrefix}.{x.VocabularyKey.Name}"
+                    : property.name)
                 .Select(property =>
                 {
-                    var name = property.name.ToSanitizedSqlName();
+                    var nameToUse = GetNameToUse(property, alreadyUsedNames);
+
                     var sqlType = SqlColumnHelper.GetColumnType(property.dataType);
                     return new MainTableColumnDefinition(
-                        name,
+                        nameToUse,
                         sqlType,
                         input =>
                         {
@@ -82,6 +89,11 @@ public static MainTableColumnDefinition[] GetColumnDefinitions(StreamMode stream
                                 return dateTimeOffsetValue.ToString("O");
                             }
 
+                            if (propertyValue is EntityType entityTypeValue)
+                            {
+                                return entityTypeValue.ToString();
+                            }
+
                             return propertyValue;
                         },
                         CanBeNull: true);
@@ -92,6 +104,37 @@ public static MainTableColumnDefinition[] GetColumnDefinitions(StreamMode stream
             return allColumns;
         }
 
+        private static string GetNameToUse((string name, ConnectorPropertyDataType dataType) property, HashSet<string> alreadyUsedNames)
+        {
+            string rawName;
+            switch (property.dataType)
+            {
+                case VocabularyKeyConnectorPropertyDataType vocabularyKeyConnectorPropertyDataType:
+                    var vocabularyKey = vocabularyKeyConnectorPropertyDataType.VocabularyKey;
+                    rawName = $"{vocabularyKey.Vocabulary.KeyPrefix}.{vocabularyKey.Name}";
+                    break;
+                default:
+                    rawName = property.name;
+                    break;
+            }
+
+            rawName = rawName.ToSanitizedSqlName();
+
+            var number = 0;
+
+            var nameToUse = rawName;
+            while (alreadyUsedNames.Contains(nameToUse))
+            {
+                number++;
+                nameToUse = $"{rawName}_{number}";
+            }
+
+            alreadyUsedNames.Add(nameToUse);
+
+            // We need to call ToSanitizedSqlName again, in case adding numbers pushed length of name over the maximum
+            return nameToUse.ToSanitizedSqlName();
+        }
+
         public static SqlServerConnectorCommand CreateUpsertCommand(IReadOnlyStreamModel streamModel, SqlConnectorEntityData connectorEntityData, SqlName schema)
         {
             var mainTableName = TableNameUtility.GetMainTableName(streamModel, schema);

diff --git a/test/unit/Connector.SqlServer.Test/Connector/SqlClientTests.cs b/test/unit/Connector.SqlServer.Test/Connector/SqlClientTests.cs
@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using CluedIn.Connector.SqlServer.Connector;
+using FluentAssertions;
 using Xunit;
 
 namespace CluedIn.Connector.SqlServer.Unit.Tests.Connector
@@ -21,7 +22,7 @@ public void BuildConnectionString_Sets_From_Dictionary()
                 [SqlServerConstants.KeyName.Password] = "password",
                 [SqlServerConstants.KeyName.Username] = "user",
                 [SqlServerConstants.KeyName.Host] = "host",
-                [SqlServerConstants.KeyName.DatabaseName] = "database"
+                [SqlServerConstants.KeyName.DatabaseName] = "database",
             };
 
             var result = _sut.BuildConnectionString(properties);
@@ -79,5 +80,130 @@ public void BuildConnectionString_WithInvalidPort_Sets_From_Dictionary()
 
             Assert.Equal("Data Source=host,1433;Initial Catalog=database;User ID=user;Password=password;Pooling=True;Max Pool Size=200;Authentication=SqlPassword", result);
         }
+
+        [Fact]
+        public void BuildConnectionString_WithConnectionPoolSize_Sets_From_Dictionary()
+        {
+            // arrange
+            var properties = new Dictionary<string, object>
+            {
+                [SqlServerConstants.KeyName.Password] = "password",
+                [SqlServerConstants.KeyName.Username] = "user",
+                [SqlServerConstants.KeyName.Host] = "host",
+                [SqlServerConstants.KeyName.DatabaseName] = "database",
+                [SqlServerConstants.KeyName.ConnectionPoolSize] = 10,
+            };
+
+            // act
+            var result = _sut.BuildConnectionString(properties);
+
+            // assert
+            Assert.Equal("Data Source=host,1433;Initial Catalog=database;User ID=user;Password=password;Pooling=True;Max Pool Size=10;Authentication=SqlPassword", result);
+        }
+
+        [Fact] public void VerifyConnectionProperties_WithValidProperties_ReturnsTrue()
+        {
+            // arrange
+            var properties = new Dictionary<string, object>
+            {
+                [SqlServerConstants.KeyName.Password] = "password",
+                [SqlServerConstants.KeyName.Username] = "user",
+                [SqlServerConstants.KeyName.Host] = "host",
+                [SqlServerConstants.KeyName.DatabaseName] = "database",
+                [SqlServerConstants.KeyName.PortNumber] = "9433",
+                [SqlServerConstants.KeyName.ConnectionPoolSize] = "10"
+            };
+
+            // act
+            var result = _sut.VerifyConnectionProperties(properties, out var connectionConfigurationError);
+
+            // assert
+            result.Should().BeTrue();
+            connectionConfigurationError.Should().BeNull();
+        }
+
+        [Fact]
+        public void VerifyConnectionProperties_WithInvalidPort_ReturnsFalse()
+        {
+            // arrange
+            var properties = new Dictionary<string, object>
+            {
+                [SqlServerConstants.KeyName.Password] = "password",
+                [SqlServerConstants.KeyName.Username] = "user",
+                [SqlServerConstants.KeyName.Host] = "host",
+                [SqlServerConstants.KeyName.DatabaseName] = "database",
+                [SqlServerConstants.KeyName.PortNumber] = "invalidPort",
+            };
+
+            // act
+            var result = _sut.VerifyConnectionProperties(properties, out var connectionConfigurationError);
+
+            // assert
+            result.Should().BeFalse();
+            connectionConfigurationError.ErrorMessage.Should().Be("Port number was set, but could not be read as a number");
+        }
+
+        [Fact]
+        public void VerifyConnectionProperties_WithInvalidConnectionPoolSize_ReturnsFalse()
+        {
+            // arrange
+            var properties = new Dictionary<string, object>
+            {
+                [SqlServerConstants.KeyName.Password] = "password",
+                [SqlServerConstants.KeyName.Username] = "user",
+                [SqlServerConstants.KeyName.Host] = "host",
+                [SqlServerConstants.KeyName.DatabaseName] = "database",
+                [SqlServerConstants.KeyName.ConnectionPoolSize] = "invalidPort",
+            };
+
+            // act
+            var result = _sut.VerifyConnectionProperties(properties, out var connectionConfigurationError);
+
+            // assert
+            result.Should().BeFalse();
+            connectionConfigurationError.ErrorMessage.Should().Be("Connection pool size was set, but could not be read as a number");
+        }
+
+        [Fact]
+        public void VerifyConnectionProperties_With0ConnectionPoolSize_ReturnsFalse()
+        {
+            // arrange
+            var properties = new Dictionary<string, object>
+            {
+                [SqlServerConstants.KeyName.Password] = "password",
+                [SqlServerConstants.KeyName.Username] = "user",
+                [SqlServerConstants.KeyName.Host] = "host",
+                [SqlServerConstants.KeyName.DatabaseName] = "database",
+                [SqlServerConstants.KeyName.ConnectionPoolSize] = "0",
+            };
+
+            // act
+            var result = _sut.VerifyConnectionProperties(properties, out var connectionConfigurationError);
+
+            // assert
+            result.Should().BeFalse();
+            connectionConfigurationError.ErrorMessage.Should().Be("Connection pool size was set to a value smaller than 1");
+        }
+
+        [Fact]
+        public void VerifyConnectionProperties_With201ConnectionPoolSize_ReturnsFalse()
+        {
+            // arrange
+            var properties = new Dictionary<string, object>
+            {
+                [SqlServerConstants.KeyName.Password] = "password",
+                [SqlServerConstants.KeyName.Username] = "user",
+                [SqlServerConstants.KeyName.Host] = "host",
+                [SqlServerConstants.KeyName.DatabaseName] = "database",
+                [SqlServerConstants.KeyName.ConnectionPoolSize] = "201",
+            };
+
+            // act
+            var result = _sut.VerifyConnectionProperties(properties, out var connectionConfigurationError);
+
+            // assert
+            result.Should().BeFalse();
+            connectionConfigurationError.ErrorMessage.Should().Be("Connection pool size was set to a value higher than 200");
+        }
     }
 }