diff --git a/docs/4.1.1-release-notes.md b/docs/4.1.1-release-notes.md new file mode 100644 index 0000000..fa2961b --- /dev/null +++ b/docs/4.1.1-release-notes.md @@ -0,0 +1,5 @@ +### Features +- Allow user to optionally configure connection pool size. + +### Fixes +- Fix issue where entity type properties were not serialized correctly \ No newline at end of file diff --git a/src/Connector.SqlServer/Connector/ISqlClient.cs b/src/Connector.SqlServer/Connector/ISqlClient.cs index 1e33e21..8ff3a31 100644 --- a/src/Connector.SqlServer/Connector/ISqlClient.cs +++ b/src/Connector.SqlServer/Connector/ISqlClient.cs @@ -5,8 +5,12 @@ namespace CluedIn.Connector.SqlServer.Connector { + public record ConnectionConfigurationError(string ErrorMessage); + public interface ISqlClient { + bool VerifyConnectionProperties(IReadOnlyDictionary config, out ConnectionConfigurationError configurationError); + Task BeginConnection(IReadOnlyDictionary config); Task GetTableColumns(SqlConnection connection, string tableName, string schema); diff --git a/src/Connector.SqlServer/Connector/SqlClient.cs b/src/Connector.SqlServer/Connector/SqlClient.cs index bf3a337..7508788 100644 --- a/src/Connector.SqlServer/Connector/SqlClient.cs +++ b/src/Connector.SqlServer/Connector/SqlClient.cs @@ -11,6 +11,7 @@ namespace CluedIn.Connector.SqlServer.Connector public class SqlClient : ISqlClient { private readonly int _defaultPort = 1433; + private readonly int _defaultConnectionPoolSize = 200; public string BuildConnectionString(IReadOnlyDictionary config) { @@ -22,17 +23,76 @@ public string BuildConnectionString(IReadOnlyDictionary config) DataSource = (string)config[SqlServerConstants.KeyName.Host], InitialCatalog = (string)config[SqlServerConstants.KeyName.DatabaseName], Pooling = true, - MaxPoolSize = 200 }; - if (config.TryGetValue(SqlServerConstants.KeyName.PortNumber, out var portEntry) && int.TryParse(portEntry.ToString(), out var port)) + // Configure port + { + var port = _defaultPort; + if (config.TryGetValue(SqlServerConstants.KeyName.PortNumber, out var portEntry) && + !string.IsNullOrEmpty(portEntry.ToString()) && + int.TryParse(portEntry.ToString(), out var parsedPort)) + { + port = parsedPort; + } + connectionStringBuilder.DataSource = $"{connectionStringBuilder.DataSource},{port}"; - else - connectionStringBuilder.DataSource = $"{connectionStringBuilder.DataSource},{_defaultPort}"; + } + + // Configure connection pool size + { + var connectionPoolSize = _defaultConnectionPoolSize; + if (config.TryGetValue(SqlServerConstants.KeyName.ConnectionPoolSize, out var connectionPoolSizeEntry) && + !string.IsNullOrEmpty(connectionPoolSizeEntry.ToString()) && + int.TryParse(connectionPoolSizeEntry.ToString(), out var parsedConnectionPoolSize)) + { + connectionPoolSize = parsedConnectionPoolSize; + } + + connectionStringBuilder.MaxPoolSize = connectionPoolSize; + } + return connectionStringBuilder.ToString(); } + public bool VerifyConnectionProperties(IReadOnlyDictionary config, out ConnectionConfigurationError configurationError) + { + if (config.TryGetValue(SqlServerConstants.KeyName.PortNumber, out var portEntry) && !string.IsNullOrEmpty(portEntry.ToString())) + { + if (!int.TryParse(portEntry.ToString(), out _)) + { + configurationError = new ConnectionConfigurationError("Port number was set, but could not be read as a number"); + return false; + } + } + + if (config.TryGetValue(SqlServerConstants.KeyName.ConnectionPoolSize, out var connectionPoolSizeEntry) && !string.IsNullOrEmpty(connectionPoolSizeEntry.ToString())) + { + if (int.TryParse(connectionPoolSizeEntry.ToString(), out var parsedPoolSize)) + { + if (parsedPoolSize < 1) + { + configurationError = new ConnectionConfigurationError("Connection pool size was set to a value smaller than 1"); + return false; + } + + if (parsedPoolSize > _defaultConnectionPoolSize) + { + configurationError = new ConnectionConfigurationError("Connection pool size was set to a value higher than 200"); + return false; + } + } + else + { + configurationError = new ConnectionConfigurationError("Connection pool size was set, but could not be read as a number"); + return false; + } + } + + configurationError = null; + return true; + } + public async Task BeginConnection(IReadOnlyDictionary config) { var connectionString = BuildConnectionString(config); diff --git a/src/Connector.SqlServer/Connector/SqlServerConnector.cs b/src/Connector.SqlServer/Connector/SqlServerConnector.cs index d82fc73..c976cdc 100644 --- a/src/Connector.SqlServer/Connector/SqlServerConnector.cs +++ b/src/Connector.SqlServer/Connector/SqlServerConnector.cs @@ -308,6 +308,11 @@ public override async Task VerifyConnection(Execut { try { + if (!_client.VerifyConnectionProperties(configurationData, out var configurationError)) + { + return new ConnectionVerificationResult(success: false, errorMessage: configurationError.ErrorMessage); + } + await using var connectionAndTransaction = await _client.BeginTransaction(configurationData); var connectionIsOpen = connectionAndTransaction.Connection.State == ConnectionState.Open; await connectionAndTransaction.DisposeAsync(); @@ -484,7 +489,7 @@ await ExecuteWithRetryAsync(async () => public override Task GetValidMappingDestinationPropertyName(ExecutionContext executionContext, Guid connectorProviderDefinitionId, string propertyName) { - return Task.FromResult(propertyName.ToSanitizedSqlName()); + return Task.FromResult(propertyName); } public override async Task RemoveContainer(ExecutionContext executionContext, IReadOnlyStreamModel streamModel) diff --git a/src/Connector.SqlServer/SqlServerConstants.cs b/src/Connector.SqlServer/SqlServerConstants.cs index f77f507..4a8ba84 100644 --- a/src/Connector.SqlServer/SqlServerConstants.cs +++ b/src/Connector.SqlServer/SqlServerConstants.cs @@ -17,6 +17,7 @@ public struct KeyName public const string Username = "username"; public const string Password = "password"; public const string PortNumber = "portNumber"; + public const string ConnectionPoolSize = "connectionPoolSize"; } public SqlServerConstants() @@ -103,6 +104,13 @@ public SqlServerConstants() displayName = "Schema", type = "input", isRequired = false + }, + new Control + { + name = KeyName.ConnectionPoolSize, + displayName = "Connection pool size", + type = "input", + isRequired = false } } }; diff --git a/src/Connector.SqlServer/Utils/TableDefinitions/MainTableDefinition.cs b/src/Connector.SqlServer/Utils/TableDefinitions/MainTableDefinition.cs index ad040d1..0fdcb6a 100644 --- a/src/Connector.SqlServer/Utils/TableDefinitions/MainTableDefinition.cs +++ b/src/Connector.SqlServer/Utils/TableDefinitions/MainTableDefinition.cs @@ -1,5 +1,6 @@ using CluedIn.Connector.SqlServer.Connector; using CluedIn.Core.Connectors; +using CluedIn.Core.Data; using CluedIn.Core.Streams.Models; using Microsoft.Data.SqlClient; using System; @@ -49,15 +50,21 @@ public static MainTableColumnDefinition[] GetColumnDefinitions(StreamMode stream var defaultColumnNamesHashSet = defaultColumns.Select(x => x.Name).ToHashSet(); + var alreadyUsedNames = new HashSet(); + var propertyColumns = properties // We need to filter out any properties, that are contained in the default columns. .Where(property => !defaultColumnNamesHashSet.Contains(property.name.ToSanitizedSqlName())) + .OrderBy(property => property.dataType is VocabularyKeyConnectorPropertyDataType x + ? $"{x.VocabularyKey.Vocabulary.KeyPrefix}.{x.VocabularyKey.Name}" + : property.name) .Select(property => { - var name = property.name.ToSanitizedSqlName(); + var nameToUse = GetNameToUse(property, alreadyUsedNames); + var sqlType = SqlColumnHelper.GetColumnType(property.dataType); return new MainTableColumnDefinition( - name, + nameToUse, sqlType, input => { @@ -82,6 +89,11 @@ public static MainTableColumnDefinition[] GetColumnDefinitions(StreamMode stream return dateTimeOffsetValue.ToString("O"); } + if (propertyValue is EntityType entityTypeValue) + { + return entityTypeValue.ToString(); + } + return propertyValue; }, CanBeNull: true); @@ -92,6 +104,37 @@ public static MainTableColumnDefinition[] GetColumnDefinitions(StreamMode stream return allColumns; } + private static string GetNameToUse((string name, ConnectorPropertyDataType dataType) property, HashSet alreadyUsedNames) + { + string rawName; + switch (property.dataType) + { + case VocabularyKeyConnectorPropertyDataType vocabularyKeyConnectorPropertyDataType: + var vocabularyKey = vocabularyKeyConnectorPropertyDataType.VocabularyKey; + rawName = $"{vocabularyKey.Vocabulary.KeyPrefix}.{vocabularyKey.Name}"; + break; + default: + rawName = property.name; + break; + } + + rawName = rawName.ToSanitizedSqlName(); + + var number = 0; + + var nameToUse = rawName; + while (alreadyUsedNames.Contains(nameToUse)) + { + number++; + nameToUse = $"{rawName}_{number}"; + } + + alreadyUsedNames.Add(nameToUse); + + // We need to call ToSanitizedSqlName again, in case adding numbers pushed length of name over the maximum + return nameToUse.ToSanitizedSqlName(); + } + public static SqlServerConnectorCommand CreateUpsertCommand(IReadOnlyStreamModel streamModel, SqlConnectorEntityData connectorEntityData, SqlName schema) { var mainTableName = TableNameUtility.GetMainTableName(streamModel, schema); diff --git a/test/unit/Connector.SqlServer.Test/Connector/SqlClientTests.cs b/test/unit/Connector.SqlServer.Test/Connector/SqlClientTests.cs index 046a418..e9b2f9f 100644 --- a/test/unit/Connector.SqlServer.Test/Connector/SqlClientTests.cs +++ b/test/unit/Connector.SqlServer.Test/Connector/SqlClientTests.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using CluedIn.Connector.SqlServer.Connector; +using FluentAssertions; using Xunit; namespace CluedIn.Connector.SqlServer.Unit.Tests.Connector @@ -21,7 +22,7 @@ public void BuildConnectionString_Sets_From_Dictionary() [SqlServerConstants.KeyName.Password] = "password", [SqlServerConstants.KeyName.Username] = "user", [SqlServerConstants.KeyName.Host] = "host", - [SqlServerConstants.KeyName.DatabaseName] = "database" + [SqlServerConstants.KeyName.DatabaseName] = "database", }; var result = _sut.BuildConnectionString(properties); @@ -79,5 +80,130 @@ public void BuildConnectionString_WithInvalidPort_Sets_From_Dictionary() Assert.Equal("Data Source=host,1433;Initial Catalog=database;User ID=user;Password=password;Pooling=True;Max Pool Size=200;Authentication=SqlPassword", result); } + + [Fact] + public void BuildConnectionString_WithConnectionPoolSize_Sets_From_Dictionary() + { + // arrange + var properties = new Dictionary + { + [SqlServerConstants.KeyName.Password] = "password", + [SqlServerConstants.KeyName.Username] = "user", + [SqlServerConstants.KeyName.Host] = "host", + [SqlServerConstants.KeyName.DatabaseName] = "database", + [SqlServerConstants.KeyName.ConnectionPoolSize] = 10, + }; + + // act + var result = _sut.BuildConnectionString(properties); + + // assert + Assert.Equal("Data Source=host,1433;Initial Catalog=database;User ID=user;Password=password;Pooling=True;Max Pool Size=10;Authentication=SqlPassword", result); + } + + [Fact] public void VerifyConnectionProperties_WithValidProperties_ReturnsTrue() + { + // arrange + var properties = new Dictionary + { + [SqlServerConstants.KeyName.Password] = "password", + [SqlServerConstants.KeyName.Username] = "user", + [SqlServerConstants.KeyName.Host] = "host", + [SqlServerConstants.KeyName.DatabaseName] = "database", + [SqlServerConstants.KeyName.PortNumber] = "9433", + [SqlServerConstants.KeyName.ConnectionPoolSize] = "10" + }; + + // act + var result = _sut.VerifyConnectionProperties(properties, out var connectionConfigurationError); + + // assert + result.Should().BeTrue(); + connectionConfigurationError.Should().BeNull(); + } + + [Fact] + public void VerifyConnectionProperties_WithInvalidPort_ReturnsFalse() + { + // arrange + var properties = new Dictionary + { + [SqlServerConstants.KeyName.Password] = "password", + [SqlServerConstants.KeyName.Username] = "user", + [SqlServerConstants.KeyName.Host] = "host", + [SqlServerConstants.KeyName.DatabaseName] = "database", + [SqlServerConstants.KeyName.PortNumber] = "invalidPort", + }; + + // act + var result = _sut.VerifyConnectionProperties(properties, out var connectionConfigurationError); + + // assert + result.Should().BeFalse(); + connectionConfigurationError.ErrorMessage.Should().Be("Port number was set, but could not be read as a number"); + } + + [Fact] + public void VerifyConnectionProperties_WithInvalidConnectionPoolSize_ReturnsFalse() + { + // arrange + var properties = new Dictionary + { + [SqlServerConstants.KeyName.Password] = "password", + [SqlServerConstants.KeyName.Username] = "user", + [SqlServerConstants.KeyName.Host] = "host", + [SqlServerConstants.KeyName.DatabaseName] = "database", + [SqlServerConstants.KeyName.ConnectionPoolSize] = "invalidPort", + }; + + // act + var result = _sut.VerifyConnectionProperties(properties, out var connectionConfigurationError); + + // assert + result.Should().BeFalse(); + connectionConfigurationError.ErrorMessage.Should().Be("Connection pool size was set, but could not be read as a number"); + } + + [Fact] + public void VerifyConnectionProperties_With0ConnectionPoolSize_ReturnsFalse() + { + // arrange + var properties = new Dictionary + { + [SqlServerConstants.KeyName.Password] = "password", + [SqlServerConstants.KeyName.Username] = "user", + [SqlServerConstants.KeyName.Host] = "host", + [SqlServerConstants.KeyName.DatabaseName] = "database", + [SqlServerConstants.KeyName.ConnectionPoolSize] = "0", + }; + + // act + var result = _sut.VerifyConnectionProperties(properties, out var connectionConfigurationError); + + // assert + result.Should().BeFalse(); + connectionConfigurationError.ErrorMessage.Should().Be("Connection pool size was set to a value smaller than 1"); + } + + [Fact] + public void VerifyConnectionProperties_With201ConnectionPoolSize_ReturnsFalse() + { + // arrange + var properties = new Dictionary + { + [SqlServerConstants.KeyName.Password] = "password", + [SqlServerConstants.KeyName.Username] = "user", + [SqlServerConstants.KeyName.Host] = "host", + [SqlServerConstants.KeyName.DatabaseName] = "database", + [SqlServerConstants.KeyName.ConnectionPoolSize] = "201", + }; + + // act + var result = _sut.VerifyConnectionProperties(properties, out var connectionConfigurationError); + + // assert + result.Should().BeFalse(); + connectionConfigurationError.ErrorMessage.Should().Be("Connection pool size was set to a value higher than 200"); + } } } diff --git a/test/unit/Connector.SqlServer.Test/Utils/TableDefinitions/MainTableDefinitionTests.cs b/test/unit/Connector.SqlServer.Test/Utils/TableDefinitions/MainTableDefinitionTests.cs index c8585dd..ac62a62 100644 --- a/test/unit/Connector.SqlServer.Test/Utils/TableDefinitions/MainTableDefinitionTests.cs +++ b/test/unit/Connector.SqlServer.Test/Utils/TableDefinitions/MainTableDefinitionTests.cs @@ -121,5 +121,92 @@ public void DateTimeOffsetPropertyValues_ShouldBeToISO8601( var sqlDateValue = discoveryDateColumnDefinition.GetValueFunc(sqlDiscoveryDatePropertyDate); sqlDateValue.Should().Be("2000-01-01T01:01:01.0000000+01:00"); } + + [Theory, AutoNData] + public void VocabularyPropertiesBeingSanitizedToTheSameName_ShouldHaveNumbersAddedAtTheEnd( + IVocabulary vocabulary1, + IVocabulary vocabulary2) + { + // arrange + vocabulary1.KeyPrefix = "test--vocabulary"; + vocabulary2.KeyPrefix = "test-.vocabulary"; + var vocabularyKey1 = new VocabularyKey("name") { Vocabulary = vocabulary1 }; + var vocabularyKey2 = new VocabularyKey("name") { Vocabulary = vocabulary2 }; + + var properties = new (string, ConnectorPropertyDataType)[] + { + ("testvocabularyname", new VocabularyKeyConnectorPropertyDataType(vocabularyKey1)), + ("testvocabularyname", new VocabularyKeyConnectorPropertyDataType(vocabularyKey2)), + }; + + // act + var syncColumnDefinitions = MainTableDefinition.GetColumnDefinitions(StreamMode.Sync, properties); + + // assert + var columnDefinitionNames = syncColumnDefinitions.Select(x => x.Name).ToList(); + + columnDefinitionNames.Should().Contain("testvocabularyname"); + columnDefinitionNames.Should().Contain("testvocabularyname_1"); + } + + [Theory, AutoNData] + public void DifferentOrderVocabularyProperties_ShouldNotImpactOrderOfColumnDefinition( + IVocabulary vocabulary1, + IVocabulary vocabulary2) + { + // arrange + vocabulary1.KeyPrefix = "test--vocabulary"; + vocabulary2.KeyPrefix = "test-.vocabulary"; + var vocabularyKey1 = new VocabularyKey("name") { Vocabulary = vocabulary1 }; + var vocabularyKey2 = new VocabularyKey("name") { Vocabulary = vocabulary2 }; + + var properties1 = new (string, ConnectorPropertyDataType)[] + { + ("testvocabularyname", new VocabularyKeyConnectorPropertyDataType(vocabularyKey1)), + ("testvocabularyname", new VocabularyKeyConnectorPropertyDataType(vocabularyKey2)), + }; + + var properties2 = new (string, ConnectorPropertyDataType)[] + { + ("testvocabularyname", new VocabularyKeyConnectorPropertyDataType(vocabularyKey2)), + ("testvocabularyname", new VocabularyKeyConnectorPropertyDataType(vocabularyKey1)), + }; + + // act + var syncColumnDefinitions1 = MainTableDefinition.GetColumnDefinitions(StreamMode.Sync, properties1); + var syncColumnDefinitions2 = MainTableDefinition.GetColumnDefinitions(StreamMode.Sync, properties2); + + // assert + var syncColumnDefinitions1Names = syncColumnDefinitions1.Select(x => (x.Name)); + var syncColumnDefinitions2Names = syncColumnDefinitions2.Select(x => (x.Name)); + + syncColumnDefinitions1Names.Should().BeEquivalentTo(syncColumnDefinitions2Names); + } + + [Theory, AutoNData] + public void EntityTypePropertyValues_ShouldBeMadeIntoStrings( + VersionChangeType versionChangeType, + Guid entityId, + Guid correlationId) + { + // arrange + var properties = new (string, ConnectorPropertyDataType)[] + { + ("Type", new EntityPropertyConnectorPropertyDataType(typeof(EntityType))), + }; + var entityTypeValue = EntityType.Person; + + var entityTypePropertyDate = new ConnectorPropertyData("Type", entityTypeValue, new EntityPropertyConnectorPropertyDataType(typeof(EntityType))); + var connectorEntityData = new ConnectorEntityData(versionChangeType, StreamMode.Sync, entityId, null, null, null, null, new[] { entityTypePropertyDate }, Array.Empty(), Array.Empty(), Array.Empty()); + var sqlEntityTypePropertyDate = new SqlConnectorEntityData(connectorEntityData, correlationId, timestamp: DateTimeOffset.Now); + + // act + var syncColumnDefinitions = MainTableDefinition.GetColumnDefinitions(StreamMode.Sync, properties); + + // assert + var discoveryDateColumnDefinition = syncColumnDefinitions.Where(column => column.Name == "Type").Should().ContainSingle().And.Subject.First(); + var sqlDateValue = discoveryDateColumnDefinition.GetValueFunc(sqlEntityTypePropertyDate); + sqlDateValue.Should().Be("/Person"); + } } }