Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(go/adbc/driver/snowflake): Made GetObjects case insensitive #1328

Merged
merged 5 commits into from
Dec 4, 2023
Merged

fix(go/adbc/driver/snowflake): Made GetObjects case insensitive #1328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4fa3bac
fix: fixed compilation error
ryan-syed Nov 24, 2023
d3cbdf3
fix: made GetObjects case insensitive and added tests
ryan-syed Nov 28, 2023
59afbcb
fix: removed % for tests where _ is used for pattern match
ryan-syed Nov 28, 2023
64e9467
fix: Moved TestConfiguration initialization to SnowflakeTestingUtils …
ryan-syed Nov 28, 2023
d84d822
fix: Removed unecessary Skip checks and some refactoring
ryan-syed Dec 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
200 changes: 75 additions & 125 deletions csharp/test/Drivers/Snowflake/DriverTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,45 @@ public class DriverTests : IDisposable
readonly AdbcDriver _snowflakeDriver;
readonly AdbcDatabase _database;
readonly AdbcConnection _connection;
readonly List<string> _tableTypes;

public static IEnumerable<object[]> GetPatterns(string name)
{
if (string.IsNullOrEmpty(name)) yield break;

yield return new object[] { name };
yield return new object[] { $"{DriverTests.GetPartialNameForPatternMatch(name)}%" };
yield return new object[] { $"{DriverTests.GetPartialNameForPatternMatch(name).ToLower()}%" };
yield return new object[] { $"{DriverTests.GetPartialNameForPatternMatch(name).ToUpper()}%" };
yield return new object[] { $"_{DriverTests.GetNameWithoutFirstChatacter(name)}" };
yield return new object[] { $"_{DriverTests.GetNameWithoutFirstChatacter(name).ToLower()}" };
yield return new object[] { $"_{DriverTests.GetNameWithoutFirstChatacter(name).ToUpper()}" };
}

public static IEnumerable<object[]> CatalogNamePatternData()
{
string databaseName = SnowflakeTestingUtils.TestConfiguration?.Metadata.Catalog;
return GetPatterns(databaseName);
}

public static IEnumerable<object[]> DbSchemasNamePatternData()
{
string dbSchemaName = SnowflakeTestingUtils.TestConfiguration?.Metadata.Schema;
return GetPatterns(dbSchemaName);
}

public static IEnumerable<object[]> TableNamePatternData()
{
string tableName = SnowflakeTestingUtils.TestConfiguration?.Metadata.Table;
return GetPatterns(tableName);
}

public DriverTests()
{
Skip.IfNot(Utils.CanExecuteTestConfig(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE));
_testConfiguration = Utils.LoadTestConfiguration<SnowflakeTestConfiguration>(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE);
_testConfiguration = SnowflakeTestingUtils.TestConfiguration;

_tableTypes = new List<string> { "BASE TABLE", "VIEW" };
Dictionary<string, string> parameters = new Dictionary<string, string>();
Dictionary<string, string> options = new Dictionary<string, string>();
_snowflakeDriver = SnowflakeTestingUtils.GetSnowflakeAdbcDriver(_testConfiguration, out parameters);
Expand Down Expand Up @@ -110,191 +143,98 @@ public void CanGetInfo()
}
}

/// <summary>
/// Validates if the driver can call GetObjects with GetObjectsDepth as Catalogs.
/// </summary>
[SkippableFact, Order(3)]
public void CanGetObjectsCatalogs()
{
string databaseName = _testConfiguration.Metadata.Catalog;

using IArrowArrayStream stream = _connection.GetObjects(
depth: AdbcConnection.GetObjectsDepth.Catalogs,
catalogPattern: databaseName,
dbSchemaPattern: null,
tableNamePattern: null,
tableTypes: new List<string> { "BASE TABLE", "VIEW" },
columnNamePattern: null);

using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result;

List<AdbcCatalog> catalogs = GetObjectsParser.ParseCatalog(recordBatch, databaseName, null);

AdbcCatalog catalog = catalogs.FirstOrDefault();

Assert.True(catalog != null, "catalog should not be null");
Assert.Equal(databaseName, catalog.Name);
}

/// <summary>
/// Validates if the driver can call GetObjects with GetObjectsDepth as Catalogs and CatalogName passed as a pattern.
/// </summary>
[SkippableFact, Order(3)]
public void CanGetObjectsCatalogsWithPattern()
[SkippableTheory, Order(3)]
[MemberData(nameof(CatalogNamePatternData))]
public void CanGetObjectsCatalogs(string catalogPattern)
{
string databaseName = _testConfiguration.Metadata.Catalog;
string schemaName = _testConfiguration.Metadata.Schema;
string partialDatabaseName = GetPartialNameForPatternMatch(databaseName);


using IArrowArrayStream stream = _connection.GetObjects(
depth: AdbcConnection.GetObjectsDepth.Catalogs,
catalogPattern: $"{partialDatabaseName}%",
catalogPattern: catalogPattern,
dbSchemaPattern: null,
tableNamePattern: null,
tableTypes: new List<string> { "BASE TABLE", "VIEW" },
tableTypes: _tableTypes,
columnNamePattern: null);

using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result;

List<AdbcCatalog> catalogs = GetObjectsParser.ParseCatalog(recordBatch, databaseName, null);

AdbcCatalog catalog = catalogs.FirstOrDefault();
AdbcCatalog catalog = catalogs.Where((catalog) => string.Equals(catalog.Name, databaseName)).FirstOrDefault();

Assert.True(catalog != null, "catalog should not be null");
Assert.StartsWith(databaseName, catalog.Name);
}

/// <summary>
/// Validates if the driver can call GetObjects with GetObjectsDepth as DbSchemas.
/// </summary>
[SkippableFact, Order(3)]
public void CanGetObjectsDbSchemas()
{
// need to add the database
string databaseName = _testConfiguration.Metadata.Catalog;
string schemaName = _testConfiguration.Metadata.Schema;

using IArrowArrayStream stream = _connection.GetObjects(
depth: AdbcConnection.GetObjectsDepth.DbSchemas,
catalogPattern: databaseName,
dbSchemaPattern: schemaName,
tableNamePattern: null,
tableTypes: new List<string> { "BASE TABLE", "VIEW" },
columnNamePattern: null);

using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result;

List<AdbcCatalog> catalogs = GetObjectsParser.ParseCatalog(recordBatch, databaseName, schemaName);

List<AdbcDbSchema> dbSchemas = catalogs
.Select(s => s.DbSchemas)
.FirstOrDefault();
AdbcDbSchema dbSchema = dbSchemas.FirstOrDefault();

Assert.True(dbSchema != null, "dbSchema should not be null");
Assert.Equal(schemaName, dbSchema.Name);
}

/// <summary>
/// Validates if the driver can call GetObjects with GetObjectsDepth as DbSchemas with DbSchemaName as a pattern.
/// </summary>
[SkippableFact, Order(3)]
public void CanGetObjectsDbSchemasWithPattern()
[SkippableTheory, Order(3)]
[MemberData(nameof(DbSchemasNamePatternData))]
public void CanGetObjectsDbSchemas(string dbSchemaPattern)
{
// need to add the database
string databaseName = _testConfiguration.Metadata.Catalog;
string schemaName = _testConfiguration.Metadata.Schema;
string partialSchemaName = GetPartialNameForPatternMatch(schemaName);

using IArrowArrayStream stream = _connection.GetObjects(
depth: AdbcConnection.GetObjectsDepth.DbSchemas,
catalogPattern: databaseName,
dbSchemaPattern: $"{partialSchemaName}%",
dbSchemaPattern: dbSchemaPattern,
tableNamePattern: null,
tableTypes: new List<string> { "BASE TABLE", "VIEW" },
tableTypes: _tableTypes,
columnNamePattern: null);

using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result;

List<AdbcCatalog> catalogs = GetObjectsParser.ParseCatalog(recordBatch, databaseName, schemaName);

List<AdbcDbSchema> dbSchemas = catalogs
.Select(s => s.DbSchemas)
.Where(c => string.Equals(c.Name, databaseName))
.Select(c => c.DbSchemas)
.FirstOrDefault();
AdbcDbSchema dbSchema = dbSchemas.FirstOrDefault();
AdbcDbSchema dbSchema = dbSchemas.Where((dbSchema) => string.Equals(dbSchema.Name, schemaName)).FirstOrDefault();

Assert.True(dbSchema != null, "dbSchema should not be null");
Assert.StartsWith(partialSchemaName, dbSchema.Name);
}

/// <summary>
/// Validates if the driver can call GetObjects with GetObjectsDepth as Tables.
/// </summary>
[SkippableFact, Order(3)]
public void CanGetObjectsTables()
{
// need to add the database
string databaseName = _testConfiguration.Metadata.Catalog;
string schemaName = _testConfiguration.Metadata.Schema;
string tableName = _testConfiguration.Metadata.Table;

using IArrowArrayStream stream = _connection.GetObjects(
depth: AdbcConnection.GetObjectsDepth.All,
catalogPattern: databaseName,
dbSchemaPattern: schemaName,
tableNamePattern: tableName,
tableTypes: new List<string> { "BASE TABLE", "VIEW" },
columnNamePattern: null);

using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result;

List<AdbcCatalog> catalogs = GetObjectsParser.ParseCatalog(recordBatch, databaseName, schemaName);

List<AdbcTable> tables = catalogs
.Select(s => s.DbSchemas)
.FirstOrDefault()
.Select(t => t.Tables)
.FirstOrDefault();
AdbcTable table = tables.FirstOrDefault();

Assert.True(table != null, "table should not be null");
Assert.Equal(tableName, table.Name);
}

/// <summary>
/// Validates if the driver can call GetObjects with GetObjectsDepth as Tables with TableName as a pattern.
/// </summary>
[SkippableFact, Order(3)]
public void CanGetObjectsTablesWithPattern()
[SkippableTheory, Order(3)]
[MemberData(nameof(TableNamePatternData))]
public void CanGetObjectsTables(string tableNamePattern)
{
// need to add the database
string databaseName = _testConfiguration.Metadata.Catalog;
string schemaName = _testConfiguration.Metadata.Schema;
string tableName = _testConfiguration.Metadata.Table;
string partialTableName = GetPartialNameForPatternMatch(tableName);

using IArrowArrayStream stream = _connection.GetObjects(
depth: AdbcConnection.GetObjectsDepth.All,
catalogPattern: databaseName,
dbSchemaPattern: schemaName,
tableNamePattern: $"{partialTableName}%",
tableTypes: new List<string> { "BASE TABLE", "VIEW" },
tableNamePattern: tableNamePattern,
tableTypes: _tableTypes,
columnNamePattern: null);

using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result;

List<AdbcCatalog> catalogs = GetObjectsParser.ParseCatalog(recordBatch, databaseName, schemaName);

List<AdbcTable> tables = catalogs
.Select(s => s.DbSchemas)
.Where(c => string.Equals(c.Name, databaseName))
.Select(c => c.DbSchemas)
.FirstOrDefault()
.Select(t => t.Tables)
.Where(s => string.Equals(s.Name, schemaName))
.Select(s => s.Tables)
.FirstOrDefault();
AdbcTable table = tables.FirstOrDefault();

AdbcTable table = tables.Where((table) => string.Equals(table.Name, tableName)).FirstOrDefault();
Assert.True(table != null, "table should not be null");
Assert.StartsWith(partialTableName, table.Name);
}

/// <summary>
Expand All @@ -314,25 +254,28 @@ public void CanGetObjectsAll()
catalogPattern: databaseName,
dbSchemaPattern: schemaName,
tableNamePattern: tableName,
tableTypes: new List<string> { "BASE TABLE", "VIEW" },
tableTypes: _tableTypes,
columnNamePattern: columnName);

using RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result;

List<AdbcCatalog> catalogs = GetObjectsParser.ParseCatalog(recordBatch, databaseName, schemaName);

List<AdbcColumn> columns = catalogs
.Select(s => s.DbSchemas)
.Where(c => string.Equals(c.Name, databaseName))
.Select(c => c.DbSchemas)
.FirstOrDefault()
.Select(t => t.Tables)
.Where(s => string.Equals(s.Name, schemaName))
.Select(s => s.Tables)
.FirstOrDefault()
.Select(c => c.Columns)
.Where(t => string.Equals(t.Name, tableName))
.Select(t => t.Columns)
.FirstOrDefault();

Assert.True(columns != null, "Columns cannot be null");
Assert.Equal(testConfiguration.Metadata.ExpectedColumnCount, columns.Count);
Assert.Equal(_testConfiguration.Metadata.ExpectedColumnCount, columns.Count);

if (testConfiguration.UseHighPrecision)
if (_testConfiguration.UseHighPrecision)
{
IEnumerable<AdbcColumn> highPrecisionColumns = columns.Where(c => c.XdbcTypeName == "NUMBER");

Expand Down Expand Up @@ -411,13 +354,20 @@ public void CanExecuteQuery()
Tests.DriverTests.CanExecuteQuery(queryResult, _testConfiguration.ExpectedResultsCount);
}

private string GetPartialNameForPatternMatch(string name)
private static string GetPartialNameForPatternMatch(string name)
{
if (string.IsNullOrEmpty(name) || name.Length == 1) return name;

return name.Substring(0, name.Length / 2);
}

private static string GetNameWithoutFirstChatacter(string name)
{
if (string.IsNullOrEmpty(name)) return name;

return name.Substring(1);
}

public void Dispose()
{
_connection.Dispose();
Expand Down
24 changes: 20 additions & 4 deletions csharp/test/Drivers/Snowflake/SnowflakeTestingUtils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@
* limitations under the License.
*/

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Apache.Arrow.Adbc.C;
using Xunit;

namespace Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake
{
Expand All @@ -40,8 +42,22 @@ internal class SnowflakeParameters

internal class SnowflakeTestingUtils
{
internal static readonly SnowflakeTestConfiguration TestConfiguration;

internal const string SNOWFLAKE_TEST_CONFIG_VARIABLE = "SNOWFLAKE_TEST_CONFIG_FILE";

static SnowflakeTestingUtils()
{
try
{
TestConfiguration = Utils.LoadTestConfiguration<SnowflakeTestConfiguration>(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE);
}
catch (InvalidOperationException ex)
{
Console.WriteLine(ex.Message);
}
}

/// <summary>
/// Gets a the Snowflake ADBC driver with settings from the
/// <see cref="SnowflakeTestConfiguration"/>.
Expand All @@ -66,12 +82,12 @@ out Dictionary<string, string> parameters
{ SnowflakeParameters.USE_HIGH_PRECISION, testConfiguration.UseHighPrecision.ToString().ToLowerInvariant() }
};

if(!string.IsNullOrWhiteSpace(testConfiguration.Host))
if (!string.IsNullOrWhiteSpace(testConfiguration.Host))
{
parameters[SnowflakeParameters.HOST] = testConfiguration.Host;
}

if(!string.IsNullOrWhiteSpace(testConfiguration.Database))
if (!string.IsNullOrWhiteSpace(testConfiguration.Database))
{
parameters[SnowflakeParameters.DATABASE] = testConfiguration.Database;
}
Expand Down Expand Up @@ -120,9 +136,9 @@ internal static string[] GetQueries(SnowflakeTestConfiguration testConfiguration
{
string modifiedLine = line;

foreach(string key in placeholderValues.Keys)
foreach (string key in placeholderValues.Keys)
{
if(modifiedLine.Contains(key))
if (modifiedLine.Contains(key))
modifiedLine = modifiedLine.Replace(key, placeholderValues[key]);
}

Expand Down
Loading
Loading