From 364f6684c10ff6e19bbf22bff572fc1a3a6493e0 Mon Sep 17 00:00:00 2001 From: Mauricio Scheffer Date: Thu, 18 Jul 2024 14:30:11 +0000 Subject: [PATCH] Add option "IgnoreDatabases" with comma-separated list for import --- metabase-exporter/Config.cs | 4 ++- metabase-exporter/MetabaseApiImport.cs | 38 ++++++++++++++++++-------- metabase-exporter/Program.cs | 22 +++++++++++++-- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/metabase-exporter/Config.cs b/metabase-exporter/Config.cs index 6832744..a15051b 100644 --- a/metabase-exporter/Config.cs +++ b/metabase-exporter/Config.cs @@ -33,11 +33,13 @@ public sealed record Import: Config { public string InputFilename { get; } public IReadOnlyDictionary DatabaseMapping { get; } + public IReadOnlyList IgnoredDatabases { get; } - public Import(MetabaseApiSettings MetabaseApiSettings, string inputFilename, IReadOnlyDictionary databaseMapping): base(MetabaseApiSettings) + public Import(MetabaseApiSettings MetabaseApiSettings, string inputFilename, IReadOnlyDictionary databaseMapping, IReadOnlyList ignoredDatabases): base(MetabaseApiSettings) { InputFilename = inputFilename; DatabaseMapping = databaseMapping; + IgnoredDatabases = ignoredDatabases; } public override T Switch(Func export, Func import, Func testQuestions) => diff --git a/metabase-exporter/MetabaseApiImport.cs b/metabase-exporter/MetabaseApiImport.cs index e9dce99..f366554 100644 --- a/metabase-exporter/MetabaseApiImport.cs +++ b/metabase-exporter/MetabaseApiImport.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -14,14 +15,10 @@ public static class MetabaseApiImport /// /// Imports Metabase data. DELETES all current dashboards/questions/etc. /// - /// - /// - /// - /// - public static async Task Import(this MetabaseApi api, MetabaseState state, IReadOnlyDictionary databaseMapping) + public static async Task Import(this MetabaseApi api, MetabaseState state, IReadOnlyDictionary databaseMapping, IReadOnlyList ignoredDatabases) { // firstly check that the database mapping is complete and correct - await api.ValidateDatabaseMapping(state, databaseMapping); + await api.ValidateDatabaseMapping(state, databaseMapping, ignoredDatabases); // now map/create collections then cards then dashboards @@ -38,7 +35,7 @@ public static async Task Import(this MetabaseApi api, MetabaseState state, IRead var partialCardMapping = await state.Cards .Traverse(async cardFromState => { var source = cardFromState.Id; - var target = await api.MapAndCreateCard(cardFromState, collectionMapping, databaseMapping); + var target = await api.MapAndCreateCard(cardFromState, collectionMapping, databaseMapping, ignoredDatabases); var mapping = new Mapping(source: source, target: target?.Id); return mapping; }); @@ -56,10 +53,20 @@ public static async Task Import(this MetabaseApi api, MetabaseState state, IRead Console.WriteLine("Done importing"); } - static void ValidateSourceDatabaseMapping(MetabaseState state, IReadOnlyDictionary databaseMapping) + static void ValidateSourceDatabaseMapping(MetabaseState state, IReadOnlyDictionary databaseMapping, IReadOnlyList ignoredDatabases) { + var definedIgnoredDatabases = databaseMapping.Keys.Intersect(ignoredDatabases).ToImmutableList(); + if (definedIgnoredDatabases.Count > 0) + { + throw new Exception("Databases marked as ignored but also defined in mappings: " + string.Join(", ", definedIgnoredDatabases)); + } + var allDatabaseIds = state.Cards.SelectMany(c => new[] { c.DatabaseId, c.DatasetQuery.DatabaseId }); - var missingDatabaseIdsInMapping = allDatabaseIds.Where(x => databaseMapping.ContainsKey(x) == false).Distinct().ToList(); + var missingDatabaseIdsInMapping = allDatabaseIds + .Where(x => databaseMapping.ContainsKey(x) == false) + .Distinct() + .Except(ignoredDatabases) + .ToList(); if (missingDatabaseIdsInMapping.Count > 0) { throw new Exception("Missing databases in mapping: " + string.Join(",", missingDatabaseIdsInMapping)); @@ -77,9 +84,9 @@ static async Task ValidateTargetDatabaseMapping(this MetabaseApi api, IReadOnlyD } } - static async Task ValidateDatabaseMapping(this MetabaseApi api, MetabaseState state, IReadOnlyDictionary databaseMapping) + static async Task ValidateDatabaseMapping(this MetabaseApi api, MetabaseState state, IReadOnlyDictionary databaseMapping, IReadOnlyList ignoredDatabases) { - ValidateSourceDatabaseMapping(state, databaseMapping); + ValidateSourceDatabaseMapping(state, databaseMapping, ignoredDatabases); await api.ValidateTargetDatabaseMapping(databaseMapping); } @@ -191,13 +198,20 @@ static async Task>> MapAndCreateCollections(th return collectionMapping; } - static async Task MapAndCreateCard(this MetabaseApi api, Card cardFromState, IReadOnlyList> collectionMapping, IReadOnlyDictionary databaseMapping) + static async Task MapAndCreateCard(this MetabaseApi api, Card cardFromState, IReadOnlyList> collectionMapping, IReadOnlyDictionary databaseMapping, IReadOnlyList ignoredDatabases) { if (cardFromState.DatasetQuery.Native == null) { Console.WriteLine("WARNING: skipping card because it does not have a SQL definition: " + cardFromState.Name); return null; } + + if (ignoredDatabases.Contains(cardFromState.DatabaseId)) + { + Console.WriteLine("WARNING: skipping card because database is marked as ignored: " + cardFromState.Name); + return null; + } + Console.WriteLine($"Creating card '{cardFromState.Name}'"); if (cardFromState.CollectionId.HasValue) { diff --git a/metabase-exporter/Program.cs b/metabase-exporter/Program.cs index 77638fb..d7cfc18 100644 --- a/metabase-exporter/Program.cs +++ b/metabase-exporter/Program.cs @@ -42,7 +42,7 @@ static async Task Import(this MetabaseApi api, Config.Import import) { var rawState = File.ReadAllText(import.InputFilename); var state = JsonConvert.DeserializeObject(rawState); - await api.Import(state, import.DatabaseMapping); + await api.Import(state, import.DatabaseMapping, import.IgnoredDatabases); Console.WriteLine($"Done importing from {import.InputFilename} into {import.MetabaseApiSettings.MetabaseApiUrl}"); } @@ -58,8 +58,9 @@ static Config ParseConfig(IConfiguration rawConfig) { throw new Exception("Missing InputFilename config"); } + var ignoreDatabases = ParseIgnoreDatabases(rawConfig); var databaseMapping = ParseDatabaseMapping(rawConfig); - return new Config.Import(apiSettings, inputFilename, databaseMapping); + return new Config.Import(apiSettings, inputFilename, databaseMapping, ignoreDatabases); } else if (StringComparer.InvariantCultureIgnoreCase.Equals(command, "export")) { @@ -82,6 +83,23 @@ static Config ParseConfig(IConfiguration rawConfig) throw new Exception($"Invalid command '{command}', must be either 'import' or 'export' or 'test-questions'"); } + static IReadOnlyList ParseIgnoreDatabases(IConfiguration rawConfig) + { + return (rawConfig["IgnoreDatabases"] ?? "") + .Split(",") + .Select(x => { + try + { + return new DatabaseId(int.Parse(x)); + } + catch (Exception e) + { + throw new Exception($"Invalid IgnoreDatabases value: '{x}'", e); + } + }) + .ToList(); + } + static IReadOnlyDictionary ParseDatabaseMapping(IConfiguration rawConfig) { var rawDatabaseMapping = rawConfig.GetSection("DatabaseMapping");