From 88b630cefdd601d4e4049cc297472afe0c6c90a5 Mon Sep 17 00:00:00 2001 From: Ben Edwards Date: Fri, 14 Apr 2023 15:21:31 +1000 Subject: [PATCH 1/2] Add support for custom indexes on events tables --- docs/configuration/hostbuilder.md | 2 +- docs/configuration/storeoptions.md | 2 +- docs/events/configuration.md | 2 +- docs/events/multitenancy.md | 2 +- .../projections/aggregate-projections.md | 16 +++--- docs/events/projections/live-aggregates.md | 8 +-- docs/events/storage.md | 25 ++++++++++ .../EventStoreCustomIndexesTests.cs | 50 +++++++++++++++++++ .../Examples/ConfiguringDocumentStore.cs | 24 ++++++++- src/Marten/Events/EventGraph.FeatureSchema.cs | 9 +++- src/Marten/Events/EventGraph.cs | 16 ++++++ src/Marten/Events/IEventStoreOptions.cs | 25 ++++++++++ 12 files changed, 161 insertions(+), 20 deletions(-) create mode 100644 src/EventSourcingTests/EventStoreCustomIndexesTests.cs diff --git a/docs/configuration/hostbuilder.md b/docs/configuration/hostbuilder.md index 3679766a67..373947959d 100644 --- a/docs/configuration/hostbuilder.md +++ b/docs/configuration/hostbuilder.md @@ -246,7 +246,7 @@ public interface IConfigureMarten void Configure(IServiceProvider services, StoreOptions options); } ``` -snippet source | anchor +snippet source | anchor You could alternatively implement a custom `IConfigureMarten` class like so: diff --git a/docs/configuration/storeoptions.md b/docs/configuration/storeoptions.md index 5627cfa7eb..d9f6b795ce 100644 --- a/docs/configuration/storeoptions.md +++ b/docs/configuration/storeoptions.md @@ -68,7 +68,7 @@ public class MyStoreOptions: StoreOptions } } ``` -snippet source | anchor +snippet source | anchor This strategy might be beneficial if you need to share Marten configuration across different applications diff --git a/docs/events/configuration.md b/docs/events/configuration.md index c247f9338f..10c186f847 100644 --- a/docs/events/configuration.md +++ b/docs/events/configuration.md @@ -60,7 +60,7 @@ var store = DocumentStore.For(opts => opts.Events.TenancyStyle = TenancyStyle.Conjoined; }); ``` -snippet source | anchor +snippet source | anchor By default, if you try to define projection with a single tenancy, Marten will throw an exception at runtime informing you about the mismatch. diff --git a/docs/events/multitenancy.md b/docs/events/multitenancy.md index 7cc705701c..41470bb96b 100644 --- a/docs/events/multitenancy.md +++ b/docs/events/multitenancy.md @@ -19,5 +19,5 @@ var store = DocumentStore.For(opts => opts.Events.TenancyStyle = TenancyStyle.Conjoined; }); ``` -snippet source | anchor +snippet source | anchor diff --git a/docs/events/projections/aggregate-projections.md b/docs/events/projections/aggregate-projections.md index ae8dd7f942..51d798622f 100644 --- a/docs/events/projections/aggregate-projections.md +++ b/docs/events/projections/aggregate-projections.md @@ -97,7 +97,7 @@ public class TripProjection: SingleStreamProjection } } ``` -snippet source | anchor +snippet source | anchor And register that projection like this: @@ -116,7 +116,7 @@ var store = DocumentStore.For(opts => opts.Projections.Add(ProjectionLifecycle.Async); }); ``` -snippet source | anchor +snippet source | anchor Any projection based on `SingleStreamProjection` will allow you to define steps by event type to either create, delete, or mutate an aggregate @@ -185,7 +185,7 @@ public class Trip internal bool ShouldDelete(VacationOver e) => Traveled > 1000; } ``` -snippet source | anchor +snippet source | anchor Or finally, you can use a method named `Create()` on a projection type as shown in this sample: @@ -221,7 +221,7 @@ public class TripProjection: SingleStreamProjection } } ``` -snippet source | anchor +snippet source | anchor The `Create()` method has to return either the aggregate document type or `Task` where `T` is the aggregate document type. There must be an argument for the specific event type or `Event` where `T` is the event type if you need access to event metadata. You can also take in an `IQuerySession` if you need to look up additional data as part of the transformation or `IEvent` in addition to the exact event type just to get at event metadata. @@ -261,7 +261,7 @@ public class TripProjection: SingleStreamProjection } } ``` -snippet source | anchor +snippet source | anchor I'm not personally that wild about using lots of inline Lambdas like the example above, and to that end, Marten now supports the `Apply()` method convention. Here's the same `TripProjection`, but this time using methods to mutate the `Trip` document: @@ -297,7 +297,7 @@ public class TripProjection: SingleStreamProjection } } ``` -snippet source | anchor +snippet source | anchor The `Apply()` methods can accept any combination of these arguments: @@ -466,7 +466,7 @@ public class Trip internal bool ShouldDelete(VacationOver e) => Traveled > 1000; } ``` -snippet source | anchor +snippet source | anchor Here's an example of using the various ways of doing `Trip` stream aggregation: @@ -500,7 +500,7 @@ internal async Task use_a_stream_aggregation() var trip = await session.Events.AggregateStreamAsync(tripId); } ``` -snippet source | anchor +snippet source | anchor ## Aggregate Versioning diff --git a/docs/events/projections/live-aggregates.md b/docs/events/projections/live-aggregates.md index 9821baf467..13ba08512a 100644 --- a/docs/events/projections/live-aggregates.md +++ b/docs/events/projections/live-aggregates.md @@ -231,7 +231,7 @@ await theSession.Events.AggregateStreamAsync( fromVersion: baseStateVersion ); ``` -snippet source | anchor +snippet source | anchor It can be helpful, for instance, in snapshotting. Snapshot is a state of the stream at a specific point of time (version). It is a performance optimization that shouldn't be your first choice, but it's an option to consider for performance-critical computations. As you're optimizing your processing, you usually don't want to store a snapshot after each event not to increase the number of writes. Usually, you'd like to do a snapshot on the specific interval or specific event type. @@ -360,7 +360,7 @@ public class CashRegisterRepository } } ``` -snippet source | anchor +snippet source | anchor Then append event and store snapshot on opening accounting month: @@ -390,7 +390,7 @@ var repository = new CashRegisterRepository(theSession); await repository.Store(openedCashierShift, cashierShiftOpened); ``` -snippet source | anchor +snippet source | anchor and read snapshot and following event with: @@ -400,7 +400,7 @@ and read snapshot and following event with: ```cs var currentState = await repository.Get(financialAccountId); ``` -snippet source | anchor +snippet source | anchor ## Live Aggregation from Linq Queries diff --git a/docs/events/storage.md b/docs/events/storage.md index e90fd5e594..74be25b146 100644 --- a/docs/events/storage.md +++ b/docs/events/storage.md @@ -112,3 +112,28 @@ public Dictionary? Headers { get; set; } The full event data is available on `EventStream` and `IEvent` objects immediately after committing a transaction that involves event capture. See [diagnostics and instrumentation](/diagnostics) for more information on capturing event data in the instrumentation hooks. + +## Adding indexes to event tables + +Additional indexes can be added to the `mt_streams` and `mt_events` tables. These can be useful if you often need to perform queries directly against the event tables. + + + +```cs +var store = DocumentStore.For(_ => +{ + _.Connection("some connection string"); + + // Add an index to the mt_streams table on "is_archived" + _.Events.AddIndexToStreamsTable( + new IndexDefinition("idx_mt_streams_is_archived") + .AgainstColumns("is_archived")); + + // Add an index to the mt_events table on "type" + _.Events.AddIndexToEventsTable( + new IndexDefinition("idx_mt_events_is_type") + .AgainstColumns("type")); +}); +``` +snippet source | anchor + diff --git a/src/EventSourcingTests/EventStoreCustomIndexesTests.cs b/src/EventSourcingTests/EventStoreCustomIndexesTests.cs new file mode 100644 index 0000000000..04efaad50e --- /dev/null +++ b/src/EventSourcingTests/EventStoreCustomIndexesTests.cs @@ -0,0 +1,50 @@ +using System.Linq; +using System.Threading.Tasks; +using Marten.Events; +using Marten.Testing; +using Marten.Testing.Harness; +using Weasel.Postgresql.Tables; +using Xunit; + +namespace EventSourcingTests; + +public class EventStoreCustomIndexesTests : OneOffConfigurationsContext +{ + [Fact] + public async Task can_create_custom_indexes_on_event_tables() + { + const string streamsTypeIndexName = "idx_mt_streams_type"; + const string eventsDataIndexName = "idx_mt_events_data_gin"; + StoreOptions(options => + { + var streamsTypeIndex = new IndexDefinition(streamsTypeIndexName).AgainstColumns("type"); + options.Events.AddIndexToStreamsTable(streamsTypeIndex); + + var eventsDataIndex = new IndexDefinition(eventsDataIndexName).AgainstColumns("data"); + eventsDataIndex.Method = IndexMethod.gin; + options.Events.AddIndexToEventsTable(eventsDataIndex); + }); + + await theStore.EnsureStorageExistsAsync(typeof(IEvent)); + + Assert.True(await CheckIfIndexExists("mt_streams", streamsTypeIndexName)); + Assert.True(await CheckIfIndexExists("mt_events", eventsDataIndexName)); + } + + private async Task CheckIfIndexExists(string tableName, string indexName) + { + var exists = await theSession.QueryAsync(@" + select exists( + select 1 + from pg_catalog.pg_indexes + where schemaname = ? + and tablename = ? + and indexname = ? + )", + _schemaName, + tableName, + indexName); + + return exists.FirstOrDefault(false); + } +} diff --git a/src/Marten.Testing/Examples/ConfiguringDocumentStore.cs b/src/Marten.Testing/Examples/ConfiguringDocumentStore.cs index 8400ddffef..73d96a9774 100644 --- a/src/Marten.Testing/Examples/ConfiguringDocumentStore.cs +++ b/src/Marten.Testing/Examples/ConfiguringDocumentStore.cs @@ -6,7 +6,7 @@ using Marten.Testing.Harness; using Newtonsoft.Json; using Weasel.Core; -using Weasel.Postgresql; +using Weasel.Postgresql.Tables; namespace Marten.Testing.Examples; // Leave this commented out please, and always use the User @@ -200,6 +200,26 @@ public void setting_event_schema() #endregion } + public void setting_event_custom_indexes() + { + #region sample_setting_event_custom_indexes + var store = DocumentStore.For(_ => + { + _.Connection("some connection string"); + + // Add an index to the mt_streams table on "is_archived" + _.Events.AddIndexToStreamsTable( + new IndexDefinition("idx_mt_streams_is_archived") + .AgainstColumns("is_archived")); + + // Add an index to the mt_events table on "type" + _.Events.AddIndexToEventsTable( + new IndexDefinition("idx_mt_events_is_type") + .AgainstColumns("type")); + }); + #endregion + } + #region sample_custom-store-options public class MyStoreOptions: StoreOptions { @@ -235,4 +255,4 @@ public void set_multi_tenancy_on_events() #endregion } -} \ No newline at end of file +} diff --git a/src/Marten/Events/EventGraph.FeatureSchema.cs b/src/Marten/Events/EventGraph.FeatureSchema.cs index c3fbfe9a39..dc085ee9de 100644 --- a/src/Marten/Events/EventGraph.FeatureSchema.cs +++ b/src/Marten/Events/EventGraph.FeatureSchema.cs @@ -1,7 +1,8 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; +using JasperFx.Core; using Marten.Events.Archiving; using Marten.Events.Daemon; using Marten.Events.Projections; @@ -38,8 +39,12 @@ void IFeatureSchema.WritePermissions(Migrator rules, TextWriter writer) private IEnumerable createAllSchemaObjects() { - yield return new StreamsTable(this); + var streamsTable = new StreamsTable(this); + streamsTable.Indexes.AddRange(_customStreamsTableIndexes); + yield return streamsTable; + var eventsTable = new EventsTable(this); + eventsTable.Indexes.AddRange(_customEventsTableIndexes); yield return eventsTable; #region sample_using-sequence diff --git a/src/Marten/Events/EventGraph.cs b/src/Marten/Events/EventGraph.cs index 97df3f68a0..ee517498c0 100644 --- a/src/Marten/Events/EventGraph.cs +++ b/src/Marten/Events/EventGraph.cs @@ -15,6 +15,7 @@ using Marten.Util; using NpgsqlTypes; using Weasel.Core; +using Weasel.Postgresql.Tables; using static Marten.Events.EventMappingExtensions; namespace Marten.Events; @@ -217,6 +218,21 @@ public IEventStoreOptions Upcast(params IEventUpcaster[] upcasters) return this; } + private readonly IList _customEventsTableIndexes = new List(); + private readonly IList _customStreamsTableIndexes = new List(); + + public IEventStoreOptions AddIndexToEventsTable(IndexDefinition index) + { + _customEventsTableIndexes.Add(index); + return this; + } + + public IEventStoreOptions AddIndexToStreamsTable(IndexDefinition index) + { + _customStreamsTableIndexes.Add(index); + return this; + } + /// /// Override the database schema name for event related tables. By default this /// is the same schema as the document storage diff --git a/src/Marten/Events/IEventStoreOptions.cs b/src/Marten/Events/IEventStoreOptions.cs index 894ac54446..080518597f 100644 --- a/src/Marten/Events/IEventStoreOptions.cs +++ b/src/Marten/Events/IEventStoreOptions.cs @@ -7,6 +7,7 @@ using Marten.Events; using Marten.Services.Json.Transformations; using Marten.Storage; +using Weasel.Postgresql.Tables; using static Marten.Events.EventMappingExtensions; namespace Marten.Events @@ -277,6 +278,30 @@ public IEventStoreOptions Upcast( /// Upcaster type transforming ("upcasting") event JSON payload from one schema to another. /// Event store options, to allow fluent definition IEventStoreOptions Upcast() where TUpcaster : IEventUpcaster, new(); + + /// + /// + /// Registers a custom index that will be applied the events "mt_events" table. + /// + /// + /// See more in documentation + /// + /// + /// The index to add to the events table. + /// Event store options, to allow fluent definition + IEventStoreOptions AddIndexToEventsTable(IndexDefinition index); + + /// + /// + /// Registers a custom index that will be applied the events "mt_streams" table. + /// + /// + /// See more in documentation + /// + /// + /// The index to add to the events table. + /// Event store options, to allow fluent definition + IEventStoreOptions AddIndexToStreamsTable(IndexDefinition index); } } From 7330045f8e0a937883ee46e1c3ef6513e649199c Mon Sep 17 00:00:00 2001 From: Ben Edwards Date: Fri, 14 Apr 2023 15:24:03 +1000 Subject: [PATCH 2/2] fix typo in example --- docs/events/storage.md | 2 +- src/Marten.Testing/Examples/ConfiguringDocumentStore.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/events/storage.md b/docs/events/storage.md index 74be25b146..2309427dfe 100644 --- a/docs/events/storage.md +++ b/docs/events/storage.md @@ -131,7 +131,7 @@ var store = DocumentStore.For(_ => // Add an index to the mt_events table on "type" _.Events.AddIndexToEventsTable( - new IndexDefinition("idx_mt_events_is_type") + new IndexDefinition("idx_mt_events_type") .AgainstColumns("type")); }); ``` diff --git a/src/Marten.Testing/Examples/ConfiguringDocumentStore.cs b/src/Marten.Testing/Examples/ConfiguringDocumentStore.cs index 73d96a9774..1cb9351a8a 100644 --- a/src/Marten.Testing/Examples/ConfiguringDocumentStore.cs +++ b/src/Marten.Testing/Examples/ConfiguringDocumentStore.cs @@ -214,7 +214,7 @@ public void setting_event_custom_indexes() // Add an index to the mt_events table on "type" _.Events.AddIndexToEventsTable( - new IndexDefinition("idx_mt_events_is_type") + new IndexDefinition("idx_mt_events_type") .AgainstColumns("type")); }); #endregion