From 6b442a6704cd4117151a9e2174b627369dcaa31a Mon Sep 17 00:00:00 2001 From: Aleksandr Andreev Date: Mon, 12 Aug 2019 14:06:11 +0300 Subject: [PATCH 01/14] add PricingFullPagedDataSource --- .../PricingFullPagedDataSource.cs | 60 +++++++++++++++++++ .../VirtoCommerce.PricingModule.Web/Module.cs | 5 ++ 2 files changed, 65 insertions(+) create mode 100644 Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullPagedDataSource.cs diff --git a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullPagedDataSource.cs b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullPagedDataSource.cs new file mode 100644 index 000000000..e5c09ef83 --- /dev/null +++ b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullPagedDataSource.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using VirtoCommerce.ExportModule.Core.Model; +using VirtoCommerce.PricingModule.Core.Model.Search; +using VirtoCommerce.PricingModule.Core.Services; + +namespace VirtoCommerce.PricingModule.Data.ExportImport +{ + public class PricingFullPagedDataSource : IPagedDataSource + { + private readonly IPricingSearchService _searchService; + + public int PageSize { get; set; } = 50; + public int CurrentPageNumber { get; } + public ExportDataQuery DataQuery { get; set; } + + + public PricingFullPagedDataSource(IPricingSearchService searchService) + { + _searchService = searchService; + } + + public int GetTotalCount() + { + int totalCount = 0; + var pricelists = _searchService.SearchPricelistsAsync(new PricelistSearchCriteria() { Skip = 0, Take = 0 }).GetAwaiter().GetResult(); + totalCount += pricelists.TotalCount; + var assignments = _searchService.SearchPricelistAssignmentsAsync(new PricelistAssignmentsSearchCriteria() { Skip = 0, Take = 0 }).GetAwaiter().GetResult(); + totalCount += assignments.TotalCount; + var prices = _searchService.SearchPricesAsync(new PricesSearchCriteria() { Skip = 0, Take = 0 }).GetAwaiter().GetResult(); + totalCount += prices.TotalCount; + return totalCount; + + + } + + public IEnumerable FetchNextPage() + { + + + int skip = PageSize * CurrentPageNumber; + int take = PageSize; + + var pricelists = _searchService.SearchPricelistsAsync(new PricelistSearchCriteria() { Skip = skip, Take = take }).GetAwaiter().GetResult(); + + var assignments = _searchService.SearchPricelistAssignmentsAsync(new PricelistAssignmentsSearchCriteria() { Skip = skip, Take = take }).GetAwaiter().GetResult(); + + var prices = _searchService.SearchPricesAsync(new PricesSearchCriteria() { Skip = skip, Take = take }).GetAwaiter().GetResult(); + + + + } + + public ViewableSearchResult GetData() + { + throw new NotImplementedException(); + } + + } +} diff --git a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Web/Module.cs b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Web/Module.cs index 72d463bd0..4d3de9934 100644 --- a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Web/Module.cs +++ b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Web/Module.cs @@ -83,6 +83,11 @@ public void Initialize(IServiceCollection serviceCollection) serviceCollection.AddSingleton>(provider => (exportDataQuery) => CreateExportPagedDataSource(provider, exportDataQuery)); + serviceCollection.AddScoped(); + serviceCollection.AddSingleton>(provider => + (exportDataQuery) => provider.CreateScope().ServiceProvider.GetRequiredService()); + + var requirements = new IAuthorizationRequirement[] { new PermissionAuthorizationRequirement(ModuleConstants.Security.Permissions.Export), new PermissionAuthorizationRequirement(ModuleConstants.Security.Permissions.Read) From 3e086cb9158e600239e74b028738c47cf1127570 Mon Sep 17 00:00:00 2001 From: Aleksandr Andreev Date: Thu, 15 Aug 2019 14:13:14 +0300 Subject: [PATCH 02/14] update metadata getters --- .../ExportedTypeMetadataExtensions.cs | 69 ++++++++---------- .../ExportedTypeMetadataTests.cs | 73 ++++--------------- .../ExportImport2/ExportablePricingFull.cs | 13 ++-- .../PricingFullExportPagedDataSource.cs | 16 ++-- .../VirtoCommerce.PricingModule.Web/Module.cs | 3 +- 5 files changed, 63 insertions(+), 111 deletions(-) diff --git a/Modules/vc-module-export/VirtoCommerce.ExportModule.Data/Extensions/ExportedTypeMetadataExtensions.cs b/Modules/vc-module-export/VirtoCommerce.ExportModule.Data/Extensions/ExportedTypeMetadataExtensions.cs index 9dfcbcc69..ca74ac645 100644 --- a/Modules/vc-module-export/VirtoCommerce.ExportModule.Data/Extensions/ExportedTypeMetadataExtensions.cs +++ b/Modules/vc-module-export/VirtoCommerce.ExportModule.Data/Extensions/ExportedTypeMetadataExtensions.cs @@ -9,44 +9,51 @@ namespace VirtoCommerce.ExportModule.Data.Extensions { public static class ExportedTypeMetadataExtensions { - private class ExportTypePropertyInfoEx + /// + /// Returns metadata for type plain (all except: Entity, IEnumerable <Entity>) properties. + /// + /// Type for getting metadata. + /// Metadata for the T type, including all non-reference properties of type. + public static ExportedTypeMetadata GetPropertyNames(this Type type) { - public ExportedTypePropertyInfo ExportedPropertyInfo { get; set; } - public bool IsReference { get; set; } - public PropertyInfo PropertyInfo { get; set; } + var result = new ExportedTypeMetadata + { + PropertyInfos = type.GetPropertyNames(type.Name, string.Empty).ToArray() + }; + + return result; } /// - /// Returns metadata about exportable entity type. + /// Returns metadata for type nested properties. /// - /// Type for getting metadata. - /// Property full paths to include to metadata - /// Metadata for the T type, including all non-reference properties of types: T and corresponding to the passed properties. - public static ExportedTypeMetadata GetPropertyNames(this Type type, params string[] propertyPathsToInclude) + /// Type for getting metadata. + /// Property paths, e.g. PropertyB.PropertyC + /// Metadata with the nested property's property paths, e.g. [{ FullName: Id, Group : 'PropertyB.PropertyC' }, ...] + public static ExportedTypeMetadata GetNestedPropertyNames(this Type type, params string[] propertyPaths) { var result = new ExportedTypeMetadata { - PropertyInfos = GetPropertyNames(type, type.Name, string.Empty, propertyPathsToInclude) - .Where(x => !x.IsReference) - .Select(x => x.ExportedPropertyInfo) - .ToArray() + PropertyInfos = propertyPaths.SelectMany(x => + type.GetPropertyType(x).GetPropertyNames(x, string.Empty)) + .ToArray() }; return result; } - private static Type GetPropertyType(Type type, string propertyName) + private static Type GetPropertyType(this Type type, string propertyPath) { - return GetPropertyType(type, propertyName.Split('.')); + return type.GetPropertyType(propertyPath.Split('.')); } - private static Type GetPropertyType(Type type, IEnumerable propertyNames) + private static Type GetPropertyType(this Type type, IEnumerable propertyNames) { Type result; var nestedType = GetNestedType(type); if (propertyNames.Any()) { - result = GetPropertyType(nestedType.GetProperty(propertyNames.First()).PropertyType, propertyNames.Skip(1)); + result = nestedType.GetProperty(propertyNames.First()).PropertyType.GetPropertyType(propertyNames.Skip(1)); } else { @@ -55,9 +62,9 @@ private static Type GetPropertyType(Type type, IEnumerable propertyNames return result; } - private static ExportTypePropertyInfoEx[] GetPropertyNames(Type type, string groupName, string baseMemberName, string[] propertyPathsToInclude) + private static ExportedTypePropertyInfo[] GetPropertyNames(this Type type, string groupName, string baseMemberName) { - var result = new List(); + var result = new List(); var properties = type.GetProperties(BindingFlags.Instance | BindingFlags.Public).Where(x => x.CanRead && x.CanWrite); foreach (var propertyInfo in properties) @@ -68,34 +75,20 @@ private static ExportTypePropertyInfoEx[] GetPropertyNames(Type type, string gro if (!isNested) { - result.Add(new ExportTypePropertyInfoEx() + result.Add(new ExportedTypePropertyInfo() { - ExportedPropertyInfo = new ExportedTypePropertyInfo - { - FullName = memberName, - DisplayName = memberName, - Group = groupName, - }, - PropertyInfo = propertyInfo, - IsReference = isNested, + FullName = memberName, + DisplayName = memberName, + Group = groupName, }); } } - //Continue searching for members in every property path - foreach (var propertyPathToInclude in propertyPathsToInclude) - { - result.AddRange(GetPropertyNames( - GetPropertyType(type, propertyPathToInclude), - string.Format($@"{groupName}.{propertyPathToInclude}"), - propertyPathToInclude, - new string[] { })); - } return result.ToArray(); } /// - /// Adds baseName as a prefixe to the property name (i.e. "{baseName}.{Name}") + /// Adds baseName as a prefix to the property name (i.e. "{baseName}.{Name}") /// /// /// diff --git a/Modules/vc-module-export/VirtoCommerce.ExportModule.Tests/ExportedTypeMetadataTests.cs b/Modules/vc-module-export/VirtoCommerce.ExportModule.Tests/ExportedTypeMetadataTests.cs index 31f7e739d..48eb2b120 100644 --- a/Modules/vc-module-export/VirtoCommerce.ExportModule.Tests/ExportedTypeMetadataTests.cs +++ b/Modules/vc-module-export/VirtoCommerce.ExportModule.Tests/ExportedTypeMetadataTests.cs @@ -8,41 +8,10 @@ namespace VirtoCommerce.ExportModule.Tests public class ExportedTypeMetadataTests { [Fact] - public Task GetFromType_Pricelist_NameBuiltCorrectly() + public Task GetPropertyNames_Plain_BuiltCorrectly() { - var metadata = typeof(Pricelist).GetPropertyNames(nameof(Pricelist.Prices), nameof(Pricelist.Assignments)); - var props = metadata.PropertyInfos.Select(x => x.FullName); - - // Check if all own property detected - Assert.Contains("Name", props); - Assert.Contains("Description", props); - Assert.Contains("Currency", props); - Assert.Contains("CreatedDate", props); - Assert.Contains("ModifiedDate", props); - Assert.Contains("CreatedBy", props); - Assert.Contains("ModifiedBy", props); - Assert.Contains("Id", props); - - // Reference properties - Assert.Contains("Prices.Id", props); - Assert.Contains("Prices.PricelistId", props); - Assert.Contains("Assignments.Id", props); - Assert.Contains("Assignments.PricelistId", props); - - // Check it doesn't contains recursive links - Assert.DoesNotContain("Prices.Pricelist", props); - Assert.DoesNotContain("Prices.Pricelist.Id", props); - Assert.DoesNotContain("Assignments.Pricelist", props); - Assert.DoesNotContain("Assignments", props); - - return Task.CompletedTask; - } - - [Fact] - public Task GetFromType_Pricelist_ExportNameBuiltCorrectly() - { - var metadata = typeof(Pricelist).GetPropertyNames(nameof(Pricelist.Prices), nameof(Pricelist.Assignments)); - var props = metadata.PropertyInfos.Select(x => x.DisplayName); + var metadata = typeof(Pricelist).GetPropertyNames(); + var props = metadata.PropertyInfos.Select(x => x.FullName).ToArray(); // Check if all own property detected Assert.Contains("Name", props); @@ -54,44 +23,32 @@ public Task GetFromType_Pricelist_ExportNameBuiltCorrectly() Assert.Contains("ModifiedBy", props); Assert.Contains("Id", props); - // Reference properties - Assert.Contains("Prices.Id", props); - Assert.Contains("Prices.PricelistId", props); - Assert.Contains("Assignments.Id", props); - Assert.Contains("Assignments.PricelistId", props); - - // Check it doesn't contains recursive links - Assert.DoesNotContain("Prices.Pricelist", props); - Assert.DoesNotContain("Prices.Pricelist.Id", props); - Assert.DoesNotContain("Assignments.Pricelist", props); - Assert.DoesNotContain("Assignments", props); + Assert.Equal(8, props.Length); return Task.CompletedTask; } - [Fact] - public Task GetFromType_Pricelist_WithoutReferences_BuiltCorrectly() + public Task GetNestedPropertyNames_BuiltCorrectly() { - var metadata = typeof(Pricelist).GetPropertyNames(); - var props = metadata.PropertyInfos.Select(x => x.FullName); + var metadata = typeof(Pricelist).GetNestedPropertyNames(nameof(Pricelist.Prices)); + var props = metadata.PropertyInfos.Select(x => x.FullName).ToArray(); - // Check if all own property detected - Assert.Contains("Name", props); - Assert.Contains("Description", props); + Assert.Contains("PricelistId", props); Assert.Contains("Currency", props); + Assert.Contains("ProductId", props); + Assert.Contains("Sale", props); + Assert.Contains("List", props); + Assert.Contains("MinQuantity", props); + Assert.Contains("StartDate", props); + Assert.Contains("EndDate", props); Assert.Contains("CreatedDate", props); Assert.Contains("ModifiedDate", props); Assert.Contains("CreatedBy", props); Assert.Contains("ModifiedBy", props); Assert.Contains("Id", props); - // And does not contain nested properties - Assert.DoesNotContain("Prices", props); - Assert.DoesNotContain("Prices.Id", props); - Assert.DoesNotContain("Prices.PricelistId", props); - Assert.DoesNotContain("Assignments.Id", props); - Assert.DoesNotContain("Assignments.PricelistId", props); + Assert.Equal(13, props.Length); return Task.CompletedTask; } diff --git a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/ExportablePricingFull.cs b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/ExportablePricingFull.cs index fa90e7942..b24147053 100644 --- a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/ExportablePricingFull.cs +++ b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/ExportablePricingFull.cs @@ -1,7 +1,6 @@ using System.Collections.Generic; using System.Linq; using VirtoCommerce.ExportModule.Core.Model; -using VirtoCommerce.PricingModule.Core.Model; namespace VirtoCommerce.PricingModule.Data.ExportImport { @@ -13,9 +12,9 @@ public class ExportablePricingFull : IExportable public string Parent { get; set; } public string Type { get; set; } - public ICollection Pricelists { get; set; } - public ICollection Prices { get; set; } - public ICollection Assignments { get; set; } + public ICollection Pricelists { get; set; } + public ICollection Prices { get; set; } + public ICollection Assignments { get; set; } public object Clone() @@ -23,15 +22,15 @@ public object Clone() var result = MemberwiseClone() as ExportablePricingFull; if (Pricelists != null) { - result.Pricelists = new List(Pricelists.Select(x => x.Clone() as Pricelist)); + result.Pricelists = new List(Pricelists.Select(x => x.Clone() as ExportablePricelist)); } if (Prices != null) { - result.Prices = new List(Prices.Select(x => x.Clone() as Price)); + result.Prices = new List(Prices.Select(x => x.Clone() as ExportablePrice)); } if (Assignments != null) { - result.Assignments = new List(Assignments.Select(x => x.Clone() as PricelistAssignment)); + result.Assignments = new List(Assignments.Select(x => x.Clone() as ExportablePricelistAssignment)); } return result; diff --git a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs index 45f3440f0..0e49c3036 100644 --- a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs +++ b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs @@ -20,9 +20,9 @@ public ExportDataSourceState() public ExportDataQuery DataQuery; public IEnumerable Result; public Func DataSourceFactory; - public int GetNextTake(int pageSize) + public int GetNextTake(int take, int pageSize) { - return pageSize - TotalCount - ReceivedCount < 0 ? pageSize : pageSize - TotalCount - ReceivedCount; + return take - TotalCount - ReceivedCount < 0 ? pageSize : take - TotalCount - ReceivedCount; } } @@ -83,17 +83,21 @@ public virtual IEnumerable FetchNextPage() state.DataQuery.Take = takeNext; state.DataQuery.Skip = CurrentPageNumber * PageSize; taskList.Add(Task.Factory.StartNew(() => { state.Result = state.DataSourceFactory(state.DataQuery).FetchNextPage().ToArray(); })); - takeNext = state.GetNextTake(PageSize); + takeNext = state.GetNextTake(takeNext, PageSize); } } Task.WhenAll(taskList).GetAwaiter().GetResult(); - var result = new List(); + List result = null; foreach (var state in _exportDataSourceStates) { - result.AddRange(state.Result); - state.ReceivedCount = state.Result.Count(); + if (result == null && state.Result.Any()) + { + result = new List(); + } + result?.AddRange(state.Result); + state.ReceivedCount += state.Result.Count(); } CurrentPageNumber++; diff --git a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Web/Module.cs b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Web/Module.cs index b1231605c..f53da2a5a 100644 --- a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Web/Module.cs +++ b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Web/Module.cs @@ -229,10 +229,9 @@ public void PostInitialize(IApplicationBuilder appBuilder) registrar.RegisterType( new ExportedTypeDefinitionBuilder($@"{typeof(ExportablePricingFull).FullName}", typeof(ExportablePricingFull).Namespace, typeof(PricingFullExportDataQuery).Name) .WithDataSourceFactory(dataQuery => pricingFullExportPagedDataSourceFactory(dataQuery)) - .WithMetadata(typeof(ExportablePricingFull).GetPropertyNames(true)) + .WithMetadata(typeof(ExportablePricingFull).GetNestedPropertyNames(nameof(ExportablePricingFull.Prices), nameof(ExportablePricingFull.Pricelists), nameof(ExportablePricingFull.Assignments))) .ExportedTypeDefinition); - AbstractTypeFactory.RegisterType(); AbstractTypeFactory.RegisterType(); AbstractTypeFactory.RegisterType(); From 2092f40ae0fcdc459d09aa58da0bdfaae01e0e67 Mon Sep 17 00:00:00 2001 From: Aleksandr Andreev Date: Thu, 15 Aug 2019 15:03:43 +0300 Subject: [PATCH 03/14] remove baseMemberName --- .../Extensions/ExportedTypeMetadataExtensions.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/vc-module-export/VirtoCommerce.ExportModule.Data/Extensions/ExportedTypeMetadataExtensions.cs b/Modules/vc-module-export/VirtoCommerce.ExportModule.Data/Extensions/ExportedTypeMetadataExtensions.cs index ca74ac645..6ef16a08f 100644 --- a/Modules/vc-module-export/VirtoCommerce.ExportModule.Data/Extensions/ExportedTypeMetadataExtensions.cs +++ b/Modules/vc-module-export/VirtoCommerce.ExportModule.Data/Extensions/ExportedTypeMetadataExtensions.cs @@ -18,7 +18,7 @@ public static ExportedTypeMetadata GetPropertyNames(this Type type) { var result = new ExportedTypeMetadata { - PropertyInfos = type.GetPropertyNames(type.Name, string.Empty).ToArray() + PropertyInfos = type.GetPropertyNames(type.Name).ToArray() }; return result; @@ -35,7 +35,7 @@ public static ExportedTypeMetadata GetNestedPropertyNames(this Type type, params var result = new ExportedTypeMetadata { PropertyInfos = propertyPaths.SelectMany(x => - type.GetPropertyType(x).GetPropertyNames(x, string.Empty)) + type.GetPropertyType(x).GetPropertyNames(x)) .ToArray() }; @@ -62,7 +62,7 @@ private static Type GetPropertyType(this Type type, IEnumerable property return result; } - private static ExportedTypePropertyInfo[] GetPropertyNames(this Type type, string groupName, string baseMemberName) + private static ExportedTypePropertyInfo[] GetPropertyNames(this Type type, string groupName) { var result = new List(); var properties = type.GetProperties(BindingFlags.Instance | BindingFlags.Public).Where(x => x.CanRead && x.CanWrite); @@ -71,7 +71,7 @@ private static ExportedTypePropertyInfo[] GetPropertyNames(this Type type, strin { var nestedType = GetNestedType(propertyInfo.PropertyType); var isNested = nestedType.IsSubclassOf(typeof(Entity)); - var memberName = propertyInfo.GetDerivedName(baseMemberName); + var memberName = propertyInfo.Name; if (!isNested) { From b3288f970325990cf0037dffd70a4d8e08167e7b Mon Sep 17 00:00:00 2001 From: Aleksandr Andreev Date: Thu, 15 Aug 2019 16:18:06 +0300 Subject: [PATCH 04/14] get rid of IF --- .../ExportImport2/ExportablePricingFull.cs | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/ExportablePricingFull.cs b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/ExportablePricingFull.cs index b24147053..f2f32e074 100644 --- a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/ExportablePricingFull.cs +++ b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/ExportablePricingFull.cs @@ -20,21 +20,11 @@ public class ExportablePricingFull : IExportable public object Clone() { var result = MemberwiseClone() as ExportablePricingFull; - if (Pricelists != null) - { - result.Pricelists = new List(Pricelists.Select(x => x.Clone() as ExportablePricelist)); - } - if (Prices != null) - { - result.Prices = new List(Prices.Select(x => x.Clone() as ExportablePrice)); - } - if (Assignments != null) - { - result.Assignments = new List(Assignments.Select(x => x.Clone() as ExportablePricelistAssignment)); - } - return result; + result.Pricelists = Pricelists?.Select(x => x.Clone() as ExportablePricelist).ToList(); + result.Prices = Prices?.Select(x => x.Clone() as ExportablePrice).ToList(); + result.Assignments = Assignments?.Select(x => x.Clone() as ExportablePricelistAssignment).ToList(); + return result; } - } } From b3c25484b23124dba412019ce33055452f04ac6f Mon Sep 17 00:00:00 2001 From: Aleksandr Andreev Date: Thu, 15 Aug 2019 16:21:37 +0300 Subject: [PATCH 05/14] rid of GetNextTake method. --- .../ExportImport2/PricingFullExportPagedDataSource.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs index 0e49c3036..dda92715e 100644 --- a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs +++ b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs @@ -20,10 +20,6 @@ public ExportDataSourceState() public ExportDataQuery DataQuery; public IEnumerable Result; public Func DataSourceFactory; - public int GetNextTake(int take, int pageSize) - { - return take - TotalCount - ReceivedCount < 0 ? pageSize : take - TotalCount - ReceivedCount; - } } public int TotalCount { get; set; } @@ -83,7 +79,7 @@ public virtual IEnumerable FetchNextPage() state.DataQuery.Take = takeNext; state.DataQuery.Skip = CurrentPageNumber * PageSize; taskList.Add(Task.Factory.StartNew(() => { state.Result = state.DataSourceFactory(state.DataQuery).FetchNextPage().ToArray(); })); - takeNext = state.GetNextTake(takeNext, PageSize); + takeNext = takeNext - state.TotalCount - state.ReceivedCount < 0 ? PageSize : takeNext - state.TotalCount - state.ReceivedCount; } } From 329f26169516cecf667850ccfca81a9d3440e754 Mon Sep 17 00:00:00 2001 From: Aleksandr Andreev Date: Fri, 16 Aug 2019 08:48:01 +0300 Subject: [PATCH 06/14] refactoring portion calculation --- .../PricingFullExportPagedDataSource.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs index dda92715e..cab51b2c4 100644 --- a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs +++ b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs @@ -68,18 +68,20 @@ public int GetTotalCount() public virtual IEnumerable FetchNextPage() { - int takeNext = PageSize; + int batchSize = PageSize; var taskList = new List(); foreach (var state in _exportDataSourceStates) { state.Result = Array.Empty(); - if (state.ReceivedCount < state.TotalCount) + + if (state.ReceivedCount < state.TotalCount && batchSize > 0) { - state.DataQuery.Take = takeNext; - state.DataQuery.Skip = CurrentPageNumber * PageSize; + var portion = state.TotalCount - state.ReceivedCount > batchSize ? batchSize : state.TotalCount - state.ReceivedCount; + state.DataQuery.Take = portion; + state.DataQuery.Skip = state.ReceivedCount; taskList.Add(Task.Factory.StartNew(() => { state.Result = state.DataSourceFactory(state.DataQuery).FetchNextPage().ToArray(); })); - takeNext = takeNext - state.TotalCount - state.ReceivedCount < 0 ? PageSize : takeNext - state.TotalCount - state.ReceivedCount; + batchSize = batchSize - portion; } } From e4bdf14db2a3cb8b9e82bf4387f39ba20f581699 Mon Sep 17 00:00:00 2001 From: Aleksandr Andreev Date: Fri, 16 Aug 2019 11:51:34 +0300 Subject: [PATCH 07/14] change "== null" to "IsNullOrEmpty" --- .../VirtoCommerce.ExportModule.Data/Services/DataExporter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/vc-module-export/VirtoCommerce.ExportModule.Data/Services/DataExporter.cs b/Modules/vc-module-export/VirtoCommerce.ExportModule.Data/Services/DataExporter.cs index 69ed579ef..848c97a4b 100644 --- a/Modules/vc-module-export/VirtoCommerce.ExportModule.Data/Services/DataExporter.cs +++ b/Modules/vc-module-export/VirtoCommerce.ExportModule.Data/Services/DataExporter.cs @@ -67,7 +67,7 @@ public void Export(Stream stream, ExportDataRequest request, Action Date: Fri, 16 Aug 2019 11:52:45 +0300 Subject: [PATCH 08/14] use AbstractTypeFactory use master DataSourse .Take and .Skip Ensure have total count --- .../PricingFullExportPagedDataSource.cs | 60 +++++++++++++------ 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs index cab51b2c4..7cd286786 100644 --- a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs +++ b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using VirtoCommerce.ExportModule.Core.Model; +using VirtoCommerce.Platform.Core.Common; namespace VirtoCommerce.PricingModule.Data.ExportImport { @@ -18,7 +19,7 @@ public ExportDataSourceState() public int TotalCount; public int ReceivedCount; public ExportDataQuery DataQuery; - public IEnumerable Result; + public IExportable[] Result; public Func DataSourceFactory; } @@ -66,10 +67,39 @@ public int GetTotalCount() return TotalCount; } + private void EnsureHaveTotals() + { + if (TotalCount < 0) + { + CalculateCounts(); + } + } + public virtual IEnumerable FetchNextPage() { - int batchSize = PageSize; + EnsureHaveTotals(); + var result = new List(); + + int batchSize = _dataQuery.Take ?? PageSize; var taskList = new List(); + int skip = _dataQuery.Skip ?? CurrentPageNumber * PageSize; + + if (skip > 0) + { + foreach (var state in _exportDataSourceStates) + { + if (state.TotalCount < skip) + { + state.ReceivedCount = state.TotalCount; + skip -= state.TotalCount; + } + else + { + state.ReceivedCount = skip; + skip = 0; + } + } + } foreach (var state in _exportDataSourceStates) { @@ -77,25 +107,20 @@ public virtual IEnumerable FetchNextPage() if (state.ReceivedCount < state.TotalCount && batchSize > 0) { - var portion = state.TotalCount - state.ReceivedCount > batchSize ? batchSize : state.TotalCount - state.ReceivedCount; - state.DataQuery.Take = portion; + var portionCount = state.TotalCount - state.ReceivedCount > batchSize ? batchSize : state.TotalCount - state.ReceivedCount; + state.DataQuery.Take = portionCount; state.DataQuery.Skip = state.ReceivedCount; taskList.Add(Task.Factory.StartNew(() => { state.Result = state.DataSourceFactory(state.DataQuery).FetchNextPage().ToArray(); })); - batchSize = batchSize - portion; + batchSize = batchSize - portionCount; } } Task.WhenAll(taskList).GetAwaiter().GetResult(); - List result = null; foreach (var state in _exportDataSourceStates) { - if (result == null && state.Result.Any()) - { - result = new List(); - } - result?.AddRange(state.Result); - state.ReceivedCount += state.Result.Count(); + result.AddRange(state.Result); + state.ReceivedCount += state.Result.Length; } CurrentPageNumber++; @@ -121,12 +146,11 @@ private void CalculateCounts() } private T BuildExportDataQuery() where T : ExportDataQuery, new() { - var newExportDataQuery = new T(); - newExportDataQuery.Skip = _dataQuery.Skip; - newExportDataQuery.Take = _dataQuery.Take; - newExportDataQuery.ObjectIds = _dataQuery.ObjectIds; - newExportDataQuery.Sort = _dataQuery.Sort; - return newExportDataQuery; + var result = AbstractTypeFactory.TryCreateInstance(); + + result.ObjectIds = _dataQuery.ObjectIds; + result.Sort = _dataQuery.Sort; + return result; } } } From 0e476b0629709e94af24b6ce7e065eb7f34b4ec9 Mon Sep 17 00:00:00 2001 From: Aleksandr Andreev Date: Fri, 16 Aug 2019 11:54:20 +0300 Subject: [PATCH 09/14] revert --- .../VirtoCommerce.SitemapsModule.Data.csproj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Modules/vc-module-sitemaps/VirtoCommerce.SitemapsModule.Data/VirtoCommerce.SitemapsModule.Data.csproj b/Modules/vc-module-sitemaps/VirtoCommerce.SitemapsModule.Data/VirtoCommerce.SitemapsModule.Data.csproj index 56a85f978..ae4efae14 100644 --- a/Modules/vc-module-sitemaps/VirtoCommerce.SitemapsModule.Data/VirtoCommerce.SitemapsModule.Data.csproj +++ b/Modules/vc-module-sitemaps/VirtoCommerce.SitemapsModule.Data/VirtoCommerce.SitemapsModule.Data.csproj @@ -24,6 +24,9 @@ + + + From a21da01d03666826db2f055c8e92472e33cabd3f Mon Sep 17 00:00:00 2001 From: Aleksandr Andreev Date: Fri, 16 Aug 2019 11:57:52 +0300 Subject: [PATCH 10/14] revert --- .../VirtoCommerce.SitemapsModule.Data.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/vc-module-sitemaps/VirtoCommerce.SitemapsModule.Data/VirtoCommerce.SitemapsModule.Data.csproj b/Modules/vc-module-sitemaps/VirtoCommerce.SitemapsModule.Data/VirtoCommerce.SitemapsModule.Data.csproj index ae4efae14..242c22dcd 100644 --- a/Modules/vc-module-sitemaps/VirtoCommerce.SitemapsModule.Data/VirtoCommerce.SitemapsModule.Data.csproj +++ b/Modules/vc-module-sitemaps/VirtoCommerce.SitemapsModule.Data/VirtoCommerce.SitemapsModule.Data.csproj @@ -24,8 +24,8 @@ - - + + From dc1d03a413c7034d1daf1da491cc9e433740a7ed Mon Sep 17 00:00:00 2001 From: Aleksandr Andreev Date: Fri, 16 Aug 2019 12:22:03 +0300 Subject: [PATCH 11/14] revert --- .../VirtoCommerce.SitemapsModule.Data.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/vc-module-sitemaps/VirtoCommerce.SitemapsModule.Data/VirtoCommerce.SitemapsModule.Data.csproj b/Modules/vc-module-sitemaps/VirtoCommerce.SitemapsModule.Data/VirtoCommerce.SitemapsModule.Data.csproj index 242c22dcd..c5663dd24 100644 --- a/Modules/vc-module-sitemaps/VirtoCommerce.SitemapsModule.Data/VirtoCommerce.SitemapsModule.Data.csproj +++ b/Modules/vc-module-sitemaps/VirtoCommerce.SitemapsModule.Data/VirtoCommerce.SitemapsModule.Data.csproj @@ -29,4 +29,4 @@ - + \ No newline at end of file From e081832e6e7459397916523d24cf1fe66738b9bb Mon Sep 17 00:00:00 2001 From: Aleksandr Andreev Date: Fri, 16 Aug 2019 13:56:39 +0300 Subject: [PATCH 12/14] FetchNextPage refactoring --- .../PricingFullExportPagedDataSource.cs | 34 +++++-------------- .../VirtoCommerce.PricingModule.Web/Module.cs | 5 ++- 2 files changed, 10 insertions(+), 29 deletions(-) diff --git a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs index 7cd286786..0b233b3ad 100644 --- a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs +++ b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs @@ -17,7 +17,6 @@ public ExportDataSourceState() } public int TotalCount; - public int ReceivedCount; public ExportDataQuery DataQuery; public IExportable[] Result; public Func DataSourceFactory; @@ -84,32 +83,19 @@ public virtual IEnumerable FetchNextPage() var taskList = new List(); int skip = _dataQuery.Skip ?? CurrentPageNumber * PageSize; - if (skip > 0) - { - foreach (var state in _exportDataSourceStates) - { - if (state.TotalCount < skip) - { - state.ReceivedCount = state.TotalCount; - skip -= state.TotalCount; - } - else - { - state.ReceivedCount = skip; - skip = 0; - } - } - } - foreach (var state in _exportDataSourceStates) { state.Result = Array.Empty(); - if (state.ReceivedCount < state.TotalCount && batchSize > 0) + if (state.TotalCount < skip) { - var portionCount = state.TotalCount - state.ReceivedCount > batchSize ? batchSize : state.TotalCount - state.ReceivedCount; + skip -= state.TotalCount; + } + else + { + var portionCount = state.TotalCount - skip > batchSize ? batchSize : state.TotalCount - skip; state.DataQuery.Take = portionCount; - state.DataQuery.Skip = state.ReceivedCount; + state.DataQuery.Skip = skip; taskList.Add(Task.Factory.StartNew(() => { state.Result = state.DataSourceFactory(state.DataQuery).FetchNextPage().ToArray(); })); batchSize = batchSize - portionCount; } @@ -117,11 +103,7 @@ public virtual IEnumerable FetchNextPage() Task.WhenAll(taskList).GetAwaiter().GetResult(); - foreach (var state in _exportDataSourceStates) - { - result.AddRange(state.Result); - state.ReceivedCount += state.Result.Length; - } + result.AddRange(_exportDataSourceStates.SelectMany(x => x.Result)); CurrentPageNumber++; return result; diff --git a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Web/Module.cs b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Web/Module.cs index e4d3d669e..8591c0c45 100644 --- a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Web/Module.cs +++ b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Web/Module.cs @@ -224,10 +224,9 @@ public void PostInitialize(IApplicationBuilder appBuilder) .WithTabularMetadata(typeof(TabularPricelistAssignment).GetPropertyNames())); registrar.RegisterType( - new ExportedTypeDefinitionBuilder($@"{typeof(ExportablePricingFull).FullName}", typeof(ExportablePricingFull).Namespace, typeof(PricingFullExportDataQuery).Name) + ExportedTypeDefinitionBuilder.Build() .WithDataSourceFactory(dataQuery => pricingFullExportPagedDataSourceFactory(dataQuery)) - .WithMetadata(typeof(ExportablePricingFull).GetNestedPropertyNames(nameof(ExportablePricingFull.Prices), nameof(ExportablePricingFull.Pricelists), nameof(ExportablePricingFull.Assignments))) - .ExportedTypeDefinition); + .WithMetadata(typeof(ExportablePricingFull).GetNestedPropertyNames(nameof(ExportablePricingFull.Prices), nameof(ExportablePricingFull.Pricelists), nameof(ExportablePricingFull.Assignments)))); AbstractTypeFactory.RegisterType(); AbstractTypeFactory.RegisterType(); From 3a9556241ae0bd1aefa4d1d3a2fb819231b0de30 Mon Sep 17 00:00:00 2001 From: Aleksandr Andreev Date: Fri, 16 Aug 2019 14:03:38 +0300 Subject: [PATCH 13/14] FetchNextPage refactoring --- .../ExportImport2/PricingFullExportPagedDataSource.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs index 0b233b3ad..c0c869dd0 100644 --- a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs +++ b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/PricingFullExportPagedDataSource.cs @@ -77,11 +77,10 @@ private void EnsureHaveTotals() public virtual IEnumerable FetchNextPage() { EnsureHaveTotals(); - var result = new List(); - - int batchSize = _dataQuery.Take ?? PageSize; - var taskList = new List(); + int take = _dataQuery.Take ?? PageSize; int skip = _dataQuery.Skip ?? CurrentPageNumber * PageSize; + var taskList = new List(); + var result = new List(); foreach (var state in _exportDataSourceStates) { @@ -93,11 +92,11 @@ public virtual IEnumerable FetchNextPage() } else { - var portionCount = state.TotalCount - skip > batchSize ? batchSize : state.TotalCount - skip; + var portionCount = state.TotalCount - skip > take ? take : state.TotalCount - skip; state.DataQuery.Take = portionCount; state.DataQuery.Skip = skip; taskList.Add(Task.Factory.StartNew(() => { state.Result = state.DataSourceFactory(state.DataQuery).FetchNextPage().ToArray(); })); - batchSize = batchSize - portionCount; + take = take - portionCount; } } From 7eabdc903db91cc155ac7ddad755aee74340b2a2 Mon Sep 17 00:00:00 2001 From: Aleksandr Andreev Date: Mon, 19 Aug 2019 08:38:03 +0300 Subject: [PATCH 14/14] ExportablePricelist -> Pricelist --- .../ExportImport2/ExportablePricingFull.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/ExportablePricingFull.cs b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/ExportablePricingFull.cs index f2f32e074..c300d4b8b 100644 --- a/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/ExportablePricingFull.cs +++ b/Modules/vc-module-pricing/VirtoCommerce.PricingModule.Data/ExportImport2/ExportablePricingFull.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Linq; using VirtoCommerce.ExportModule.Core.Model; +using VirtoCommerce.PricingModule.Core.Model; namespace VirtoCommerce.PricingModule.Data.ExportImport { @@ -12,17 +13,17 @@ public class ExportablePricingFull : IExportable public string Parent { get; set; } public string Type { get; set; } - public ICollection Pricelists { get; set; } - public ICollection Prices { get; set; } - public ICollection Assignments { get; set; } + public ICollection Pricelists { get; set; } + public ICollection Prices { get; set; } + public ICollection Assignments { get; set; } public object Clone() { var result = MemberwiseClone() as ExportablePricingFull; - result.Pricelists = Pricelists?.Select(x => x.Clone() as ExportablePricelist).ToList(); - result.Prices = Prices?.Select(x => x.Clone() as ExportablePrice).ToList(); - result.Assignments = Assignments?.Select(x => x.Clone() as ExportablePricelistAssignment).ToList(); + result.Pricelists = Pricelists?.Select(x => x.Clone() as Pricelist).ToList(); + result.Prices = Prices?.Select(x => x.Clone() as Price).ToList(); + result.Assignments = Assignments?.Select(x => x.Clone() as PricelistAssignment).ToList(); return result; }