From 4293080e4028deb3034f91eb6b32346c6a97e3d5 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Wed, 7 Aug 2024 14:00:02 +0200 Subject: [PATCH 1/3] Added new add-in to fail the job when endpoint is busy. --- ...taIntegration.Providers.ODataProvider.csproj | 2 +- src/ODataProvider.cs | 17 ++++++++++++++++- src/ODataSourceReader.cs | 13 +++++++++++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj b/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj index ef42dce..c708fdf 100644 --- a/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj +++ b/src/Dynamicweb.DataIntegration.Providers.ODataProvider.csproj @@ -1,6 +1,6 @@  - 10.6.0 + 10.6.1 1.0.0.0 OData Provider The Odata Provider lets you fetch and map data from or to any OData endpoint. diff --git a/src/ODataProvider.cs b/src/ODataProvider.cs index 55e12d6..1500491 100644 --- a/src/ODataProvider.cs +++ b/src/ODataProvider.cs @@ -102,6 +102,12 @@ public string EndpointId [AddInParameterSection("Advanced activity settings")] public string DeltaModifier { get; set; } + [AddInParameter("Fail job if endpoint is busy or down")] + [AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=If endpoint is busy or down doing the run time, it will not insert already imported rows.")] + [AddInParameterGroup("Source")] + [AddInParameterSection("Advanced activity settings")] + public bool FailJobOnEndpointIsBusy { get; set; } + #endregion #region AddInManager/ConfigurableAddIn Destination @@ -433,7 +439,7 @@ public override ISourceReader GetReader(Mapping mapping) DoNotStoreLastResponseInLogFile = false; } - return new ODataSourceReader(new HttpRestClient(_credentials, RequestTimeout), Logger, mapping, _endpoint, Mode, DeltaModifier, MaximumPageSize, RunLastRequest, RequestIntervals, DoNotStoreLastResponseInLogFile); + return new ODataSourceReader(new HttpRestClient(_credentials, RequestTimeout), Logger, mapping, _endpoint, Mode, DeltaModifier, MaximumPageSize, RunLastRequest, RequestIntervals, DoNotStoreLastResponseInLogFile, FailJobOnEndpointIsBusy); } /// @@ -515,6 +521,7 @@ public override void UpdateSourceSettings(ISource source) RequestIntervals = newProvider.RequestIntervals; DoNotStoreLastResponseInLogFile = newProvider.DoNotStoreLastResponseInLogFile; EndpointId = newProvider.EndpointId; + FailJobOnEndpointIsBusy = newProvider.FailJobOnEndpointIsBusy; } public ODataProvider() { } @@ -588,6 +595,12 @@ public ODataProvider(XmlNode xmlNode) : this() ContinueOnError = node.FirstChild.Value == "True"; } break; + case "Failjobonendpointisbusy": + if (node.HasChildNodes) + { + FailJobOnEndpointIsBusy = node.FirstChild.Value == "True"; + } + break; } } } @@ -607,6 +620,7 @@ public override string Serialize() root.Add(CreateParameterNode(GetType(), "Predefined endpoint", EndpointId)); root.Add(CreateParameterNode(GetType(), "Destination endpoint", DestinationEndpointId)); root.Add(CreateParameterNode(GetType(), "Continue on error", ContinueOnError.ToString())); + root.Add(CreateParameterNode(GetType(), "Fail job if endpoint is busy or down", FailJobOnEndpointIsBusy.ToString())); return document.ToString(); } @@ -623,6 +637,7 @@ public override void SaveAsXml(XmlTextWriter textWriter) textWriter.WriteElementString("Predefinedendpoint", EndpointId); textWriter.WriteElementString("Destinationendpoint", DestinationEndpointId); textWriter.WriteElementString("Continueonerror", ContinueOnError.ToString()); + textWriter.WriteElementString("Failjobonendpointisbusy", FailJobOnEndpointIsBusy.ToString()); GetSchema().SaveAsXml(textWriter); } diff --git a/src/ODataSourceReader.cs b/src/ODataSourceReader.cs index 0b694f6..b3128bb 100644 --- a/src/ODataSourceReader.cs +++ b/src/ODataSourceReader.cs @@ -44,6 +44,7 @@ internal class ODataSourceReader : ISourceReader private bool _requestTimedOutFromGlobalSettings; private readonly int _maximumCharacterLengthOfAutoAddedSelectStatement = 1250; private readonly int _timeoutInMilliseconds; + private readonly bool _failJobOnEndpointIsBusy; internal void SaveRequestResponseFile() { @@ -95,7 +96,7 @@ private void DeleteHighWaterMarkFile() /// The mapping. /// The endpoint. /// Name of the next pagination URL. "odata.nextLink" (case insensitive) is supposed to be a standard. - internal ODataSourceReader(IHttpRestClient httpRestClient, ILogger logger, Mapping mapping, Endpoint endpoint, string mode, string deltaModifier, int maximumPageSize, bool readFromLastRequestResponse, int requestIntervals, bool doNotStoreLastResponseInLogFile, string nextPaginationUrlName = "odata.nextLink") + internal ODataSourceReader(IHttpRestClient httpRestClient, ILogger logger, Mapping mapping, Endpoint endpoint, string mode, string deltaModifier, int maximumPageSize, bool readFromLastRequestResponse, int requestIntervals, bool doNotStoreLastResponseInLogFile, bool failJobOnEndpointIsBusy, string nextPaginationUrlName = "odata.nextLink") { _totalResponseResult = new List>(); _httpRestClient = httpRestClient; @@ -109,6 +110,7 @@ internal ODataSourceReader(IHttpRestClient httpRestClient, ILogger logger, Mappi _requestIntervals = requestIntervals; _doNotStoreLastResponseInLogFile = doNotStoreLastResponseInLogFile; _timeoutInMilliseconds = GetTimeOutInMilliseconds(); + _failJobOnEndpointIsBusy = failJobOnEndpointIsBusy; string logFileName = Scheduling.Task.MakeSafeFileName(mapping.Job.Name) + $"_{_mapping.SourceTable.Name}.log"; IDictionary headers = GetAllHeaders(); @@ -707,7 +709,14 @@ void HandleResponse(Stream responseStream, HttpStatusCode responseStatusCode, Di } else { - _logger?.Info($"{checkUrl} returned the HttpStatusCode of: '{responseStatusCode}' "); + if (_failJobOnEndpointIsBusy) + { + throw new WebException($"{checkUrl} returned the HttpStatusCode of: '{responseStatusCode}' "); + } + else + { + _logger?.Info($"{checkUrl} returned the HttpStatusCode of: '{responseStatusCode}' "); + } } } return result; From cfbc47005af42898121f09cc509164b2832b68f0 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Thu, 8 Aug 2024 10:20:19 +0200 Subject: [PATCH 2/3] Code-review feedback - typo change in tooltip --- src/ODataProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ODataProvider.cs b/src/ODataProvider.cs index 1500491..37e16c3 100644 --- a/src/ODataProvider.cs +++ b/src/ODataProvider.cs @@ -103,7 +103,7 @@ public string EndpointId public string DeltaModifier { get; set; } [AddInParameter("Fail job if endpoint is busy or down")] - [AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=If endpoint is busy or down doing the run time, it will not insert already imported rows.")] + [AddInParameterEditor(typeof(YesNoParameterEditor), "Tooltip=If endpoint is busy or down during the run time, it will not insert already imported rows.")] [AddInParameterGroup("Source")] [AddInParameterSection("Advanced activity settings")] public bool FailJobOnEndpointIsBusy { get; set; } From 32ae96d5cac296c93d0d079b642cd010300ea128 Mon Sep 17 00:00:00 2001 From: Matthias Sebastian Sort Date: Tue, 27 Aug 2024 08:54:54 +0200 Subject: [PATCH 3/3] Added responseStream to the exception and log info when endpoint is busy --- src/ODataSourceReader.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ODataSourceReader.cs b/src/ODataSourceReader.cs index 30ef340..d29288d 100644 --- a/src/ODataSourceReader.cs +++ b/src/ODataSourceReader.cs @@ -8,7 +8,6 @@ using System; using System.Collections.Generic; using System.Data; -using System.Globalization; using System.IO; using System.Linq; using System.Net; @@ -606,7 +605,7 @@ private bool HandleRequest(string url, string loggerInfo, IDictionary