diff --git a/src/Sources/RestApi/RestApiSource.cs b/src/Sources/RestApi/RestApiSource.cs index 49d232f..e1d25f9 100644 --- a/src/Sources/RestApi/RestApiSource.cs +++ b/src/Sources/RestApi/RestApiSource.cs @@ -172,6 +172,37 @@ public static RestApiSource Create( lookBackInterval, httpRequestTimeout, stopAfterBackfill, rateLimitPolicy, apiSchema, responsePropertyKeyChain); } + /// + /// Creates new instance of + /// + /// URI provider + /// How often to track changes. + /// Look back interval + /// Http request rimeout + /// Api Schema + /// Rate limiting policy instance + /// Set to true to stream full current version of the table first. + /// Set to true if stream should stop after full load is finished + /// Authenticated message provider + /// Response property key chain + [ExcludeFromCodeCoverage(Justification = "Factory method")] + public static RestApiSource Create( + SimpleUriProvider uriProvider, + DynamicBearerAuthenticatedMessageProvider headerAuthenticatedMessageProvider, + bool isBackfilling, + TimeSpan changeCaptureInterval, + TimeSpan lookBackInterval, + TimeSpan httpRequestTimeout, + bool stopAfterBackfill, + AsyncRateLimitPolicy rateLimitPolicy, + OpenApiSchema apiSchema, + string[] responsePropertyKeyChain = null) + { + return new RestApiSource(uriProvider, headerAuthenticatedMessageProvider, isBackfilling, + changeCaptureInterval, + lookBackInterval, httpRequestTimeout, stopAfterBackfill, rateLimitPolicy, apiSchema, responsePropertyKeyChain); + } + /// /// Creates new instance of /// diff --git a/src/Sources/RestApi/Services/AuthenticatedMessageProviders/DynamicBearerAuthenticatedMessageProvider.cs b/src/Sources/RestApi/Services/AuthenticatedMessageProviders/DynamicBearerAuthenticatedMessageProvider.cs index b6ff38d..7ad7d4c 100644 --- a/src/Sources/RestApi/Services/AuthenticatedMessageProviders/DynamicBearerAuthenticatedMessageProvider.cs +++ b/src/Sources/RestApi/Services/AuthenticatedMessageProviders/DynamicBearerAuthenticatedMessageProvider.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Net.Http; using System.Net.Http.Headers; using System.Text; @@ -20,7 +21,10 @@ public record DynamicBearerAuthenticatedMessageProvider : IRestApiAuthenticatedM private readonly HttpMethod requestMethod; private readonly string tokenPropertyName; private readonly string tokenRequestBody; + private readonly string authHeaderName; + private readonly string authScheme; private readonly Uri tokenSource; + private readonly Dictionary additionalHeaders; private string currentToken; private DateTimeOffset? validTo; @@ -32,14 +36,26 @@ public record DynamicBearerAuthenticatedMessageProvider : IRestApiAuthenticatedM /// Token expiration property name /// HTTP method for token request /// HTTP body for token request - public DynamicBearerAuthenticatedMessageProvider(string tokenSource, string tokenPropertyName, - string expirationPeriodPropertyName, HttpMethod requestMethod = null, string tokenRequestBody = null) + /// Authorization header name + /// Authorization scheme + /// Additional token headers + public DynamicBearerAuthenticatedMessageProvider(string tokenSource, + string tokenPropertyName, + string expirationPeriodPropertyName, + HttpMethod requestMethod = null, + string tokenRequestBody = null, + Dictionary additionalHeaders = null, + string authHeaderName = null, + string authScheme = null) { this.tokenSource = new Uri(tokenSource); this.tokenPropertyName = tokenPropertyName; this.expirationPeriodPropertyName = expirationPeriodPropertyName; this.tokenRequestBody = tokenRequestBody; this.requestMethod = requestMethod ?? HttpMethod.Get; + this.authHeaderName = authHeaderName; + this.authScheme = authScheme; + this.additionalHeaders = additionalHeaders ?? new Dictionary(); } /// @@ -47,29 +63,38 @@ public DynamicBearerAuthenticatedMessageProvider(string tokenSource, string toke /// /// Token source address /// Token property name + /// Token expiration period /// HTTP method for token request /// HTTP body for token request - public DynamicBearerAuthenticatedMessageProvider(string tokenSource, string tokenPropertyName, + /// Additional token headers + /// Authorization header name + /// Authorization scheme + public DynamicBearerAuthenticatedMessageProvider(string tokenSource, + string tokenPropertyName, TimeSpan expirationPeriod, - HttpMethod requestMethod = null, string tokenRequestBody = null) + HttpMethod requestMethod = null, + string tokenRequestBody = null, + Dictionary additionalHeaders = null, + string authHeaderName = null, + string authScheme = null) { this.tokenSource = new Uri(tokenSource); this.tokenPropertyName = tokenPropertyName; this.expirationPeriod = expirationPeriod; this.tokenRequestBody = tokenRequestBody; this.requestMethod = requestMethod ?? HttpMethod.Get; + this.authHeaderName = authHeaderName; + this.authScheme = authScheme; + this.additionalHeaders = additionalHeaders ?? new Dictionary(); } /// public Task GetAuthenticatedMessage(HttpClient httpClient) { - if (this.validTo.GetValueOrDefault(DateTimeOffset.MaxValue) < - DateTimeOffset.UtcNow.Subtract(TimeSpan.FromMinutes(1))) + + if (this.validTo.GetValueOrDefault(DateTimeOffset.MaxValue) < DateTimeOffset.UtcNow.Subtract(TimeSpan.FromMinutes(1))) { - return Task.FromResult(new HttpRequestMessage - { - Headers = { Authorization = new AuthenticationHeaderValue("Bearer", this.currentToken) } - }); + return Task.FromResult(this.GetRequest()); } var tokenHrm = new HttpRequestMessage(this.requestMethod, this.tokenSource); @@ -97,4 +122,26 @@ public Task GetAuthenticatedMessage(HttpClient httpClient) }; }); } + + private HttpRequestMessage GetRequest() + { + var request = new HttpRequestMessage(); + switch (this.authHeaderName) + { + case null or "" or "Authorization": + request.Headers.Authorization = new AuthenticationHeaderValue(scheme: this.authScheme ?? "Bearer", this.currentToken); + break; + default: + request.Headers.Add(this.authHeaderName, string.IsNullOrEmpty(this.authScheme) ? this.currentToken : $"{this.authScheme} {this.currentToken}"); + break; + } + + foreach (var (headerKey, headerValue) in this.additionalHeaders ?? new Dictionary()) + { + request.Headers.Add(headerKey, headerValue); + } + + return request; + } + } diff --git a/src/Sources/RestApi/Services/PageResolvers/PageNextTokenResolver.cs b/src/Sources/RestApi/Services/PageResolvers/PageNextTokenResolver.cs index a29eb85..6c2014e 100644 --- a/src/Sources/RestApi/Services/PageResolvers/PageNextTokenResolver.cs +++ b/src/Sources/RestApi/Services/PageResolvers/PageNextTokenResolver.cs @@ -26,6 +26,13 @@ public override bool Next(Option apiResponse) { if (!apiResponse.IsEmpty) { + // exit immediately if next page token property is not present + if (!this.GetResponseContent(apiResponse, this.nextPageTokenPropertyKeyChain).Any()) + { + this.pagePointer = null; + return false; + } + // read next page token from response this.pagePointer = this.nextPageTokenPropertyKeyChain .Aggregate(this.GetResponse(apiResponse), (je, property) => je.GetProperty(property)).GetString(); @@ -38,6 +45,8 @@ public override bool Next(Option apiResponse) }; } - return string.IsNullOrEmpty(this.pagePointer); + // in case of empty response - reset page pointer to empty string and report ready for next + this.pagePointer = string.Empty; + return true; } } diff --git a/src/Sources/RestApi/Services/RestApiTemplate.cs b/src/Sources/RestApi/Services/RestApiTemplate.cs index ad63c37..292a3d3 100644 --- a/src/Sources/RestApi/Services/RestApiTemplate.cs +++ b/src/Sources/RestApi/Services/RestApiTemplate.cs @@ -64,14 +64,23 @@ public RestApiTemplate ResolveField(string fieldName, string fieldValue) return this; } - if (this.remainingFieldNames.Contains(fieldName)) + if (!this.remainingFieldNames.Contains(fieldName)) { - var parameters = new Dictionary { { $"@{fieldName}", fieldValue } }; - this.resolvedTemplate = parameters.Aggregate(this.resolvedTemplate, - (current, parameter) => current.Replace(parameter.Key, parameter.Value.ToString())); - this.remainingFieldNames.Remove(fieldName); + return this; + } + + // some resolvers may return a full uri - in this case we replace the resolved template with that value and clear out the template queue + if (Uri.TryCreate(fieldValue, UriKind.Absolute, out _)) + { + this.resolvedTemplate = fieldValue; + this.remainingFieldNames.Clear(); + return this; } + var parameters = new Dictionary { { $"@{fieldName}", fieldValue } }; + this.resolvedTemplate = parameters.Aggregate(this.resolvedTemplate, (current, parameter)=> current.Replace(parameter.Key, parameter.Value.ToString())); + this.remainingFieldNames.Remove(fieldName); + return this; } diff --git a/src/Sources/RestApi/Services/UriProviders/SimpleUriProvider.cs b/src/Sources/RestApi/Services/UriProviders/SimpleUriProvider.cs index e83e08c..38b47dc 100644 --- a/src/Sources/RestApi/Services/UriProviders/SimpleUriProvider.cs +++ b/src/Sources/RestApi/Services/UriProviders/SimpleUriProvider.cs @@ -53,6 +53,12 @@ public SimpleUriProvider(string urlTemplate, List templat var resultUri = this.urlTemplate.CreateResolver(); var resultBody = this.bodyTemplate.CreateResolver(); + + if (isBackfill && !paginatedResponse.IsEmpty) + { + return (Option.None, this.requestMethod, Option.None); + } + var filterTimestamp = (isFullLoad: isBackfill, paginatedResponse.IsEmpty) switch { (true, _) => this.backFillStartDate, diff --git a/test/Sources/PageResolverTests.cs b/test/Sources/PageResolverTests.cs index f595915..63be2d5 100644 --- a/test/Sources/PageResolverTests.cs +++ b/test/Sources/PageResolverTests.cs @@ -17,7 +17,7 @@ public void TestCounterPageResolver() foreach (var (message, continuePagination) in RestApiArrayResponseSequence()) { - Assert.Equal(resolver.Next(message), continuePagination); + Assert.Equal( continuePagination, resolver.Next(message)); } } @@ -28,7 +28,7 @@ public void TestTokenPageResolver() foreach (var (message, continuePagination) in RestApiTokenResponseSequence()) { - Assert.Equal(resolver.Next(message), continuePagination); + Assert.Equal(continuePagination, resolver.Next(message)); } } @@ -73,8 +73,8 @@ public void TestTokenPageResolver() }; yield return (Option.None, true); - yield return (filledMessage, true); - yield return (filledMessage, true); + yield return (filledMessage, false); + yield return (filledMessage, false); yield return (emptyMessage, false); } }