diff --git a/src/Sources/RestApi/RestApiSource.cs b/src/Sources/RestApi/RestApiSource.cs
index 49d232f..e1d25f9 100644
--- a/src/Sources/RestApi/RestApiSource.cs
+++ b/src/Sources/RestApi/RestApiSource.cs
@@ -172,6 +172,37 @@ public static RestApiSource Create(
lookBackInterval, httpRequestTimeout, stopAfterBackfill, rateLimitPolicy, apiSchema, responsePropertyKeyChain);
}
+ ///
+ /// Creates new instance of
+ ///
+ /// URI provider
+ /// How often to track changes.
+ /// Look back interval
+ /// Http request rimeout
+ /// Api Schema
+ /// Rate limiting policy instance
+ /// Set to true to stream full current version of the table first.
+ /// Set to true if stream should stop after full load is finished
+ /// Authenticated message provider
+ /// Response property key chain
+ [ExcludeFromCodeCoverage(Justification = "Factory method")]
+ public static RestApiSource Create(
+ SimpleUriProvider uriProvider,
+ DynamicBearerAuthenticatedMessageProvider headerAuthenticatedMessageProvider,
+ bool isBackfilling,
+ TimeSpan changeCaptureInterval,
+ TimeSpan lookBackInterval,
+ TimeSpan httpRequestTimeout,
+ bool stopAfterBackfill,
+ AsyncRateLimitPolicy rateLimitPolicy,
+ OpenApiSchema apiSchema,
+ string[] responsePropertyKeyChain = null)
+ {
+ return new RestApiSource(uriProvider, headerAuthenticatedMessageProvider, isBackfilling,
+ changeCaptureInterval,
+ lookBackInterval, httpRequestTimeout, stopAfterBackfill, rateLimitPolicy, apiSchema, responsePropertyKeyChain);
+ }
+
///
/// Creates new instance of
///
diff --git a/src/Sources/RestApi/Services/AuthenticatedMessageProviders/DynamicBearerAuthenticatedMessageProvider.cs b/src/Sources/RestApi/Services/AuthenticatedMessageProviders/DynamicBearerAuthenticatedMessageProvider.cs
index b6ff38d..7ad7d4c 100644
--- a/src/Sources/RestApi/Services/AuthenticatedMessageProviders/DynamicBearerAuthenticatedMessageProvider.cs
+++ b/src/Sources/RestApi/Services/AuthenticatedMessageProviders/DynamicBearerAuthenticatedMessageProvider.cs
@@ -1,4 +1,5 @@
using System;
+using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
@@ -20,7 +21,10 @@ public record DynamicBearerAuthenticatedMessageProvider : IRestApiAuthenticatedM
private readonly HttpMethod requestMethod;
private readonly string tokenPropertyName;
private readonly string tokenRequestBody;
+ private readonly string authHeaderName;
+ private readonly string authScheme;
private readonly Uri tokenSource;
+ private readonly Dictionary additionalHeaders;
private string currentToken;
private DateTimeOffset? validTo;
@@ -32,14 +36,26 @@ public record DynamicBearerAuthenticatedMessageProvider : IRestApiAuthenticatedM
/// Token expiration property name
/// HTTP method for token request
/// HTTP body for token request
- public DynamicBearerAuthenticatedMessageProvider(string tokenSource, string tokenPropertyName,
- string expirationPeriodPropertyName, HttpMethod requestMethod = null, string tokenRequestBody = null)
+ /// Authorization header name
+ /// Authorization scheme
+ /// Additional token headers
+ public DynamicBearerAuthenticatedMessageProvider(string tokenSource,
+ string tokenPropertyName,
+ string expirationPeriodPropertyName,
+ HttpMethod requestMethod = null,
+ string tokenRequestBody = null,
+ Dictionary additionalHeaders = null,
+ string authHeaderName = null,
+ string authScheme = null)
{
this.tokenSource = new Uri(tokenSource);
this.tokenPropertyName = tokenPropertyName;
this.expirationPeriodPropertyName = expirationPeriodPropertyName;
this.tokenRequestBody = tokenRequestBody;
this.requestMethod = requestMethod ?? HttpMethod.Get;
+ this.authHeaderName = authHeaderName;
+ this.authScheme = authScheme;
+ this.additionalHeaders = additionalHeaders ?? new Dictionary();
}
///
@@ -47,29 +63,38 @@ public DynamicBearerAuthenticatedMessageProvider(string tokenSource, string toke
///
/// Token source address
/// Token property name
+ /// Token expiration period
/// HTTP method for token request
/// HTTP body for token request
- public DynamicBearerAuthenticatedMessageProvider(string tokenSource, string tokenPropertyName,
+ /// Additional token headers
+ /// Authorization header name
+ /// Authorization scheme
+ public DynamicBearerAuthenticatedMessageProvider(string tokenSource,
+ string tokenPropertyName,
TimeSpan expirationPeriod,
- HttpMethod requestMethod = null, string tokenRequestBody = null)
+ HttpMethod requestMethod = null,
+ string tokenRequestBody = null,
+ Dictionary additionalHeaders = null,
+ string authHeaderName = null,
+ string authScheme = null)
{
this.tokenSource = new Uri(tokenSource);
this.tokenPropertyName = tokenPropertyName;
this.expirationPeriod = expirationPeriod;
this.tokenRequestBody = tokenRequestBody;
this.requestMethod = requestMethod ?? HttpMethod.Get;
+ this.authHeaderName = authHeaderName;
+ this.authScheme = authScheme;
+ this.additionalHeaders = additionalHeaders ?? new Dictionary();
}
///
public Task GetAuthenticatedMessage(HttpClient httpClient)
{
- if (this.validTo.GetValueOrDefault(DateTimeOffset.MaxValue) <
- DateTimeOffset.UtcNow.Subtract(TimeSpan.FromMinutes(1)))
+
+ if (this.validTo.GetValueOrDefault(DateTimeOffset.MaxValue) < DateTimeOffset.UtcNow.Subtract(TimeSpan.FromMinutes(1)))
{
- return Task.FromResult(new HttpRequestMessage
- {
- Headers = { Authorization = new AuthenticationHeaderValue("Bearer", this.currentToken) }
- });
+ return Task.FromResult(this.GetRequest());
}
var tokenHrm = new HttpRequestMessage(this.requestMethod, this.tokenSource);
@@ -97,4 +122,26 @@ public Task GetAuthenticatedMessage(HttpClient httpClient)
};
});
}
+
+ private HttpRequestMessage GetRequest()
+ {
+ var request = new HttpRequestMessage();
+ switch (this.authHeaderName)
+ {
+ case null or "" or "Authorization":
+ request.Headers.Authorization = new AuthenticationHeaderValue(scheme: this.authScheme ?? "Bearer", this.currentToken);
+ break;
+ default:
+ request.Headers.Add(this.authHeaderName, string.IsNullOrEmpty(this.authScheme) ? this.currentToken : $"{this.authScheme} {this.currentToken}");
+ break;
+ }
+
+ foreach (var (headerKey, headerValue) in this.additionalHeaders ?? new Dictionary())
+ {
+ request.Headers.Add(headerKey, headerValue);
+ }
+
+ return request;
+ }
+
}
diff --git a/src/Sources/RestApi/Services/PageResolvers/PageNextTokenResolver.cs b/src/Sources/RestApi/Services/PageResolvers/PageNextTokenResolver.cs
index a29eb85..6c2014e 100644
--- a/src/Sources/RestApi/Services/PageResolvers/PageNextTokenResolver.cs
+++ b/src/Sources/RestApi/Services/PageResolvers/PageNextTokenResolver.cs
@@ -26,6 +26,13 @@ public override bool Next(Option apiResponse)
{
if (!apiResponse.IsEmpty)
{
+ // exit immediately if next page token property is not present
+ if (!this.GetResponseContent(apiResponse, this.nextPageTokenPropertyKeyChain).Any())
+ {
+ this.pagePointer = null;
+ return false;
+ }
+
// read next page token from response
this.pagePointer = this.nextPageTokenPropertyKeyChain
.Aggregate(this.GetResponse(apiResponse), (je, property) => je.GetProperty(property)).GetString();
@@ -38,6 +45,8 @@ public override bool Next(Option apiResponse)
};
}
- return string.IsNullOrEmpty(this.pagePointer);
+ // in case of empty response - reset page pointer to empty string and report ready for next
+ this.pagePointer = string.Empty;
+ return true;
}
}
diff --git a/src/Sources/RestApi/Services/RestApiTemplate.cs b/src/Sources/RestApi/Services/RestApiTemplate.cs
index ad63c37..292a3d3 100644
--- a/src/Sources/RestApi/Services/RestApiTemplate.cs
+++ b/src/Sources/RestApi/Services/RestApiTemplate.cs
@@ -64,14 +64,23 @@ public RestApiTemplate ResolveField(string fieldName, string fieldValue)
return this;
}
- if (this.remainingFieldNames.Contains(fieldName))
+ if (!this.remainingFieldNames.Contains(fieldName))
{
- var parameters = new Dictionary { { $"@{fieldName}", fieldValue } };
- this.resolvedTemplate = parameters.Aggregate(this.resolvedTemplate,
- (current, parameter) => current.Replace(parameter.Key, parameter.Value.ToString()));
- this.remainingFieldNames.Remove(fieldName);
+ return this;
+ }
+
+ // some resolvers may return a full uri - in this case we replace the resolved template with that value and clear out the template queue
+ if (Uri.TryCreate(fieldValue, UriKind.Absolute, out _))
+ {
+ this.resolvedTemplate = fieldValue;
+ this.remainingFieldNames.Clear();
+ return this;
}
+ var parameters = new Dictionary { { $"@{fieldName}", fieldValue } };
+ this.resolvedTemplate = parameters.Aggregate(this.resolvedTemplate, (current, parameter)=> current.Replace(parameter.Key, parameter.Value.ToString()));
+ this.remainingFieldNames.Remove(fieldName);
+
return this;
}
diff --git a/src/Sources/RestApi/Services/UriProviders/SimpleUriProvider.cs b/src/Sources/RestApi/Services/UriProviders/SimpleUriProvider.cs
index e83e08c..38b47dc 100644
--- a/src/Sources/RestApi/Services/UriProviders/SimpleUriProvider.cs
+++ b/src/Sources/RestApi/Services/UriProviders/SimpleUriProvider.cs
@@ -53,6 +53,12 @@ public SimpleUriProvider(string urlTemplate, List templat
var resultUri = this.urlTemplate.CreateResolver();
var resultBody = this.bodyTemplate.CreateResolver();
+
+ if (isBackfill && !paginatedResponse.IsEmpty)
+ {
+ return (Option.None, this.requestMethod, Option.None);
+ }
+
var filterTimestamp = (isFullLoad: isBackfill, paginatedResponse.IsEmpty) switch
{
(true, _) => this.backFillStartDate,
diff --git a/test/Sources/PageResolverTests.cs b/test/Sources/PageResolverTests.cs
index f595915..63be2d5 100644
--- a/test/Sources/PageResolverTests.cs
+++ b/test/Sources/PageResolverTests.cs
@@ -17,7 +17,7 @@ public void TestCounterPageResolver()
foreach (var (message, continuePagination) in RestApiArrayResponseSequence())
{
- Assert.Equal(resolver.Next(message), continuePagination);
+ Assert.Equal( continuePagination, resolver.Next(message));
}
}
@@ -28,7 +28,7 @@ public void TestTokenPageResolver()
foreach (var (message, continuePagination) in RestApiTokenResponseSequence())
{
- Assert.Equal(resolver.Next(message), continuePagination);
+ Assert.Equal(continuePagination, resolver.Next(message));
}
}
@@ -73,8 +73,8 @@ public void TestTokenPageResolver()
};
yield return (Option.None, true);
- yield return (filledMessage, true);
- yield return (filledMessage, true);
+ yield return (filledMessage, false);
+ yield return (filledMessage, false);
yield return (emptyMessage, false);
}
}