Enhance dynamic auth capabilities (#59)

* Port changes from old Arcane repository

* Also backport other changes

* Arrange unit tests with new behavior

* Fix

src/Sources/RestApi/RestApiSource.cs

@@ -172,6 +172,37 @@ public static RestApiSource Create(
             lookBackInterval, httpRequestTimeout, stopAfterBackfill, rateLimitPolicy, apiSchema, responsePropertyKeyChain);
     }
 
+    /// <summary>
+    /// Creates new instance of <see cref="RestApiSource"/>
+    /// </summary>
+    /// <param name="uriProvider">URI provider</param>
+    /// <param name="changeCaptureInterval">How often to track changes.</param>
+    /// <param name="lookBackInterval">Look back interval</param>
+    /// <param name="httpRequestTimeout">Http request rimeout</param>
+    /// <param name="apiSchema">Api Schema</param>
+    /// <param name="rateLimitPolicy">Rate limiting policy instance</param>
+    /// <param name="isBackfilling">Set to true to stream full current version of the table first.</param>
+    /// <param name="stopAfterBackfill">Set to true if stream should stop after full load is finished</param>
+    /// <param name="headerAuthenticatedMessageProvider">Authenticated message provider</param>
+    /// <param name="responsePropertyKeyChain">Response property key chain</param>
+    [ExcludeFromCodeCoverage(Justification = "Factory method")]
+    public static RestApiSource Create(
+        SimpleUriProvider uriProvider,
+        DynamicBearerAuthenticatedMessageProvider headerAuthenticatedMessageProvider,
+        bool isBackfilling,
+        TimeSpan changeCaptureInterval,
+        TimeSpan lookBackInterval,
+        TimeSpan httpRequestTimeout,
+        bool stopAfterBackfill,
+        AsyncRateLimitPolicy rateLimitPolicy,
+        OpenApiSchema apiSchema,
+        string[] responsePropertyKeyChain = null)
+    {
+        return new RestApiSource(uriProvider, headerAuthenticatedMessageProvider, isBackfilling,
+            changeCaptureInterval,
+            lookBackInterval, httpRequestTimeout, stopAfterBackfill, rateLimitPolicy, apiSchema, responsePropertyKeyChain);
+    }
+
     /// <summary>
     /// Creates new instance of <see cref="RestApiSource"/>
     /// </summary>

src/Sources/RestApi/Services/AuthenticatedMessageProviders/DynamicBearerAuthenticatedMessageProvider.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Text;
@@ -20,7 +21,10 @@ public record DynamicBearerAuthenticatedMessageProvider : IRestApiAuthenticatedM
     private readonly HttpMethod requestMethod;
     private readonly string tokenPropertyName;
     private readonly string tokenRequestBody;
+    private readonly string authHeaderName;
+    private readonly string authScheme;
     private readonly Uri tokenSource;
+    private readonly Dictionary<string, string> additionalHeaders;
     private string currentToken;
     private DateTimeOffset? validTo;
 
@@ -32,44 +36,65 @@ public record DynamicBearerAuthenticatedMessageProvider : IRestApiAuthenticatedM
     /// <param name="expirationPeriodPropertyName">Token expiration property name</param>
     /// <param name="requestMethod">HTTP method for token request</param>
     /// <param name="tokenRequestBody">HTTP body for token request</param>
-    public DynamicBearerAuthenticatedMessageProvider(string tokenSource, string tokenPropertyName,
-        string expirationPeriodPropertyName, HttpMethod requestMethod = null, string tokenRequestBody = null)
+    /// <param name="authHeaderName">Authorization header name</param>
+    /// <param name="authScheme">Authorization scheme</param>
+    /// <param name="additionalHeaders">Additional token headers</param>
+    public DynamicBearerAuthenticatedMessageProvider(string tokenSource,
+        string tokenPropertyName,
+        string expirationPeriodPropertyName,
+        HttpMethod requestMethod = null,
+        string tokenRequestBody = null,
+        Dictionary<string, string> additionalHeaders = null,
+        string authHeaderName = null,
+        string authScheme = null)
     {
         this.tokenSource = new Uri(tokenSource);
         this.tokenPropertyName = tokenPropertyName;
         this.expirationPeriodPropertyName = expirationPeriodPropertyName;
         this.tokenRequestBody = tokenRequestBody;
         this.requestMethod = requestMethod ?? HttpMethod.Get;
+        this.authHeaderName = authHeaderName;
+        this.authScheme = authScheme;
+        this.additionalHeaders = additionalHeaders ?? new Dictionary<string, string>();
     }
 
     /// <summary>
     /// Authenticated message provider that generated dynamic bearer token header.
     /// </summary>
     /// <param name="tokenSource">Token source address</param>
     /// <param name="tokenPropertyName">Token property name</param>
+    /// <param name="expirationPeriod">Token expiration period</param>
     /// <param name="requestMethod">HTTP method for token request</param>
     /// <param name="tokenRequestBody">HTTP body for token request</param>
-    public DynamicBearerAuthenticatedMessageProvider(string tokenSource, string tokenPropertyName,
+    /// <param name="additionalHeaders">Additional token headers</param>
+    /// <param name="authHeaderName">Authorization header name</param>
+    /// <param name="authScheme">Authorization scheme</param>
+    public DynamicBearerAuthenticatedMessageProvider(string tokenSource,
+        string tokenPropertyName,
         TimeSpan expirationPeriod,
-        HttpMethod requestMethod = null, string tokenRequestBody = null)
+        HttpMethod requestMethod = null,
+        string tokenRequestBody = null,
+        Dictionary<string, string> additionalHeaders = null,
+        string authHeaderName = null,
+        string authScheme = null)
     {
         this.tokenSource = new Uri(tokenSource);
         this.tokenPropertyName = tokenPropertyName;
         this.expirationPeriod = expirationPeriod;
         this.tokenRequestBody = tokenRequestBody;
         this.requestMethod = requestMethod ?? HttpMethod.Get;
+        this.authHeaderName = authHeaderName;
+        this.authScheme = authScheme;
+        this.additionalHeaders = additionalHeaders ?? new Dictionary<string, string>();
     }
 
     /// <inheritdoc cref="IRestApiAuthenticatedMessageProvider.GetAuthenticatedMessage"/>
     public Task<HttpRequestMessage> GetAuthenticatedMessage(HttpClient httpClient)
     {
-        if (this.validTo.GetValueOrDefault(DateTimeOffset.MaxValue) <
-            DateTimeOffset.UtcNow.Subtract(TimeSpan.FromMinutes(1)))
+
+        if (this.validTo.GetValueOrDefault(DateTimeOffset.MaxValue) <  DateTimeOffset.UtcNow.Subtract(TimeSpan.FromMinutes(1)))
         {
-            return Task.FromResult(new HttpRequestMessage
-            {
-                Headers = { Authorization = new AuthenticationHeaderValue("Bearer", this.currentToken) }
-            });
+            return Task.FromResult(this.GetRequest());
         }
 
         var tokenHrm = new HttpRequestMessage(this.requestMethod, this.tokenSource);
@@ -97,4 +122,26 @@ public Task<HttpRequestMessage> GetAuthenticatedMessage(HttpClient httpClient)
             };
         });
     }
+
+    private HttpRequestMessage GetRequest()
+    {
+        var request = new HttpRequestMessage();
+        switch (this.authHeaderName)
+        {
+            case null or "" or "Authorization":
+                request.Headers.Authorization = new AuthenticationHeaderValue(scheme: this.authScheme ?? "Bearer", this.currentToken);
+                break;
+            default:
+                request.Headers.Add(this.authHeaderName,  string.IsNullOrEmpty(this.authScheme) ? this.currentToken : $"{this.authScheme} {this.currentToken}");
+                break;
+        }
+
+        foreach (var (headerKey, headerValue) in this.additionalHeaders ?? new Dictionary<string, string>())
+        {
+            request.Headers.Add(headerKey, headerValue);
+        }
+
+        return request;
+    }
+
 }

src/Sources/RestApi/Services/PageResolvers/PageNextTokenResolver.cs

@@ -26,6 +26,13 @@ public override bool Next(Option<HttpResponseMessage> apiResponse)
     {
         if (!apiResponse.IsEmpty)
         {
+            // exit immediately if next page token property is not present
+            if (!this.GetResponseContent(apiResponse, this.nextPageTokenPropertyKeyChain).Any())
+            {
+                this.pagePointer = null;
+                return false;
+            }
+
             // read next page token from response
             this.pagePointer = this.nextPageTokenPropertyKeyChain
                 .Aggregate(this.GetResponse(apiResponse), (je, property) => je.GetProperty(property)).GetString();
@@ -38,6 +45,8 @@ public override bool Next(Option<HttpResponseMessage> apiResponse)
             };
         }
 
-        return string.IsNullOrEmpty(this.pagePointer);
+        // in case of empty response - reset page pointer to empty string and report ready for next
+        this.pagePointer = string.Empty;
+        return true;
     }
 }

src/Sources/RestApi/Services/RestApiTemplate.cs

@@ -64,14 +64,23 @@ public RestApiTemplate ResolveField(string fieldName, string fieldValue)
             return this;
         }
 
-        if (this.remainingFieldNames.Contains(fieldName))
+        if (!this.remainingFieldNames.Contains(fieldName))
         {
-            var parameters = new Dictionary<string, object> { { $"@{fieldName}", fieldValue } };
-            this.resolvedTemplate = parameters.Aggregate(this.resolvedTemplate,
-                (current, parameter) => current.Replace(parameter.Key, parameter.Value.ToString()));
-            this.remainingFieldNames.Remove(fieldName);
+            return this;
+        }
+
+        // some resolvers may return a full uri - in this case we replace the resolved template with that value and clear out the template queue
+        if (Uri.TryCreate(fieldValue, UriKind.Absolute, out _))
+        {
+            this.resolvedTemplate = fieldValue;
+            this.remainingFieldNames.Clear();
+            return this;
         }
 
+        var parameters = new Dictionary<string, object> { { $"@{fieldName}", fieldValue } };
+        this.resolvedTemplate = parameters.Aggregate(this.resolvedTemplate, (current, parameter)=> current.Replace(parameter.Key, parameter.Value.ToString()));
+        this.remainingFieldNames.Remove(fieldName);
+
         return this;
     }
 

src/Sources/RestApi/Services/UriProviders/SimpleUriProvider.cs

@@ -53,6 +53,12 @@ public SimpleUriProvider(string urlTemplate, List<RestApiTemplatedField> templat
         var resultUri = this.urlTemplate.CreateResolver();
         var resultBody = this.bodyTemplate.CreateResolver();
 
+
+        if (isBackfill && !paginatedResponse.IsEmpty)
+        {
+            return (Option<Uri>.None, this.requestMethod, Option<string>.None);
+        }
+
         var filterTimestamp = (isFullLoad: isBackfill, paginatedResponse.IsEmpty) switch
         {
             (true, _) => this.backFillStartDate,

test/Sources/PageResolverTests.cs

@@ -17,7 +17,7 @@ public void TestCounterPageResolver()
         foreach (var (message, continuePagination) in RestApiArrayResponseSequence())
         {
 
-            Assert.Equal(resolver.Next(message), continuePagination);
+            Assert.Equal( continuePagination, resolver.Next(message));
         }
     }
 
@@ -28,7 +28,7 @@ public void TestTokenPageResolver()
         foreach (var (message, continuePagination) in RestApiTokenResponseSequence())
         {
 
-            Assert.Equal(resolver.Next(message), continuePagination);
+            Assert.Equal(continuePagination, resolver.Next(message));
         }
     }
 
@@ -73,8 +73,8 @@ public void TestTokenPageResolver()
         };
 
         yield return (Option<HttpResponseMessage>.None, true);
-        yield return (filledMessage, true);
-        yield return (filledMessage, true);
+        yield return (filledMessage, false);
+        yield return (filledMessage, false);
         yield return (emptyMessage,  false);
     }
 }