Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added Azure Blob Storage as storage backend for Persisted Operations #7830

Merged
merged 2 commits into from
Dec 16, 2024
Merged

Added Azure Blob Storage as storage backend for Persisted Operations #7830

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b946434
Added Azure Blob Storage as storage backend for Persisted Operations …
cimnine Dec 16, 2024
7b8b77e
Optimize Azure Blob Storage Persisted Operation Store.
michaelstaib Dec 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions src/Directory.Packages.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
<PropertyGroup>
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>

<ItemGroup>
<PackageVersion Include="AlterNats.Hosting" Version="1.0.6" />
<PackageVersion Include="Aspire.Hosting" Version="8.0.0" />
Expand All @@ -11,6 +10,7 @@
<PackageVersion Include="Aspire.Hosting.RabbitMQ" Version="8.0.0" />
<PackageVersion Include="Aspire.Hosting.Redis" Version="8.0.0" />
<PackageVersion Include="AutoMapper" Version="10.1.1" />
<PackageVersion Include="Azure.Storage.Blobs" Version="12.23.0" />
<PackageVersion Include="Basic.Reference.Assemblies.Net80" Version="1.7.8" />
<PackageVersion Include="Basic.Reference.Assemblies.Net90" Version="1.7.8" />
<PackageVersion Include="ChilliCream.Nitro.App" Version="$(NitroVersion)" />
Expand Down Expand Up @@ -53,6 +53,7 @@
<PackageVersion Include="Squadron.RabbitMQ" Version="0.18.0" />
<PackageVersion Include="Squadron.RavenDB" Version="0.18.0" />
<PackageVersion Include="Squadron.Redis" Version="0.18.0" />
<PackageVersion Include="Squadron.AzureStorage" Version="0.18.0" />
<PackageVersion Include="StackExchange.Redis" Version="2.6.80" />
<PackageVersion Include="System.Collections.Immutable" Version="8.0.0" />
<PackageVersion Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
Expand All @@ -63,7 +64,6 @@
<PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="17.12.5" />
<PackageVersion Include="TUnit" Version="0.4.71" />
</ItemGroup>

<ItemGroup Condition="'$(TargetFramework)' == 'net9.0'">
<PackageVersion Include="Microsoft.AspNetCore.Components.Web" Version="9.0.0" />
<PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="9.0.0" />
Expand All @@ -88,7 +88,6 @@
<PackageVersion Include="System.IO.Packaging" Version="9.0.0" />
<PackageVersion Include="System.IO.Pipelines" Version="9.0.0" />
</ItemGroup>

<ItemGroup Condition="'$(TargetFramework)' == 'net8.0'">
<PackageVersion Include="Microsoft.AspNetCore.Components.Web" Version="8.0.0" />
<PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="8.0.0" />
Expand All @@ -113,13 +112,11 @@
<PackageVersion Include="System.IO.Packaging" Version="8.0.1" />
<PackageVersion Include="System.IO.Pipelines" Version="8.0.0" />
</ItemGroup>

<ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
<PackageVersion Include="Microsoft.Extensions.ObjectPool" Version="6.0.0" />
<PackageVersion Include="System.Memory" Version="4.5.5" />
<PackageVersion Include="System.Threading.Tasks.Extensions" Version="4.5.4" />
</ItemGroup>

<ItemGroup Condition="'$(Configuration)' == 'debug'">
<PackageVersion Include="Roslynator.Analyzers" Version="4.12.4" />
<PackageVersion Include="Roslynator.CodeAnalysis.Analyzers" Version="4.12.4" />
Expand Down
30 changes: 30 additions & 0 deletions src/HotChocolate/PersistedOperations/HotChocolate.PersistedOperations.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HotChocolate.PersistedOpera
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HotChocolate.PersistedOperations.InMemory.Tests", "test\PersistedOperations.InMemory.Tests\HotChocolate.PersistedOperations.InMemory.Tests.csproj", "{406929B9-06ED-428D-B478-A21B41B9BDC0}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HotChocolate.PersistedOperations.AzureBlobStorage.Tests", "test\PersistedOperations.AzureBlobStorage.Tests\HotChocolate.PersistedOperations.AzureBlobStorage.Tests.csproj", "{705B7DF2-F2B6-4561-BDCC-365C92BD3E52}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HotChocolate.PersistedOperations.AzureBlobStorage", "src\PersistedOperations.AzureBlobStorage\HotChocolate.PersistedOperations.AzureBlobStorage.csproj", "{4F022EBA-58D1-419B-9667-7291316BD401}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -235,6 +239,30 @@ Global
{406929B9-06ED-428D-B478-A21B41B9BDC0}.Release|x64.Build.0 = Release|Any CPU
{406929B9-06ED-428D-B478-A21B41B9BDC0}.Release|x86.ActiveCfg = Release|Any CPU
{406929B9-06ED-428D-B478-A21B41B9BDC0}.Release|x86.Build.0 = Release|Any CPU
{705B7DF2-F2B6-4561-BDCC-365C92BD3E52}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{705B7DF2-F2B6-4561-BDCC-365C92BD3E52}.Debug|Any CPU.Build.0 = Debug|Any CPU
{705B7DF2-F2B6-4561-BDCC-365C92BD3E52}.Debug|x64.ActiveCfg = Debug|Any CPU
{705B7DF2-F2B6-4561-BDCC-365C92BD3E52}.Debug|x64.Build.0 = Debug|Any CPU
{705B7DF2-F2B6-4561-BDCC-365C92BD3E52}.Debug|x86.ActiveCfg = Debug|Any CPU
{705B7DF2-F2B6-4561-BDCC-365C92BD3E52}.Debug|x86.Build.0 = Debug|Any CPU
{705B7DF2-F2B6-4561-BDCC-365C92BD3E52}.Release|Any CPU.ActiveCfg = Release|Any CPU
{705B7DF2-F2B6-4561-BDCC-365C92BD3E52}.Release|Any CPU.Build.0 = Release|Any CPU
{705B7DF2-F2B6-4561-BDCC-365C92BD3E52}.Release|x64.ActiveCfg = Release|Any CPU
{705B7DF2-F2B6-4561-BDCC-365C92BD3E52}.Release|x64.Build.0 = Release|Any CPU
{705B7DF2-F2B6-4561-BDCC-365C92BD3E52}.Release|x86.ActiveCfg = Release|Any CPU
{705B7DF2-F2B6-4561-BDCC-365C92BD3E52}.Release|x86.Build.0 = Release|Any CPU
{4F022EBA-58D1-419B-9667-7291316BD401}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4F022EBA-58D1-419B-9667-7291316BD401}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4F022EBA-58D1-419B-9667-7291316BD401}.Debug|x64.ActiveCfg = Debug|Any CPU
{4F022EBA-58D1-419B-9667-7291316BD401}.Debug|x64.Build.0 = Debug|Any CPU
{4F022EBA-58D1-419B-9667-7291316BD401}.Debug|x86.ActiveCfg = Debug|Any CPU
{4F022EBA-58D1-419B-9667-7291316BD401}.Debug|x86.Build.0 = Debug|Any CPU
{4F022EBA-58D1-419B-9667-7291316BD401}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4F022EBA-58D1-419B-9667-7291316BD401}.Release|Any CPU.Build.0 = Release|Any CPU
{4F022EBA-58D1-419B-9667-7291316BD401}.Release|x64.ActiveCfg = Release|Any CPU
{4F022EBA-58D1-419B-9667-7291316BD401}.Release|x64.Build.0 = Release|Any CPU
{4F022EBA-58D1-419B-9667-7291316BD401}.Release|x86.ActiveCfg = Release|Any CPU
{4F022EBA-58D1-419B-9667-7291316BD401}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand All @@ -255,6 +283,8 @@ Global
{8BCB768F-4588-41A2-9C7F-7AE7F3FAC404} = {0FE82812-5154-4AF6-9053-487CBD1C5E74}
{FDCD436C-3276-4F21-AC8F-E3DCA4EB41EB} = {9313BB55-188C-4067-9514-9317E83847BC}
{406929B9-06ED-428D-B478-A21B41B9BDC0} = {A302B0E8-3C01-458D-8C2D-D59FF528287A}
{705B7DF2-F2B6-4561-BDCC-365C92BD3E52} = {A302B0E8-3C01-458D-8C2D-D59FF528287A}
{4F022EBA-58D1-419B-9667-7291316BD401} = {9313BB55-188C-4067-9514-9317E83847BC}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {FAA72987-7EEF-40D2-B232-34E3D16C9489}
Expand Down
167 changes: 167 additions & 0 deletions ...dOperations/src/PersistedOperations.AzureBlobStorage/AzureBlobOperationDocumentStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
using System.Buffers;
using Azure;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using HotChocolate.Execution;
using HotChocolate.Language;

namespace HotChocolate.PersistedOperations.AzureBlobStorage;

/// <summary>
/// An implementation of <see cref="IOperationDocumentStorage"/> that uses Redis as a storage.
/// </summary>
public class AzureBlobOperationDocumentStorage : IOperationDocumentStorage
{
private static readonly char[] _fileExtension = ".graphql".ToCharArray();

private static readonly BlobOpenWriteOptions _writeOptions = new()
{
HttpHeaders = new BlobHttpHeaders
{
ContentType = "application/graphql",
ContentDisposition = "inline",
CacheControl = "public, max-age=604800, immutable"
}
};

private readonly BlobContainerClient _client;

/// <summary>
/// Initializes a new instance of the class.
/// </summary>
/// <param name="client">The blob container client instance.</param>
public AzureBlobOperationDocumentStorage(BlobContainerClient client)
{
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}

_client = client;
}

/// <inheritdoc />
public ValueTask<IOperationDocument?> TryReadAsync(
OperationDocumentId documentId,
CancellationToken cancellationToken = default)
{
if (OperationDocumentId.IsNullOrEmpty(documentId))
{
throw new ArgumentNullException(nameof(documentId));
}

return TryReadInternalAsync(documentId, cancellationToken);
}

private async ValueTask<IOperationDocument?> TryReadInternalAsync(
OperationDocumentId documentId,
CancellationToken ct)
{
var blobClient = _client.GetBlobClient(CreateFileName(documentId));
var buffer = ArrayPool<byte>.Shared.Rent(1024);
var position = 0;

try
{
await using var blobStream = await blobClient.OpenReadAsync(cancellationToken: ct).ConfigureAwait(false);
while (true)
{
if (buffer.Length < position + 256)
{
var newBuffer = ArrayPool<byte>.Shared.Rent(buffer.Length * 2);
Array.Copy(buffer, newBuffer, buffer.Length);
ArrayPool<byte>.Shared.Return(buffer);
buffer = newBuffer;
}

var read = await blobStream.ReadAsync(buffer, position, 256, ct);
position += read;

if (read < 256)
{
break;
}
}

if (position == 0)
{
return null;
}

var span = new ReadOnlySpan<byte>(buffer, 0, position);
return new OperationDocument(Utf8GraphQLParser.Parse(span));
}
catch (RequestFailedException e)
{
if (e.Status == 404)
{
return null;
}

throw;
}
finally
{
if(position > 0)
{
buffer.AsSpan().Slice(0, position).Clear();
}

ArrayPool<byte>.Shared.Return(buffer);
}
}

/// <inheritdoc />
public ValueTask SaveAsync(
OperationDocumentId documentId,
IOperationDocument document,
CancellationToken cancellationToken = default)
{
if(document == null)
{
throw new ArgumentNullException(nameof(document));
}

if (OperationDocumentId.IsNullOrEmpty(documentId))
{
throw new ArgumentException(nameof(documentId));
}

return SaveInternalAsync(documentId, document, cancellationToken);
}

private async ValueTask SaveInternalAsync(
OperationDocumentId documentId,
IOperationDocument document,
CancellationToken ct)
{
var blobClient = _client.GetBlobClient(CreateFileName(documentId));
await using var outStream = await blobClient.OpenWriteAsync(true, _writeOptions, ct).ConfigureAwait(false);
await document.WriteToAsync(outStream, ct).ConfigureAwait(false);
await outStream.FlushAsync(ct).ConfigureAwait(false);
}


private static string CreateFileName(OperationDocumentId documentId)
{
var length = documentId.Value.Length + _fileExtension.Length;
char[]? rented = null;
Span<char> span = length <= 256
? stackalloc char[length]
: rented = ArrayPool<char>.Shared.Rent(length);

try
{
documentId.Value.AsSpan().CopyTo(span);
_fileExtension.AsSpan().CopyTo(span.Slice(documentId.Value.Length));
return new string(span);
}
finally
{
if (rented != null)
{
ArrayPool<char>.Shared.Return(rented);
}
}
}
}
40 changes: 40 additions & 0 deletions ...nsions/HotChocolateAzureBlobStoragePersistedOperationsRequestExecutorBuilderExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
using Azure.Storage.Blobs;
using HotChocolate.Execution.Configuration;
using HotChocolate;

namespace Microsoft.Extensions.DependencyInjection;

/// <summary>
/// Provides utility methods to setup dependency injection.
/// </summary>
public static class HotChocolateAzureBlobStoragePersistedOperationsRequestExecutorBuilderExtensions
{
/// <summary>
/// Adds an Azure Blob Storage based operation document storage to the service collection.
/// </summary>
/// <param name="builder">
/// The service collection to which the services are added.
/// </param>
/// <param name="containerClientFactory">
/// A factory that resolves the Azure Blob Container Client that
/// shall be used for persistence.
/// </param>
public static IRequestExecutorBuilder AddAzureBlobStorageOperationDocumentStorage(
this IRequestExecutorBuilder builder,
Func<IServiceProvider, BlobContainerClient> containerClientFactory)
{
if(builder == null)
{
throw new ArgumentNullException(nameof(builder));
}

if(containerClientFactory is null)
{
throw new ArgumentNullException(nameof(containerClientFactory));
}

return builder.ConfigureSchemaServices(
s => s.AddAzureBlobStorageOperationDocumentStorage(
sp => containerClientFactory(sp.GetCombinedServices())));
}
}
55 changes: 55 additions & 0 deletions .../Extensions/HotChocolateAzureBlobStoragePersistedOperationsServiceCollectionExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
using Azure.Storage.Blobs;
using HotChocolate.Execution;
using HotChocolate.PersistedOperations.AzureBlobStorage;
using Microsoft.Extensions.DependencyInjection;

namespace HotChocolate;

/// <summary>
/// Provides utility methods to set up dependency injection.
/// </summary>
public static class HotChocolateAzureBlobStoragePersistedOperationsServiceCollectionExtensions
{
/// <summary>
/// Adds an Azure Blob Storage based operation document storage to the service collection.
/// </summary>
/// <param name="services">
/// The service collection to which the services are added.
/// </param>
/// <param name="containerClientFactory">
/// A factory that resolves the Azure Blob Container Client that
/// shall be used for persistence.
/// </param>
public static IServiceCollection AddAzureBlobStorageOperationDocumentStorage(
this IServiceCollection services,
Func<IServiceProvider, BlobContainerClient> containerClientFactory)
{
if(services == null)
{
throw new ArgumentNullException(nameof(services));
}

if(containerClientFactory == null)
{
throw new ArgumentNullException(nameof(containerClientFactory));
}

return services
.RemoveService<IOperationDocumentStorage>()
.AddSingleton<IOperationDocumentStorage>(
sp => new AzureBlobOperationDocumentStorage(containerClientFactory(sp)));
}

private static IServiceCollection RemoveService<TService>(
this IServiceCollection services)
{
var serviceDescriptor = services.FirstOrDefault(t => t.ServiceType == typeof(TService));

if (serviceDescriptor != null)
{
services.Remove(serviceDescriptor);
}

return services;
}
}
Loading
Loading