diff --git a/src/DotNext.Tests/Net/Cluster/Consensus/Raft/MemoryBasedStateMachineTests.cs b/src/DotNext.Tests/Net/Cluster/Consensus/Raft/MemoryBasedStateMachineTests.cs index e2bd37cb8..14fa2979d 100644 --- a/src/DotNext.Tests/Net/Cluster/Consensus/Raft/MemoryBasedStateMachineTests.cs +++ b/src/DotNext.Tests/Net/Cluster/Consensus/Raft/MemoryBasedStateMachineTests.cs @@ -839,6 +839,37 @@ public static async Task RestoreBackup() } } + [Fact] + public static async Task CreateSparseBackup() + { + var entry1 = new TestLogEntry("SET X = 0") { Term = 42L }; + var entry2 = new TestLogEntry("SET Y = 1") { Term = 43L }; + var entry3 = new TestLogEntry("SET Z = 2") { Term = 44L }; + var entry4 = new TestLogEntry("SET U = 3") { Term = 45L }; + var entry5 = new TestLogEntry("SET V = 4") { Term = 46L }; + var dir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + var backupFile = Path.GetTempFileName(); + IPersistentState state = new PersistentStateWithoutSnapshot(dir, RecordsPerPartition, new() { MaxLogEntrySize = 1024 * 1024, BackupFormat = System.Formats.Tar.TarEntryFormat.Gnu }); + var member = ClusterMemberId.FromEndPoint(new IPEndPoint(IPAddress.IPv6Loopback, 3232)); + try + { + //define node state + Equal(1, await state.IncrementTermAsync(member)); + True(state.IsVotedFor(member)); + //define log entries + Equal(1L, await state.AppendAsync(new LogEntryList(entry1, entry2, entry3, entry4, entry5))); + //commit some of them + Equal(2L, await state.CommitAsync(2L)); + //save backup + await using var backupStream = new FileStream(backupFile, FileMode.Truncate, FileAccess.Write, FileShare.None, 1024, true); + await state.CreateBackupAsync(backupStream); + } + finally + { + (state as IDisposable)?.Dispose(); + } + } + [Fact] public static async Task Reconstruction() { diff --git a/src/cluster/DotNext.Net.Cluster/Net/Cluster/Consensus/Raft/PersistentState.Backup.cs b/src/cluster/DotNext.Net.Cluster/Net/Cluster/Consensus/Raft/PersistentState.Backup.cs index e1680a636..e769c0db8 100644 --- a/src/cluster/DotNext.Net.Cluster/Net/Cluster/Consensus/Raft/PersistentState.Backup.cs +++ b/src/cluster/DotNext.Net.Cluster/Net/Cluster/Consensus/Raft/PersistentState.Backup.cs @@ -46,6 +46,12 @@ private static void ImportAttributes(SafeFileHandle handle, TarEntry entry) File.SetUnixFileMode(handle, entry.Mode); } + + var attributes = FileAttributes.NotContentIndexed; + if (entry.EntryType is TarEntryType.SparseFile) + attributes |= FileAttributes.SparseFile; + + File.SetAttributes(handle, attributes); } /// @@ -55,7 +61,64 @@ private static void ImportAttributes(SafeFileHandle handle, TarEntry entry) /// The token that can be used to cancel the operation. /// A task representing state of asynchronous execution. /// The operation has been canceled. - public async Task CreateBackupAsync(Stream output, CancellationToken token = default) + public Task CreateBackupAsync(Stream output, CancellationToken token = default) + => maxLogEntrySize.HasValue ? CreateSparseBackupAsync(output, token) : CreateRegularBackupAsync(output, token); + + private async Task CreateSparseBackupAsync(Stream output, CancellationToken token) + { + var tarProcess = new Process + { + StartInfo = new() + { + FileName = "tar", + WorkingDirectory = Location.FullName, + }, + }; + + var outputArchive = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + tarProcess.StartInfo.ArgumentList.Add("cfS"); + tarProcess.StartInfo.ArgumentList.Add(outputArchive); + + FileStream? archiveStream = null; + await syncRoot.AcquireAsync(LockType.StrongReadLock, token).ConfigureAwait(false); + try + { + foreach (var file in Location.EnumerateFiles()) + { + tarProcess.StartInfo.ArgumentList.Add(file.Name); + } + + tarProcess.StartInfo.ArgumentList.Add($"--format={GetArchiveFormat(backupFormat)}"); + tarProcess.Start(); + await tarProcess.WaitForExitAsync(token).ConfigureAwait(false); + + if (tarProcess.ExitCode is not 0) + throw new InvalidOperationException() { HResult = tarProcess.ExitCode }; + + archiveStream = new(outputArchive, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.SequentialScan | FileOptions.Asynchronous | FileOptions.DeleteOnClose); + await archiveStream.CopyToAsync(output, token).ConfigureAwait(false); + await output.FlushAsync(token).ConfigureAwait(false); + } + finally + { + syncRoot.Release(LockType.StrongReadLock); + tarProcess.Dispose(); + + if (archiveStream is not null) + await archiveStream.DisposeAsync().ConfigureAwait(false); + } + + static string GetArchiveFormat(TarEntryFormat format) => format switch + { + TarEntryFormat.Gnu => "gnu", + TarEntryFormat.Pax => "pax", + TarEntryFormat.Ustar => "ustar", + TarEntryFormat.V7 => "v7", + _ => "gnu", + }; + } + + private async Task CreateRegularBackupAsync(Stream output, CancellationToken token) { TarWriter? archive = null; await syncRoot.AcquireAsync(LockType.StrongReadLock, token).ConfigureAwait(false); diff --git a/src/cluster/DotNext.Net.Cluster/Net/Cluster/Consensus/Raft/PersistentState.Options.cs b/src/cluster/DotNext.Net.Cluster/Net/Cluster/Consensus/Raft/PersistentState.Options.cs index 6419f7263..e4ac1b255 100644 --- a/src/cluster/DotNext.Net.Cluster/Net/Cluster/Consensus/Raft/PersistentState.Options.cs +++ b/src/cluster/DotNext.Net.Cluster/Net/Cluster/Consensus/Raft/PersistentState.Options.cs @@ -138,6 +138,9 @@ public int MaxConcurrentReads /// /// /// If enabled, WAL uses sparse files to optimize performance. + /// method supports backup of sparse + /// files on Linux only. + /// method cannot restore the backup, you need to use tar utility to extract files. /// public long? MaxLogEntrySize {