[DEVOPS-1537 & DEVOPS-1519] Update dbo.Migrations table to support own migration schema

util/Migrator/DbMigrator.cs

@@ -24,6 +24,8 @@ public DbMigrator(string connectionString, ILogger<DbMigrator> logger)
     }
 
     public bool MigrateMsSqlDatabaseWithRetries(bool enableLogging = true,
+        bool repeatable = false,
+        string folderName = "DbScripts",
         CancellationToken cancellationToken = default(CancellationToken))
     {
         var attempt = 1;
@@ -32,7 +34,7 @@ public bool MigrateMsSqlDatabaseWithRetries(bool enableLogging = true,
         {
             try
             {
-                var success = MigrateDatabase(enableLogging, cancellationToken);
+                var success = MigrateDatabase(enableLogging, repeatable, folderName, cancellationToken);
                 return success;
             }
             catch (SqlException ex)
@@ -54,6 +56,8 @@ public bool MigrateMsSqlDatabaseWithRetries(bool enableLogging = true,
     }
 
     public bool MigrateDatabase(bool enableLogging = true,
+        bool repeatable = false,
+        string folderName = "DbScripts",
         CancellationToken cancellationToken = default(CancellationToken))
     {
         if (_logger != null)
@@ -98,9 +102,9 @@ public bool MigrateDatabase(bool enableLogging = true,
         cancellationToken.ThrowIfCancellationRequested();
         var builder = DeployChanges.To
             .SqlDatabase(_connectionString)
-            .JournalToSqlTable("dbo", "Migration")
+            .JournalRepeatableToSqlTable("dbo", "Migration", repeatable)
             .WithScriptsAndCodeEmbeddedInAssembly(Assembly.GetExecutingAssembly(),
-                s => s.Contains($".DbScripts.") && !s.Contains(".Archive."))
+                s => s.Contains($".{folderName}.") && !s.Contains(".Archive."))
             .WithTransaction()
             .WithExecutionTimeout(new TimeSpan(0, 5, 0));
 

util/Migrator/DbScripts/2023-08-24_00_AddRerunableColumnIndboMigrationsTable.sql

@@ -0,0 +1,3 @@
+ALTER TABLE dbo.Migration
+ADD Repeatable bit NOT NULL DEFAULT 0
+GO

util/Migrator/Migrator.csproj

@@ -2,6 +2,7 @@
 
   <ItemGroup>
     <EmbeddedResource Include="DbScripts\**\*.sql" />
+    <EmbeddedResource Include="DbScripts_data_migration\**\*.sql" />
   </ItemGroup>
 
   <ItemGroup>

util/Migrator/SqlTableJournalExtensions.cs

@@ -0,0 +1,105 @@
+using System.Data;
+using DbUp.Builder;
+using DbUp.Engine;
+using DbUp.Engine.Output;
+using DbUp.Engine.Transactions;
+using DbUp.SqlServer;
+
+namespace Bit.Migrator;
+
+public static class SqlTableJournalExtensions
+{
+    public static UpgradeEngineBuilder JournalRepeatableToSqlTable(this UpgradeEngineBuilder builder, string schema, string table, bool repeatable = false)
+    {
+        builder.Configure(c => c.Journal = new RepeatableSqlTableJournal(() => c.ConnectionManager, () => c.Log, schema, table, repeatable));
+        return builder;
+    }
+}
+
+public class RepeatableSqlTableJournal : SqlTableJournal
+{
+    private bool Repeatable { get; set; }
+
+    public RepeatableSqlTableJournal(Func<IConnectionManager> connectionManager, Func<IUpgradeLog> logger, string schema, string table, bool repeatable = false)
+        : base(connectionManager, logger, schema, table)
+    {
+        Repeatable = repeatable;
+    }
+
+    public override void StoreExecutedScript(SqlScript script, Func<IDbCommand> dbCommandFactory)
+    {
+        EnsureTableExistsAndIsLatestVersion(dbCommandFactory);
+        using (var command = GetInsertScriptCommand(dbCommandFactory, script))
+        {
+            command.ExecuteNonQuery();
+        }
+    }
+
+    protected new IDbCommand GetInsertScriptCommand(Func<IDbCommand> dbCommandFactory, SqlScript script)
+    {
+        var command = dbCommandFactory();
+
+        var scriptNameParam = command.CreateParameter();
+        scriptNameParam.ParameterName = "scriptName";
+        scriptNameParam.Value = script.Name;
+        command.Parameters.Add(scriptNameParam);
+
+        var scriptFilename = script.Name.Replace("Bit.Migrator.", "");
+        scriptFilename = scriptFilename.Substring(scriptFilename.IndexOf('.') + 1);
+
+        var scriptFileNameParam = command.CreateParameter();
+        scriptFileNameParam.ParameterName = "scriptFileName";
+        scriptFileNameParam.Value = $"%{scriptFilename}";
+        command.Parameters.Add(scriptFileNameParam);
+
+        var appliedParam = command.CreateParameter();
+        appliedParam.ParameterName = "applied";
+        appliedParam.Value = DateTime.Now;
+        command.Parameters.Add(appliedParam);
+
+        var repeatableParam = command.CreateParameter();
+        repeatableParam.ParameterName = "repeatable";
+        repeatableParam.Value = Repeatable;
+        command.Parameters.Add(repeatableParam);
+
+        command.CommandText = GetInsertJournalEntrySql("@scriptName", "@applied", "@repeatable", "@scriptFileName");
+        command.CommandType = CommandType.Text;
+        return command;
+    }
+
+    protected string GetInsertJournalEntrySql(string @scriptName, string @applied, string @repeatable, string @scriptFileName)
+    {
+        return @$"IF EXISTS (SELECT * FROM {FqSchemaTableName} WHERE Repeatable = 1 AND ScriptName like {@scriptFileName})
+            BEGIN
+                UPDATE {FqSchemaTableName} SET ScriptName = {@scriptName}, Applied = {@applied}, Repeatable = {@repeatable} WHERE ScriptName like {@scriptFileName}
+            END
+        ELSE
+            BEGIN
+                insert into {FqSchemaTableName} (ScriptName, Applied, Repeatable) values ({@scriptName}, {@applied}, {@repeatable})
+            END ";
+    }
+
+    protected override string GetJournalEntriesSql()
+    {
+        return @$"DECLARE @columnVariable AS NVARCHAR(max)
+
+            SELECT @columnVariable =
+            CASE WHEN EXISTS
+            (
+                SELECT 1 FROM sys.columns WHERE Name = N'Repeatable' AND Object_ID = Object_ID(N'dbo.Migration')
+            )
+            THEN
+            (
+                'where [Repeatable] = 0'
+            )
+            ELSE
+            (
+                ''
+            )
+            END
+
+            DECLARE @SQLString AS NVARCHAR(max) = N'select [ScriptName] from dbo.Migration ' + @columnVariable + ' order by [ScriptName]'
+
+            EXECUTE sp_executesql @SQLString";
+    }
+}

util/MsSqlMigratorUtility/MsSqlMigratorUtility.csproj

@@ -10,6 +10,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="CommandDotNet" Version="7.0.2" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
   </ItemGroup>

util/MsSqlMigratorUtility/Program.cs

@@ -1,55 +1,51 @@
 using Bit.Migrator;
+using CommandDotNet;
 using Microsoft.Extensions.Logging;
 
 internal class Program
 {
+    private static IDictionary<string, string> Parameters { get; set; }
+
     private static int Main(string[] args)
     {
-        if (args.Length == 0)
-        {
-            Console.WriteLine("Please enter a database connection string argument.");
-            WriteUsageToConsole();
-            return 1;
-        }
-
-        if (args.Length == 1 && (args[0] == "--verbose" || args[0] == "-v"))
-        {
-            Console.WriteLine($"Please enter a database connection string argument before {args[0]} option.");
-            WriteUsageToConsole();
-            return 1;
-        }
-
-        var databaseConnectionString = args[0];
-
-        var verbose = false;
-
-        if (args.Length == 2 && (args[1] == "--verbose" || args[1] == "-v"))
-        {
-            verbose = true;
-        }
-
-        var success = MigrateDatabase(databaseConnectionString, verbose);
-
-        if (!success)
-        {
-            return -1;
-        }
-
-        return 0;
+        return new AppRunner<Program>().Run(args);
     }
 
+    [DefaultCommand]
+    public void Execute(
+        [Operand(Description = "Database connection string")]
+        string databaseConnectionString,
+        [Option('v', "verbose", Description = "Enable verbose output of migrator logs")]
+        bool verbose = false,
+        [Option('r', "repeatable", Description = "Mark scripts as repeatable")]
+        bool repeatable = false,
+        [Option('f', "folder", Description = "Folder name of database scripts")]
+        string folderName = "DbScripts") => MigrateDatabase(databaseConnectionString, verbose, repeatable, folderName);
+
     private static void WriteUsageToConsole()
     {
         Console.WriteLine("Usage: MsSqlMigratorUtility <database-connection-string>");
         Console.WriteLine("Usage: MsSqlMigratorUtility <database-connection-string> -v|--verbose (for verbose output of migrator logs)");
+        Console.WriteLine("Usage: MsSqlMigratorUtility <database-connection-string> -r|--repeatable (for marking scripts as repeatable) -f|--folder <folder-name-in-migrator-project> (for specifying folder name of scripts)");
+        Console.WriteLine("Usage: MsSqlMigratorUtility <database-connection-string> -v|--verbose (for verbose output of migrator logs) -r|--repeatable (for marking scripts as repeatable) -f|--folder <folder-name-in-migrator-project> (for specifying folder name of scripts)");
     }
 
-    private static bool MigrateDatabase(string databaseConnectionString, bool verbose = false, int attempt = 1)
+    private static bool MigrateDatabase(string databaseConnectionString, bool verbose = false, bool repeatable = false, string folderName = "")
     {
+        Console.WriteLine($"repeatable: {repeatable}");
+        Console.WriteLine($"folderName: {folderName}");
         var logger = CreateLogger(verbose);
 
         var migrator = new DbMigrator(databaseConnectionString, logger);
-        var success = migrator.MigrateMsSqlDatabaseWithRetries(verbose);
+        bool success = false;
+        if (!string.IsNullOrWhiteSpace(folderName))
+        {
+            success = migrator.MigrateMsSqlDatabaseWithRetries(verbose, repeatable, folderName);
+        }
+        else
+        {
+            success = migrator.MigrateMsSqlDatabaseWithRetries(verbose, repeatable);
+        }
 
         return success;
     }
@@ -75,4 +71,24 @@ private static ILogger<DbMigrator> CreateLogger(bool verbose)
         var logger = loggerFactory.CreateLogger<DbMigrator>();
         return logger;
     }
+
+    private static void ParseParameters(string[] args)
+    {
+        Parameters = new Dictionary<string, string>();
+        for (var i = 0; i < args.Length; i += 2)
+        {
+            if (!args[i].StartsWith("-"))
+            {
+                continue;
+            }
+            if (i + 1 <= args.Length && !args[i + 1].StartsWith("-"))
+            {
+                Parameters.Add(args[i].Substring(1), args[i + 1]);
+            }
+            else
+            {
+                Parameters.Add(args[i].Substring(1), null);
+            }
+        }
+    }
 }

util/MsSqlMigratorUtility/packages.lock.json

@@ -2,6 +2,15 @@
   "version": 1,
   "dependencies": {
     "net6.0": {
+      "CommandDotNet": {
+        "type": "Direct",
+        "requested": "[7.0.2, )",
+        "resolved": "7.0.2",
+        "contentHash": "sFfqn4T6Ux4AbGnhS+BZvf9iVeP6b9p9bwaMT8nsefVcYH2tC9MHj3AoY+GG0nnBPmFawRqdgud08cBjBdPByQ==",
+        "dependencies": {
+          "Microsoft.CSharp": "4.7.0"
+        }
+      },
       "Microsoft.Extensions.Logging": {
         "type": "Direct",
         "requested": "[6.0.0, )",

util/Setup/Program.cs

@@ -190,7 +190,12 @@ private static void MigrateDatabase(int attempt = 1)
         var vaultConnectionString = Helpers.GetValueFromEnvFile("global",
             "globalSettings__sqlServer__connectionString");
         var migrator = new DbMigrator(vaultConnectionString, null);
-        migrator.MigrateMsSqlDatabaseWithRetries(false);
+
+        var log = false;
+
+        migrator.MigrateMsSqlDatabaseWithRetries(log);
+
+        migrator.MigrateMsSqlDatabaseWithRetries(log, true, "DbScripts_data_migration");
     }
 
     private static bool ValidateInstallation()