From 470357be6b41e07bb46d5f3afab715386b546552 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 5 Sep 2023 10:20:00 -0700 Subject: [PATCH 01/10] Update submodule --- src/ga4gh-tes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ga4gh-tes b/src/ga4gh-tes index 3229c6294..23abdadb4 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit 3229c629494ee35ff5ec595d8106a93433f15ac4 +Subproject commit 23abdadb4f0ae9ef72d84fa12ea8695523005bd3 From c6a52a7117588897e39076b339cfc6a3be59111c Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 7 Sep 2023 11:02:23 -0700 Subject: [PATCH 02/10] Update TES submodule --- src/ga4gh-tes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ga4gh-tes b/src/ga4gh-tes index 23abdadb4..5d6ce72ec 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit 23abdadb4f0ae9ef72d84fa12ea8695523005bd3 +Subproject commit 5d6ce72ec47de71584217eb0e321dd69132465f7 From 1fe034722a293878746b77afba7dd5923d33c95c Mon Sep 17 00:00:00 2001 From: Blair L Murri Date: Wed, 13 Sep 2023 08:49:27 -0700 Subject: [PATCH 03/10] Revert blob-csi-driver version (#699) --- src/deploy-cromwell-on-azure/KubernetesManager.cs | 2 +- src/ga4gh-tes | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/deploy-cromwell-on-azure/KubernetesManager.cs b/src/deploy-cromwell-on-azure/KubernetesManager.cs index c3275a30e..75a746780 100644 --- a/src/deploy-cromwell-on-azure/KubernetesManager.cs +++ b/src/deploy-cromwell-on-azure/KubernetesManager.cs @@ -40,7 +40,7 @@ internal class KubernetesManager // "master" is used despite not being a best practice: https://github.com/kubernetes-sigs/blob-csi-driver/issues/783 private const string BlobCsiDriverGithubReleaseBranch = "master"; - private const string BlobCsiDriverGithubReleaseVersion = "v1.22.1"; + private const string BlobCsiDriverGithubReleaseVersion = "v1.21.4"; private const string BlobCsiRepo = $"https://raw.githubusercontent.com/kubernetes-sigs/blob-csi-driver/{BlobCsiDriverGithubReleaseBranch}/charts"; private const string AadPluginGithubReleaseVersion = "v1.8.13"; private const string AadPluginRepo = $"https://raw.githubusercontent.com/Azure/aad-pod-identity/{AadPluginGithubReleaseVersion}/charts"; diff --git a/src/ga4gh-tes b/src/ga4gh-tes index 5d6ce72ec..a621e1282 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit 5d6ce72ec47de71584217eb0e321dd69132465f7 +Subproject commit a621e12824e44edcd947ef4e8b6362ef0f54e9f9 From fa59a044fa854755671e642cccc0783f3b291cb7 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 14 Sep 2023 10:57:48 -0700 Subject: [PATCH 04/10] Update TES submodule --- src/ga4gh-tes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ga4gh-tes b/src/ga4gh-tes index a621e1282..da81f82cd 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit a621e12824e44edcd947ef4e8b6362ef0f54e9f9 +Subproject commit da81f82cd49b0695b93354f1eaf352eca112bf4e From 6a23830b181713f4b085a0dc56310482ca7d4b60 Mon Sep 17 00:00:00 2001 From: Jonathon Saunders Date: Fri, 15 Sep 2023 06:09:58 -0700 Subject: [PATCH 05/10] Register EncryptionAtHost and enable for AKS (#670) * Register EncryptionAtHost and enable for AKS --- src/deploy-cromwell-on-azure/Deployer.cs | 77 +++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/src/deploy-cromwell-on-azure/Deployer.cs b/src/deploy-cromwell-on-azure/Deployer.cs index 8780930e1..c11e4d27c 100644 --- a/src/deploy-cromwell-on-azure/Deployer.cs +++ b/src/deploy-cromwell-on-azure/Deployer.cs @@ -17,6 +17,7 @@ using Azure.ResourceManager; using Azure.ResourceManager.Network; using Azure.ResourceManager.Network.Models; +using Azure.ResourceManager.Resources; using Azure.Security.KeyVault.Secrets; using Azure.Storage; using Azure.Storage.Blobs; @@ -106,6 +107,11 @@ public class Deployer "Microsoft.DBforPostgreSQL" }; + private readonly Dictionary> requiredResourceProviderFeatures = new Dictionary>() + { + { "Microsoft.Compute", new List { "EncryptionAtHost" } } + }; + private Configuration configuration { get; set; } private ITokenProvider tokenProvider; private TokenCredentials tokenCredentials; @@ -374,7 +380,7 @@ await Execute("Connecting to Azure Services...", async () => } await RegisterResourceProvidersAsync(); - await ValidateVmAsync(); + await RegisterResourceProviderFeaturesAsync(); if (batchAccount is null) { @@ -780,6 +786,7 @@ private async Task ProvisionManagedCluster(IResource resourceGro VmSize = configuration.VmSize, OsDiskSizeGB = 128, OsDiskType = OSDiskType.Managed, + EnableEncryptionAtHost = true, Type = "VirtualMachineScaleSets", EnableAutoScaling = false, EnableNodePublicIP = false, @@ -1059,6 +1066,74 @@ private async Task> GetRequiredResourceProvidersNotRegisteredAsync( return notRegisteredResourceProviders; } + private async Task RegisterResourceProviderFeaturesAsync() + { + var unregisteredFeatures = new List(); + try + { + await Execute( + $"Registering resource provider features...", + async () => + { + var subscription = armClient.GetSubscriptionResource(new ResourceIdentifier($"/subscriptions/{configuration.SubscriptionId}")); + + foreach (var rpName in requiredResourceProviderFeatures.Keys) + { + var rp = await subscription.GetResourceProviderAsync(rpName); + + foreach (var featureName in requiredResourceProviderFeatures[rpName]) + { + var feature = await rp.Value.GetFeatureAsync(featureName); + + if (!string.Equals(feature.Value.Data.FeatureState, "Registered", StringComparison.OrdinalIgnoreCase)) + { + unregisteredFeatures.Add(feature); + _ = await feature.Value.RegisterAsync(); + } + } + } + + while (!cts.IsCancellationRequested) + { + if (unregisteredFeatures.Count == 0) + { + break; + } + + await Task.Delay(System.TimeSpan.FromSeconds(30)); + var finished = new List(); + + foreach (var feature in unregisteredFeatures) + { + var update = await feature.GetAsync(); + + if (string.Equals(update.Value.Data.FeatureState, "Registered", StringComparison.OrdinalIgnoreCase)) + { + finished.Add(feature); + } + } + unregisteredFeatures.RemoveAll(x => finished.Contains(x)); + } + }); + } + catch (Microsoft.Rest.Azure.CloudException ex) when (ex.ToCloudErrorType() == CloudErrorType.AuthorizationFailed) + { + ConsoleEx.WriteLine(); + ConsoleEx.WriteLine("Unable to programatically register the required features.", ConsoleColor.Red); + ConsoleEx.WriteLine("This can happen if you don't have the Owner or Contributor role assignment for the subscription.", ConsoleColor.Red); + ConsoleEx.WriteLine(); + ConsoleEx.WriteLine("Please contact the Owner or Contributor of your Azure subscription, and have them:", ConsoleColor.Yellow); + ConsoleEx.WriteLine(); + ConsoleEx.WriteLine("1. For each of the following, execute 'az feature register --namespace {RESOURCE_PROVIDER_NAME} --name {FEATURE_NAME}'", ConsoleColor.Yellow); + ConsoleEx.WriteLine(); + unregisteredFeatures.ForEach(f => ConsoleEx.WriteLine($"- {f.Data.Name}", ConsoleColor.Yellow)); + ConsoleEx.WriteLine(); + ConsoleEx.WriteLine("After completion, please re-attempt deployment."); + + Environment.Exit(1); + } + } + private Task AssignManagedIdOperatorToResourceAsync(IIdentity managedIdentity, IResource resource) { // https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#managed-identity-operator From 877d6feadf42f7dee208332658679265b7208ec5 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Fri, 15 Sep 2023 09:45:02 -0700 Subject: [PATCH 06/10] Update TES submodule --- src/ga4gh-tes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ga4gh-tes b/src/ga4gh-tes index da81f82cd..ae1bb20f5 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit da81f82cd49b0695b93354f1eaf352eca112bf4e +Subproject commit ae1bb20f51584522d150d1e37bce14562d9ea11c From fef42b8f1b3efcd5fa7c3fb9b641fef7b448093c Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Mon, 18 Sep 2023 13:11:21 -0700 Subject: [PATCH 07/10] Update TES submodule --- src/ga4gh-tes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ga4gh-tes b/src/ga4gh-tes index ae1bb20f5..01d51ae64 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit ae1bb20f51584522d150d1e37bce14562d9ea11c +Subproject commit 01d51ae648db62bac2f8d3903927561bd1068b25 From bd680c5cc5ed12aadb69186d5e6be312056ce1c4 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Mon, 18 Sep 2023 16:11:24 -0700 Subject: [PATCH 08/10] Update TES submodule --- src/ga4gh-tes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ga4gh-tes b/src/ga4gh-tes index 01d51ae64..5e7cf0feb 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit 01d51ae648db62bac2f8d3903927561bd1068b25 +Subproject commit 5e7cf0feb74797d93c2f4136a8ce07bf671475ca From 011c74129eca30824370be3107a43bd29934087a Mon Sep 17 00:00:00 2001 From: Jonathon Saunders Date: Wed, 20 Sep 2023 12:22:08 -0700 Subject: [PATCH 09/10] Attach nsg to all subnets (#698) * Create NSGs for all vnets * Switch to one default nsg --- src/deploy-cromwell-on-azure/Deployer.cs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/deploy-cromwell-on-azure/Deployer.cs b/src/deploy-cromwell-on-azure/Deployer.cs index c11e4d27c..f1b3ebd4f 100644 --- a/src/deploy-cromwell-on-azure/Deployer.cs +++ b/src/deploy-cromwell-on-azure/Deployer.cs @@ -1400,21 +1400,27 @@ private Task AssignVmAsContributorToAppInsightsAsync(IIdentity managedIdentity, $"Creating virtual network and subnets: {configuration.VnetName}...", async () => { + var defaultNsg = await CreateNetworkSecurityGroupAsync(resourceGroup, $"{configuration.VnetName}-default-nsg"); + var vnetDefinition = azureSubscriptionClient.Networks .Define(configuration.VnetName) .WithRegion(configuration.RegionName) .WithExistingResourceGroup(resourceGroup) .WithAddressSpace(configuration.VnetAddressSpace) .DefineSubnet(configuration.VmSubnetName) - .WithAddressPrefix(configuration.VmSubnetAddressSpace).Attach(); + .WithAddressPrefix(configuration.VmSubnetAddressSpace) + .WithExistingNetworkSecurityGroup(defaultNsg) + .Attach(); vnetDefinition = vnetDefinition.DefineSubnet(configuration.PostgreSqlSubnetName) .WithAddressPrefix(configuration.PostgreSqlSubnetAddressSpace) + .WithExistingNetworkSecurityGroup(defaultNsg) .WithDelegation("Microsoft.DBforPostgreSQL/flexibleServers") .Attach(); vnetDefinition = vnetDefinition.DefineSubnet(configuration.BatchSubnetName) .WithAddressPrefix(configuration.BatchNodesSubnetAddressSpace) + .WithExistingNetworkSecurityGroup(defaultNsg) .Attach(); var vnet = await vnetDefinition.CreateAsync(); @@ -1433,6 +1439,14 @@ private Task AssignVmAsContributorToAppInsightsAsync(IIdentity managedIdentity, batchSubnet); }); + private Task CreateNetworkSecurityGroupAsync(IResourceGroup resourceGroup, string networkSecurityGroupName) + { + return azureSubscriptionClient.NetworkSecurityGroups.Define(networkSecurityGroupName) + .WithRegion(configuration.RegionName) + .WithExistingResourceGroup(resourceGroup) + .CreateAsync(cts.Token); + } + private string GetFormattedPostgresqlUser(bool isCromwellPostgresUser) { var user = isCromwellPostgresUser ? From e4b0d0a45bdfd47a4f66f3ef08f797758e3ee275 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 21 Sep 2023 14:59:31 -0700 Subject: [PATCH 10/10] Update TES submodule --- src/ga4gh-tes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ga4gh-tes b/src/ga4gh-tes index 5e7cf0feb..0165bbea4 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit 5e7cf0feb74797d93c2f4136a8ce07bf671475ca +Subproject commit 0165bbea44b3eb6a33dade0ad1ac73a829adbd94