From 246fafa320e5d36f636ca552203788a8b597cd48 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 27 Jun 2023 15:32:00 -0700 Subject: [PATCH 01/48] Add long running test --- src/TriggerService.Tests/IntegrationTests.cs | 68 +++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 113f06d7..34515653 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -2,13 +2,16 @@ // Licensed under the MIT License. using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net.Http; using System.Threading.Tasks; using Azure.Storage.Blobs; using Common; using Microsoft.Azure.Batch; using Microsoft.Azure.Batch.Auth; +using Microsoft.Azure.Management.CosmosDB.Fluent.Models; using Microsoft.VisualStudio.TestTools.UnitTesting; namespace TriggerService.Tests @@ -77,6 +80,22 @@ public async Task RunScaleTestWithMutect2Async() await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName); } + /// + /// To run this test, specify a testStorageAccountName, a workflowsContainerSasToken, and remove the [Ignore] attribute + /// + /// + [Ignore] + [TestCategory("Integration")] + [TestMethod] + public async Task RunScaleTestWithMutect2WaitTilDoneAsync() + { + const int countOfWorkflowsToRun = 10; + const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.trigger.json"; + const string workflowFriendlyName = $"mutect2"; + + await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, waitTilDone: true); + } + /// /// To run this test, specify a testStorageAccountName, a workflowsContainerSasToken, and remove the [Ignore] attribute /// @@ -150,22 +169,69 @@ public async Task DeleteOldBatchPoolsAsync() Console.WriteLine($"Deleted {count} pools."); } - private static async Task StartWorkflowsAsync(int countOfWorkflowsToRun, string triggerFile, string workflowFriendlyName) + private static async Task StartWorkflowsAsync(int countOfWorkflowsToRun, string triggerFile, string workflowFriendlyName, bool waitTilDone = false) { const string containerName = "workflows"; var n = DateTime.UtcNow; var date = $"{n.Year}-{n.Month}-{n.Day}-{n.Hour}-{n.Minute}"; using var httpClient = new HttpClient(); var triggerFileJson = await (await httpClient.GetAsync(triggerFile)).Content.ReadAsStringAsync(); + var blobNames = new List(); for (var i = 1; i <= countOfWorkflowsToRun; i++) { // example: new/mutect2-001-of-100-2023-4-7-3-9.json var blobName = $"new/{workflowFriendlyName}-{i:D4}-of-{countOfWorkflowsToRun:D4}-{date}.json"; + blobNames.Add(blobName); var blobClient = new BlobServiceClient(new Uri($"https://{testStorageAccountName}.blob.core.windows.net/{containerName}/{blobName}?{workflowsContainerSasToken.TrimStart('?')}")); var container = blobClient.GetBlobContainerClient(containerName); await container.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(triggerFileJson), true); } + + if (waitTilDone) + { + int succeededCount = 0; + int failedCount = 0; + + while (succeededCount + failedCount < countOfWorkflowsToRun) + { + try + { + var blobClient = new BlobServiceClient(new Uri($"https://{testStorageAccountName}.blob.core.windows.net/{containerName}/?{workflowsContainerSasToken.TrimStart('?')}")); + var container = blobClient.GetBlobContainerClient(containerName); + var enumerator = container.GetBlobsAsync().GetAsyncEnumerator(); + var existingBlobNames = new List(); + + while (await enumerator.MoveNextAsync()) + { + // example: inprogress/mutect2-001-of-100-2023-4-7-3-9.0fb0858a-3166-4a22-85b6-4337df2f53c5.json + var blobName = enumerator.Current.Name; + existingBlobNames.Add(blobName); + } + + succeededCount = existingBlobNames + .Count(name => blobNames.Any(b => name + .StartsWith(b.Replace("new/", "succeeded/")))); + + failedCount = existingBlobNames + .Count(name => blobNames.Any(b => name + .StartsWith(b.Replace("new/", "failed/")))); + + Console.WriteLine($"Succeeded count: {succeededCount}"); + Console.WriteLine($"Failed count: {succeededCount}"); + } + catch (Exception exc) + { + Console.WriteLine(exc); + } + + await Task.Delay(TimeSpan.FromSeconds(5)); + } + + Console.WriteLine($"Completed in {(DateTime.UtcNow - n).TotalHours:n1} hours"); + Console.WriteLine($"Succeeded count: {succeededCount}"); + Console.WriteLine($"Failed count: {succeededCount}"); + } } } } From 4f4c24d84e4d1c28551f8e70dfd6e174b3ea7c48 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 27 Jun 2023 16:02:11 -0700 Subject: [PATCH 02/48] update test --- src/TriggerService.Tests/IntegrationTests.cs | 51 +++++++++---------- .../TriggerService.Tests.csproj | 1 + 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 34515653..868638a8 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -7,11 +7,11 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; +using Azure.Identity; using Azure.Storage.Blobs; using Common; using Microsoft.Azure.Batch; using Microsoft.Azure.Batch.Auth; -using Microsoft.Azure.Management.CosmosDB.Fluent.Models; using Microsoft.VisualStudio.TestTools.UnitTesting; namespace TriggerService.Tests @@ -20,7 +20,6 @@ namespace TriggerService.Tests public class IntegrationTests { private const string testStorageAccountName = ""; - private const string workflowsContainerSasToken = ""; /// /// To run this test, specify a testStorageAccountName, a workflowsContainerSasToken, and remove the [Ignore] attribute @@ -32,38 +31,33 @@ public class IntegrationTests public async Task RunGlobTestWdlAsync() { const string containerName = "inputs"; + var blobServiceClient = new BlobServiceClient(new Uri($"https://{testStorageAccountName}.blob.core.windows.net/"), new AzureCliCredential()); var wdlBlobName = $"globtest.wdl"; var wdlPath = Path.Combine(Path.GetFullPath(@"..\..\..\test-wdls\globtest"), wdlBlobName); - string wdlUrl = $"https://{testStorageAccountName}.blob.core.windows.net/{containerName}/test/{wdlBlobName}?{workflowsContainerSasToken.TrimStart('?')}"; - var blobClient = new BlobServiceClient(new Uri(wdlUrl)); - var container = blobClient.GetBlobContainerClient(containerName); + var container = blobServiceClient.GetBlobContainerClient(containerName); var text = (await File.ReadAllTextAsync(wdlPath)).Replace(@"\r\n\", @"\n"); await container.GetBlobClient("test/" + wdlBlobName).UploadAsync(BinaryData.FromString(text), true); var wdlInputsBlobName = $"globtestinputs.json"; var wdlInputsPath = Path.Combine(Path.GetFullPath(@"..\..\..\test-wdls\globtest"), wdlInputsBlobName); - string wdlInputsUrl = $"https://{testStorageAccountName}.blob.core.windows.net/{containerName}/test/{wdlInputsBlobName}?{workflowsContainerSasToken.TrimStart('?')}"; - blobClient = new BlobServiceClient(new Uri(wdlInputsUrl)); - container = blobClient.GetBlobContainerClient(containerName); text = (await File.ReadAllTextAsync(wdlInputsPath)).Replace(@"\r\n\", @"\n"); await container.GetBlobClient("test/" + wdlInputsBlobName).UploadAsync(BinaryData.FromString(text), true); var workflowTrigger = new Workflow { - WorkflowUrl = wdlUrl, - WorkflowInputsUrl = wdlInputsUrl + WorkflowUrl = blobServiceClient.Uri + "/" + containerName + "/test/" + wdlBlobName, + WorkflowInputsUrl = blobServiceClient.Uri + "/" + containerName + "/test/" + wdlInputsBlobName }; var n = DateTime.UtcNow; var date = $"{n.Year}-{n.Month}-{n.Day}-{n.Hour}-{n.Minute}"; var triggerFileBlobName = $"new/globtest-{date}.json"; string triggerJson = System.Text.Json.JsonSerializer.Serialize(workflowTrigger).Replace(@"\r\n\", @"\n"); - var triggerUrl = $"https://{testStorageAccountName}.blob.core.windows.net?{workflowsContainerSasToken.TrimStart('?')}"; - blobClient = new BlobServiceClient(new Uri(triggerUrl)); - container = blobClient.GetBlobContainerClient("workflows"); + container = blobServiceClient.GetBlobContainerClient("workflows"); await container.GetBlobClient(triggerFileBlobName).UploadAsync(BinaryData.FromString(triggerJson), true); } + /// /// To run this test, specify a testStorageAccountName, a workflowsContainerSasToken, and remove the [Ignore] attribute /// @@ -77,7 +71,7 @@ public async Task RunScaleTestWithMutect2Async() const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; - await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName); + await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, testStorageAccountName); } /// @@ -92,8 +86,9 @@ public async Task RunScaleTestWithMutect2WaitTilDoneAsync() const int countOfWorkflowsToRun = 10; const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; + string storageAccountName = Environment.GetEnvironmentVariable("STORAGE_ACCOUNT_NAME"); - await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, waitTilDone: true); + await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, storageAccountName, waitTilDone: true); } /// @@ -109,7 +104,7 @@ public async Task RunScaleTestWithWholeGenomeGermlineSingleSampleAsync() const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-genome-processing-pipeline-azure/main-azure/WholeGenomeGermlineSingleSample.trigger.json"; const string workflowFriendlyName = $"wgs-germline"; - await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName); + await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, testStorageAccountName); } /// @@ -122,9 +117,8 @@ public async Task RunScaleTestWithWholeGenomeGermlineSingleSampleAsync() public async Task CancelAllRunningWorkflowsAsync() { const string containerName = "workflows"; - - var blobClient = new BlobServiceClient(new Uri($"https://{testStorageAccountName}.blob.core.windows.net?{workflowsContainerSasToken.TrimStart('?')}")); - var container = blobClient.GetBlobContainerClient(containerName); + var blobServiceClient = new BlobServiceClient(new Uri($"https://{testStorageAccountName}.blob.core.windows.net/"), new AzureCliCredential()); + var container = blobServiceClient.GetBlobContainerClient(containerName); var enumerator = container.GetBlobsAsync(prefix: "inprogress/").GetAsyncEnumerator(); while (await enumerator.MoveNextAsync()) @@ -169,7 +163,12 @@ public async Task DeleteOldBatchPoolsAsync() Console.WriteLine($"Deleted {count} pools."); } - private static async Task StartWorkflowsAsync(int countOfWorkflowsToRun, string triggerFile, string workflowFriendlyName, bool waitTilDone = false) + private static async Task StartWorkflowsAsync( + int countOfWorkflowsToRun, + string triggerFile, + string workflowFriendlyName, + string storageAccountName, + bool waitTilDone = false) { const string containerName = "workflows"; var n = DateTime.UtcNow; @@ -177,14 +176,14 @@ private static async Task StartWorkflowsAsync(int countOfWorkflowsToRun, string using var httpClient = new HttpClient(); var triggerFileJson = await (await httpClient.GetAsync(triggerFile)).Content.ReadAsStringAsync(); var blobNames = new List(); + var blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/"), new AzureCliCredential()); for (var i = 1; i <= countOfWorkflowsToRun; i++) { // example: new/mutect2-001-of-100-2023-4-7-3-9.json var blobName = $"new/{workflowFriendlyName}-{i:D4}-of-{countOfWorkflowsToRun:D4}-{date}.json"; blobNames.Add(blobName); - var blobClient = new BlobServiceClient(new Uri($"https://{testStorageAccountName}.blob.core.windows.net/{containerName}/{blobName}?{workflowsContainerSasToken.TrimStart('?')}")); - var container = blobClient.GetBlobContainerClient(containerName); + var container = blobServiceClient.GetBlobContainerClient(containerName); await container.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(triggerFileJson), true); } @@ -197,8 +196,7 @@ private static async Task StartWorkflowsAsync(int countOfWorkflowsToRun, string { try { - var blobClient = new BlobServiceClient(new Uri($"https://{testStorageAccountName}.blob.core.windows.net/{containerName}/?{workflowsContainerSasToken.TrimStart('?')}")); - var container = blobClient.GetBlobContainerClient(containerName); + var container = blobServiceClient.GetBlobContainerClient(containerName); var enumerator = container.GetBlobsAsync().GetAsyncEnumerator(); var existingBlobNames = new List(); @@ -218,7 +216,7 @@ private static async Task StartWorkflowsAsync(int countOfWorkflowsToRun, string .StartsWith(b.Replace("new/", "failed/")))); Console.WriteLine($"Succeeded count: {succeededCount}"); - Console.WriteLine($"Failed count: {succeededCount}"); + Console.WriteLine($"Failed count: {failedCount}"); } catch (Exception exc) { @@ -230,8 +228,9 @@ private static async Task StartWorkflowsAsync(int countOfWorkflowsToRun, string Console.WriteLine($"Completed in {(DateTime.UtcNow - n).TotalHours:n1} hours"); Console.WriteLine($"Succeeded count: {succeededCount}"); - Console.WriteLine($"Failed count: {succeededCount}"); + Console.WriteLine($"Failed count: {failedCount}"); } } + } } diff --git a/src/TriggerService.Tests/TriggerService.Tests.csproj b/src/TriggerService.Tests/TriggerService.Tests.csproj index df7ff7f1..1ff98457 100644 --- a/src/TriggerService.Tests/TriggerService.Tests.csproj +++ b/src/TriggerService.Tests/TriggerService.Tests.csproj @@ -15,6 +15,7 @@ + From 9d8ac214a4007949ec09827701a7834c07503fa7 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 27 Jun 2023 16:23:03 -0700 Subject: [PATCH 03/48] add file --- src/TriggerService.Tests/IntegrationTests.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 868638a8..878190d7 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -78,15 +78,22 @@ public async Task RunScaleTestWithMutect2Async() /// To run this test, specify a testStorageAccountName, a workflowsContainerSasToken, and remove the [Ignore] attribute /// /// - [Ignore] [TestCategory("Integration")] [TestMethod] public async Task RunScaleTestWithMutect2WaitTilDoneAsync() { + // This is set in the Azure Devops pipeline + const string storageAccountNamePath = "temp_storage_account_name.txt"; + + if (!File.Exists(storageAccountNamePath)) + { + return; + } + + string storageAccountName = (await File.ReadAllTextAsync(storageAccountNamePath)).Trim(); const int countOfWorkflowsToRun = 10; const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; - string storageAccountName = Environment.GetEnvironmentVariable("STORAGE_ACCOUNT_NAME"); await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, storageAccountName, waitTilDone: true); } From 82c5389010720639a41d247016fdd93853813b9f Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 27 Jun 2023 16:26:24 -0700 Subject: [PATCH 04/48] test --- src/TriggerService.Tests/IntegrationTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 878190d7..ea7458f6 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -238,6 +238,5 @@ private static async Task StartWorkflowsAsync( Console.WriteLine($"Failed count: {failedCount}"); } } - } } From 2ec8e974fed109515bd93f72bda4701b475821fc Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 29 Jun 2023 10:55:21 -0700 Subject: [PATCH 05/48] update working directory --- src/TriggerService.Tests/IntegrationTests.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index ea7458f6..c4721f26 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -84,13 +84,17 @@ public async Task RunScaleTestWithMutect2WaitTilDoneAsync() { // This is set in the Azure Devops pipeline const string storageAccountNamePath = "temp_storage_account_name.txt"; + var path = Path.Combine(Directory.GetCurrentDirectory(), storageAccountNamePath); - if (!File.Exists(storageAccountNamePath)) + if (!File.Exists(path)) { + Console.WriteLine($"Path not found - exiting integration test: {path}"); return; } - string storageAccountName = (await File.ReadAllTextAsync(storageAccountNamePath)).Trim(); + Console.WriteLine($"Found path: {path}"); + + string storageAccountName = (await File.ReadAllTextAsync(path)).Trim(); const int countOfWorkflowsToRun = 10; const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; From 33e97469a65bb4de424cfc48c9dc59097c6f0566 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 29 Jun 2023 13:43:19 -0700 Subject: [PATCH 06/48] fix path --- src/TriggerService.Tests/IntegrationTests.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index c4721f26..4b91616e 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -82,9 +82,11 @@ public async Task RunScaleTestWithMutect2Async() [TestMethod] public async Task RunScaleTestWithMutect2WaitTilDoneAsync() { - // This is set in the Azure Devops pipeline - const string storageAccountNamePath = "temp_storage_account_name.txt"; - var path = Path.Combine(Directory.GetCurrentDirectory(), storageAccountNamePath); + // This is set in the Azure Devops pipeline, which writes the file to the .csproj directory + // The current working directory is this: /mnt/vss/_work/r1/a/CoaArtifacts/AllSource/TriggerService.Tests/bin/Debug/net7.0/ + // And the file is available here: /mnt/vss/_work/r1/a/CoaArtifacts/AllSource/TriggerService.Tests/temp_storage_account_name.txt + const string storageAccountNamePath = "../../../temp_storage_account_name.txt"; + var path = storageAccountNamePath; if (!File.Exists(path)) { From c13c3cc85f1c4b44da17c7a69aa8a2ef27c5e080 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 29 Jun 2023 14:53:57 -0700 Subject: [PATCH 07/48] use default azure creds --- src/TriggerService.Tests/IntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 4b91616e..381b9f88 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -189,7 +189,7 @@ private static async Task StartWorkflowsAsync( using var httpClient = new HttpClient(); var triggerFileJson = await (await httpClient.GetAsync(triggerFile)).Content.ReadAsStringAsync(); var blobNames = new List(); - var blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/"), new AzureCliCredential()); + var blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/"), new DefaultAzureCredential()); for (var i = 1; i <= countOfWorkflowsToRun; i++) { From bc58e45f1025eed9bfe58bd8533b0dc7483b8af7 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Fri, 30 Jun 2023 15:28:03 -0700 Subject: [PATCH 08/48] fix azure cli credential --- src/TriggerService.Tests/IntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 381b9f88..4b91616e 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -189,7 +189,7 @@ private static async Task StartWorkflowsAsync( using var httpClient = new HttpClient(); var triggerFileJson = await (await httpClient.GetAsync(triggerFile)).Content.ReadAsStringAsync(); var blobNames = new List(); - var blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/"), new DefaultAzureCredential()); + var blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/"), new AzureCliCredential()); for (var i = 1; i <= countOfWorkflowsToRun; i++) { From b87f4138294af987e1e6270c51aa1b98441dc591 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Fri, 30 Jun 2023 16:16:38 -0700 Subject: [PATCH 09/48] fail test if any failed workflows --- src/TriggerService.Tests/IntegrationTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 4b91616e..e8b77d4e 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -242,6 +242,8 @@ private static async Task StartWorkflowsAsync( Console.WriteLine($"Completed in {(DateTime.UtcNow - n).TotalHours:n1} hours"); Console.WriteLine($"Succeeded count: {succeededCount}"); Console.WriteLine($"Failed count: {failedCount}"); + + Assert.IsTrue(failedCount == 0); } } } From f630c9134e29693a09e8a6aadac7eaf183256ce9 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Fri, 30 Jun 2023 16:28:45 -0700 Subject: [PATCH 10/48] add comments and minor optimizations --- src/TriggerService.Tests/IntegrationTests.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index e8b77d4e..d64accbf 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -187,10 +187,13 @@ private static async Task StartWorkflowsAsync( var n = DateTime.UtcNow; var date = $"{n.Year}-{n.Month}-{n.Day}-{n.Hour}-{n.Minute}"; using var httpClient = new HttpClient(); + + // 1. Get the publically available trigger file var triggerFileJson = await (await httpClient.GetAsync(triggerFile)).Content.ReadAsStringAsync(); var blobNames = new List(); var blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/"), new AzureCliCredential()); + // 2. Start the workflows by uploading new trigger files for (var i = 1; i <= countOfWorkflowsToRun; i++) { // example: new/mutect2-001-of-100-2023-4-7-3-9.json @@ -200,6 +203,7 @@ private static async Task StartWorkflowsAsync( await container.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(triggerFileJson), true); } + // 3. Loop forever until they are all in a terminal state (succeeded or failed) if (waitTilDone) { int succeededCount = 0; @@ -236,7 +240,12 @@ private static async Task StartWorkflowsAsync( Console.WriteLine(exc); } - await Task.Delay(TimeSpan.FromSeconds(5)); + if (succeededCount + failedCount >= countOfWorkflowsToRun) + { + break; + } + + await Task.Delay(TimeSpan.FromMinutes(1)); } Console.WriteLine($"Completed in {(DateTime.UtcNow - n).TotalHours:n1} hours"); From b12be46d540512a558e885fab64c84a054b9e342 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Fri, 30 Jun 2023 16:39:05 -0700 Subject: [PATCH 11/48] update comments --- src/TriggerService.Tests/IntegrationTests.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index d64accbf..2b527164 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -183,17 +183,18 @@ private static async Task StartWorkflowsAsync( string storageAccountName, bool waitTilDone = false) { + var startTime = DateTime.UtcNow; const string containerName = "workflows"; - var n = DateTime.UtcNow; - var date = $"{n.Year}-{n.Month}-{n.Day}-{n.Hour}-{n.Minute}"; - using var httpClient = new HttpClient(); + var blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/"), new AzureCliCredential()); // 1. Get the publically available trigger file + using var httpClient = new HttpClient(); var triggerFileJson = await (await httpClient.GetAsync(triggerFile)).Content.ReadAsStringAsync(); - var blobNames = new List(); - var blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/"), new AzureCliCredential()); // 2. Start the workflows by uploading new trigger files + var blobNames = new List(); + var date = $"{startTime.Year}-{startTime.Month}-{startTime.Day}-{startTime.Hour}-{startTime.Minute}"; + for (var i = 1; i <= countOfWorkflowsToRun; i++) { // example: new/mutect2-001-of-100-2023-4-7-3-9.json @@ -248,7 +249,7 @@ private static async Task StartWorkflowsAsync( await Task.Delay(TimeSpan.FromMinutes(1)); } - Console.WriteLine($"Completed in {(DateTime.UtcNow - n).TotalHours:n1} hours"); + Console.WriteLine($"Completed in {(DateTime.UtcNow - startTime).TotalHours:n1} hours"); Console.WriteLine($"Succeeded count: {succeededCount}"); Console.WriteLine($"Failed count: {failedCount}"); From 605a77616a00ca0df8d775afb8ebfb42aaa2f09a Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Fri, 30 Jun 2023 21:22:16 -0700 Subject: [PATCH 12/48] add support for container sas --- src/TriggerService.Tests/IntegrationTests.cs | 33 +++++++++++++++----- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 2b527164..288635da 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -95,13 +95,14 @@ public async Task RunScaleTestWithMutect2WaitTilDoneAsync() } Console.WriteLine($"Found path: {path}"); - - string storageAccountName = (await File.ReadAllTextAsync(path)).Trim(); + var lines = await File.ReadAllLinesAsync(path); + string storageAccountName = lines[0].Trim(); + string workflowsContainerSasToken = lines[1].Trim(); const int countOfWorkflowsToRun = 10; const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; - await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, storageAccountName, waitTilDone: true); + await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, storageAccountName, waitTilDone: true, workflowsContainerSasToken); } /// @@ -181,11 +182,31 @@ private static async Task StartWorkflowsAsync( string triggerFile, string workflowFriendlyName, string storageAccountName, - bool waitTilDone = false) + bool waitTilDone = false, + string workflowsContainerSasToken = null) { var startTime = DateTime.UtcNow; const string containerName = "workflows"; - var blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/"), new AzureCliCredential()); + + BlobServiceClient blobServiceClient; + if (string.IsNullOrEmpty(workflowsContainerSasToken)) + { + Console.WriteLine("No container SAS token specified; using AzureCliCredential"); + blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/"), new AzureCliCredential()); + } + else + { + Console.WriteLine("Using the specified container SAS token."); + blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/")); + } + + BlobContainerClient container = blobServiceClient.GetBlobContainerClient(containerName); + + if (!string.IsNullOrEmpty(workflowsContainerSasToken)) + { + var containerSasUri = new Uri($"https://{storageAccountName}.blob.core.windows.net/{containerName}?{workflowsContainerSasToken}"); + container = new BlobContainerClient(containerSasUri); + } // 1. Get the publically available trigger file using var httpClient = new HttpClient(); @@ -200,7 +221,6 @@ private static async Task StartWorkflowsAsync( // example: new/mutect2-001-of-100-2023-4-7-3-9.json var blobName = $"new/{workflowFriendlyName}-{i:D4}-of-{countOfWorkflowsToRun:D4}-{date}.json"; blobNames.Add(blobName); - var container = blobServiceClient.GetBlobContainerClient(containerName); await container.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(triggerFileJson), true); } @@ -214,7 +234,6 @@ private static async Task StartWorkflowsAsync( { try { - var container = blobServiceClient.GetBlobContainerClient(containerName); var enumerator = container.GetBlobsAsync().GetAsyncEnumerator(); var existingBlobNames = new List(); From 989cb2480ede8794729c30cb6e28bdba998de2de Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Fri, 30 Jun 2023 21:23:46 -0700 Subject: [PATCH 13/48] minor --- src/TriggerService.Tests/IntegrationTests.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 288635da..e3996e97 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -189,21 +189,22 @@ private static async Task StartWorkflowsAsync( const string containerName = "workflows"; BlobServiceClient blobServiceClient; + if (string.IsNullOrEmpty(workflowsContainerSasToken)) { Console.WriteLine("No container SAS token specified; using AzureCliCredential"); blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/"), new AzureCliCredential()); } else - { - Console.WriteLine("Using the specified container SAS token."); + { blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/")); } - BlobContainerClient container = blobServiceClient.GetBlobContainerClient(containerName); + var container = blobServiceClient.GetBlobContainerClient(containerName); if (!string.IsNullOrEmpty(workflowsContainerSasToken)) { + Console.WriteLine("Using the specified container SAS token."); var containerSasUri = new Uri($"https://{storageAccountName}.blob.core.windows.net/{containerName}?{workflowsContainerSasToken}"); container = new BlobContainerClient(containerSasUri); } From b354224e6937a93d73a4aba608322fcea0f899c1 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 12 Jul 2023 11:45:58 -0700 Subject: [PATCH 14/48] debug --- src/TriggerService.Tests/IntegrationTests.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index e3996e97..e3a3bf10 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -98,6 +98,10 @@ public async Task RunScaleTestWithMutect2WaitTilDoneAsync() var lines = await File.ReadAllLinesAsync(path); string storageAccountName = lines[0].Trim(); string workflowsContainerSasToken = lines[1].Trim(); + + Console.WriteLine($"storageAccountName: [{storageAccountName}]"); + Console.WriteLine($"workflowsContainerSasToken: [{workflowsContainerSasToken}]"); + const int countOfWorkflowsToRun = 10; const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; From ae935ecf638930db8044ac77bd3ce87c87d8035e Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 12 Jul 2023 13:11:35 -0700 Subject: [PATCH 15/48] minor --- src/TriggerService.Tests/IntegrationTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index e3a3bf10..b179dc0a 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -99,6 +99,9 @@ public async Task RunScaleTestWithMutect2WaitTilDoneAsync() string storageAccountName = lines[0].Trim(); string workflowsContainerSasToken = lines[1].Trim(); + Console.WriteLine("All test start:"); + Console.WriteLine($"{File.ReadAllText(path)}"); + Console.WriteLine("All text stop"); Console.WriteLine($"storageAccountName: [{storageAccountName}]"); Console.WriteLine($"workflowsContainerSasToken: [{workflowsContainerSasToken}]"); From e5b968c648155a879339c30385b464e8649e4a54 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 12 Jul 2023 16:07:36 -0700 Subject: [PATCH 16/48] remove double quotes --- src/TriggerService.Tests/IntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index b179dc0a..41fdd083 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -97,7 +97,7 @@ public async Task RunScaleTestWithMutect2WaitTilDoneAsync() Console.WriteLine($"Found path: {path}"); var lines = await File.ReadAllLinesAsync(path); string storageAccountName = lines[0].Trim(); - string workflowsContainerSasToken = lines[1].Trim(); + string workflowsContainerSasToken = lines[1].Trim('"'); Console.WriteLine("All test start:"); Console.WriteLine($"{File.ReadAllText(path)}"); From cb1d3c51f538dd6c1e26e16ff0bbd96d34a0c96a Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 13 Jul 2023 12:40:25 -0700 Subject: [PATCH 17/48] fix waiting logic --- src/TriggerService.Tests/IntegrationTests.cs | 79 +++++++++++--------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 41fdd083..a0b40da2 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -235,53 +235,58 @@ private static async Task StartWorkflowsAsync( // 3. Loop forever until they are all in a terminal state (succeeded or failed) if (waitTilDone) { - int succeededCount = 0; - int failedCount = 0; + await WaitTilAllWorkflowsInTerminalStateAsync(countOfWorkflowsToRun, startTime, container, blobNames); + } + } - while (succeededCount + failedCount < countOfWorkflowsToRun) + private static async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsToRun, DateTime startTime, BlobContainerClient container, List originalBlobNames) + { + int succeededCount = 0; + int failedCount = 0; + + while (succeededCount + failedCount < countOfWorkflowsToRun) + { + try { - try - { - var enumerator = container.GetBlobsAsync().GetAsyncEnumerator(); - var existingBlobNames = new List(); - - while (await enumerator.MoveNextAsync()) - { - // example: inprogress/mutect2-001-of-100-2023-4-7-3-9.0fb0858a-3166-4a22-85b6-4337df2f53c5.json - var blobName = enumerator.Current.Name; - existingBlobNames.Add(blobName); - } - - succeededCount = existingBlobNames - .Count(name => blobNames.Any(b => name - .StartsWith(b.Replace("new/", "succeeded/")))); - - failedCount = existingBlobNames - .Count(name => blobNames.Any(b => name - .StartsWith(b.Replace("new/", "failed/")))); - - Console.WriteLine($"Succeeded count: {succeededCount}"); - Console.WriteLine($"Failed count: {failedCount}"); - } - catch (Exception exc) - { - Console.WriteLine(exc); - } + var enumerator = container.GetBlobsAsync().GetAsyncEnumerator(); + var existingBlobNames = new List(); - if (succeededCount + failedCount >= countOfWorkflowsToRun) + while (await enumerator.MoveNextAsync()) { - break; + // example: inprogress/mutect2-001-of-100-2023-4-7-3-9.0fb0858a-3166-4a22-85b6-4337df2f53c5.json + var blobName = enumerator.Current.Name; + existingBlobNames.Add(blobName); } - await Task.Delay(TimeSpan.FromMinutes(1)); + succeededCount = existingBlobNames + .Count(existingBlobName => originalBlobNames.Any(b => b + .Equals(existingBlobName.Replace("succeeded/", "new/"), StringComparison.OrdinalIgnoreCase))); + + failedCount = existingBlobNames + .Count(name => originalBlobNames.Any(b => b + .Equals(name.Replace("failed/", "new/"), StringComparison.OrdinalIgnoreCase)))); + + Console.WriteLine($"Succeeded count: {succeededCount}"); + Console.WriteLine($"Failed count: {failedCount}"); + } + catch (Exception exc) + { + Console.WriteLine(exc); } - Console.WriteLine($"Completed in {(DateTime.UtcNow - startTime).TotalHours:n1} hours"); - Console.WriteLine($"Succeeded count: {succeededCount}"); - Console.WriteLine($"Failed count: {failedCount}"); + if (succeededCount + failedCount >= countOfWorkflowsToRun) + { + break; + } - Assert.IsTrue(failedCount == 0); + await Task.Delay(TimeSpan.FromMinutes(1)); } + + Console.WriteLine($"Completed in {(DateTime.UtcNow - startTime).TotalHours:n1} hours"); + Console.WriteLine($"Succeeded count: {succeededCount}"); + Console.WriteLine($"Failed count: {failedCount}"); + + Assert.IsTrue(failedCount == 0); } } } From 34c72c6cb43e18ff9897abe85d649e2560d24dfe Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 13 Jul 2023 12:43:37 -0700 Subject: [PATCH 18/48] update logs --- src/TriggerService.Tests/IntegrationTests.cs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index a0b40da2..c1f5eeba 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -99,13 +99,7 @@ public async Task RunScaleTestWithMutect2WaitTilDoneAsync() string storageAccountName = lines[0].Trim(); string workflowsContainerSasToken = lines[1].Trim('"'); - Console.WriteLine("All test start:"); - Console.WriteLine($"{File.ReadAllText(path)}"); - Console.WriteLine("All text stop"); - Console.WriteLine($"storageAccountName: [{storageAccountName}]"); - Console.WriteLine($"workflowsContainerSasToken: [{workflowsContainerSasToken}]"); - - const int countOfWorkflowsToRun = 10; + const int countOfWorkflowsToRun = 1; const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; @@ -266,8 +260,9 @@ private static async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWor .Count(name => originalBlobNames.Any(b => b .Equals(name.Replace("failed/", "new/"), StringComparison.OrdinalIgnoreCase)))); - Console.WriteLine($"Succeeded count: {succeededCount}"); - Console.WriteLine($"Failed count: {failedCount}"); + var elapsed = DateTime.UtcNow - startTime; + Console.WriteLine($"[{elapsed.TotalMinutes:n0}m] Succeeded count: {succeededCount}"); + Console.WriteLine($"[{elapsed.TotalMinutes:n0}m] Failed count: {failedCount}"); } catch (Exception exc) { From c601a5776eed365680466ad7fca532e347d63e39 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 13 Jul 2023 12:46:41 -0700 Subject: [PATCH 19/48] update submodule --- src/ga4gh-tes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ga4gh-tes b/src/ga4gh-tes index cf53f4af..8693d86a 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit cf53f4af98e02919e486edf3ba2d6573cf5038c4 +Subproject commit 8693d86afee4e30a95751be33f0b2065edc98618 From f69fef1e76187c5aab6404a47461b973968b0a1a Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 13 Jul 2023 13:05:56 -0700 Subject: [PATCH 20/48] update trigger file for larger scatter --- src/TriggerService.Tests/IntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index c1f5eeba..9d43c1e0 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -100,7 +100,7 @@ public async Task RunScaleTestWithMutect2WaitTilDoneAsync() string workflowsContainerSasToken = lines[1].Trim('"'); const int countOfWorkflowsToRun = 1; - const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.trigger.json"; + const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/mattmcl4475/update-scatter-vm-size/mutect2.trigger.json"; // "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, storageAccountName, waitTilDone: true, workflowsContainerSasToken); From 743b56275717223fdb01ede8a79532a9895f36be Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 13 Jul 2023 13:06:45 -0700 Subject: [PATCH 21/48] minor --- src/TriggerService.Tests/IntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 9d43c1e0..b2cda718 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -258,7 +258,7 @@ private static async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWor failedCount = existingBlobNames .Count(name => originalBlobNames.Any(b => b - .Equals(name.Replace("failed/", "new/"), StringComparison.OrdinalIgnoreCase)))); + .Equals(name.Replace("failed/", "new/"), StringComparison.OrdinalIgnoreCase))); var elapsed = DateTime.UtcNow - startTime; Console.WriteLine($"[{elapsed.TotalMinutes:n0}m] Succeeded count: {succeededCount}"); From 21c37b3fc14d0fe6fc5d13988de77e6b199dbb0b Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 13 Jul 2023 15:43:11 -0700 Subject: [PATCH 22/48] filter by state --- src/TriggerService.Tests/IntegrationTests.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index b2cda718..6d0aac6c 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -253,12 +253,14 @@ private static async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWor } succeededCount = existingBlobNames + .Where(b => b.StartsWith("succeeded/")) .Count(existingBlobName => originalBlobNames.Any(b => b .Equals(existingBlobName.Replace("succeeded/", "new/"), StringComparison.OrdinalIgnoreCase))); failedCount = existingBlobNames - .Count(name => originalBlobNames.Any(b => b - .Equals(name.Replace("failed/", "new/"), StringComparison.OrdinalIgnoreCase))); + .Where(b => b.StartsWith("failed/")) + .Count(existingBlobName => originalBlobNames.Any(b => b + .Equals(existingBlobName.Replace("failed/", "new/"), StringComparison.OrdinalIgnoreCase))); var elapsed = DateTime.UtcNow - startTime; Console.WriteLine($"[{elapsed.TotalMinutes:n0}m] Succeeded count: {succeededCount}"); From dd4703a9630c4b900b1d40b34602d3c76535c8f8 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 13 Jul 2023 16:17:17 -0700 Subject: [PATCH 23/48] add new wdl to set disk size --- src/TriggerService.Tests/IntegrationTests.cs | 14 ++++---- .../test-wdls/mutect2/mutect2.inputs.json | 32 +++++++++++++++++++ .../test-wdls/mutect2/mutect2.trigger.json | 6 ++++ 3 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 src/TriggerService.Tests/test-wdls/mutect2/mutect2.inputs.json create mode 100644 src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 6d0aac6c..7a832a37 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -100,7 +100,7 @@ public async Task RunScaleTestWithMutect2WaitTilDoneAsync() string workflowsContainerSasToken = lines[1].Trim('"'); const int countOfWorkflowsToRun = 1; - const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/mattmcl4475/update-scatter-vm-size/mutect2.trigger.json"; // "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.trigger.json"; + const string triggerFile = "https://raw.githubusercontent.com/microsoft/CromwellOnAzure/mattmcl4475/long-running-int-test/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, storageAccountName, waitTilDone: true, workflowsContainerSasToken); @@ -187,7 +187,7 @@ private static async Task StartWorkflowsAsync( string workflowsContainerSasToken = null) { var startTime = DateTime.UtcNow; - const string containerName = "workflows"; + const string workflowsContainerName = "workflows"; BlobServiceClient blobServiceClient; @@ -201,13 +201,13 @@ private static async Task StartWorkflowsAsync( blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/")); } - var container = blobServiceClient.GetBlobContainerClient(containerName); + var workflowsContainer = blobServiceClient.GetBlobContainerClient(workflowsContainerName); if (!string.IsNullOrEmpty(workflowsContainerSasToken)) { Console.WriteLine("Using the specified container SAS token."); - var containerSasUri = new Uri($"https://{storageAccountName}.blob.core.windows.net/{containerName}?{workflowsContainerSasToken}"); - container = new BlobContainerClient(containerSasUri); + var containerSasUri = new Uri($"https://{storageAccountName}.blob.core.windows.net/{workflowsContainerName}?{workflowsContainerSasToken}"); + workflowsContainer = new BlobContainerClient(containerSasUri); } // 1. Get the publically available trigger file @@ -223,13 +223,13 @@ private static async Task StartWorkflowsAsync( // example: new/mutect2-001-of-100-2023-4-7-3-9.json var blobName = $"new/{workflowFriendlyName}-{i:D4}-of-{countOfWorkflowsToRun:D4}-{date}.json"; blobNames.Add(blobName); - await container.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(triggerFileJson), true); + await workflowsContainer.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(triggerFileJson), true); } // 3. Loop forever until they are all in a terminal state (succeeded or failed) if (waitTilDone) { - await WaitTilAllWorkflowsInTerminalStateAsync(countOfWorkflowsToRun, startTime, container, blobNames); + await WaitTilAllWorkflowsInTerminalStateAsync(countOfWorkflowsToRun, startTime, workflowsContainer, blobNames); } } diff --git a/src/TriggerService.Tests/test-wdls/mutect2/mutect2.inputs.json b/src/TriggerService.Tests/test-wdls/mutect2/mutect2.inputs.json new file mode 100644 index 00000000..4abb8926 --- /dev/null +++ b/src/TriggerService.Tests/test-wdls/mutect2/mutect2.inputs.json @@ -0,0 +1,32 @@ +{ + "Mutect2.gatk_docker": "broadinstitute/gatk:4.1.4.1", + + "Mutect2.intervals": "/datasettestinputs/dataset/references/hg19/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.baits.interval_list", + "Mutect2.scatter_count": 50, + "Mutect2.m2_extra_args": "--downsampling-stride 20 --max-reads-per-alignment-start 6 --max-suspicious-reads-per-alignment-start 6", + "Mutect2.filter_funcotations": "True", + "Mutect2.funco_reference_version": "hg19", + "Mutect2.funco_data_sources_tar_gz": "/datasettestinputs/dataset/gatk4-somatic-snvs-indels/funcotator/funcotator_dataSources.v1.6.20190124s.tar.gz", + "Mutect2.funco_transcript_selection_list": "/datasettestinputs/dataset/gatk4-somatic-snvs-indels/funcotator/transcriptList.exact_uniprot_matches.AKT1_CRLF2_FGFR1.txt", + "Mutect2.max_retries": 3, + + "Mutect2.ref_fasta": "/datasettestinputs/dataset/references/hg19/Homo_sapiens_assembly19.fasta", + "Mutect2.ref_dict": "/datasettestinputs/dataset/references/hg19/Homo_sapiens_assembly19.dict", + "Mutect2.ref_fai": "/datasettestinputs/dataset/references/hg19/Homo_sapiens_assembly19.fasta.fai", + "Mutect2.normal_reads": "/datasettestinputs/dataset/gatk4-somatic-snvs-indels/HCC1143/HCC1143_normal.bam", + "Mutect2.normal_reads_index": "/datasettestinputs/dataset/gatk4-somatic-snvs-indels/HCC1143/HCC1143_normal.bai", + "Mutect2.tumor_reads": "/datasettestinputs/dataset/gatk4-somatic-snvs-indels/HCC1143/HCC1143.bam", + "Mutect2.tumor_reads_index": "/datasettestinputs/dataset/gatk4-somatic-snvs-indels/HCC1143/HCC1143.bai", + + "Mutect2.pon": "/datasettestinputs/dataset/gatk4-somatic-snvs-indels/Mutect2-exome-panel.vcf", + "Mutect2.pon_idx": "/datasettestinputs/dataset/gatk4-somatic-snvs-indels/Mutect2-exome-panel.vcf.idx", + "Mutect2.gnomad": "/datasettestinputs/dataset/gatk4-somatic-snvs-indels/af-only-gnomad.raw.sites.vcf", + "Mutect2.gnomad_idx": "/datasettestinputs/dataset/gatk4-somatic-snvs-indels/af-only-gnomad.raw.sites.vcf.idx", + "Mutect2.variants_for_contamination": "/datasettestinputs/dataset/gatk4-somatic-snvs-indels/small_exac_common_3.vcf", + "Mutect2.variants_for_contamination_idx": "/datasettestinputs/dataset/gatk4-somatic-snvs-indels/small_exac_common_3.vcf.idx", + "Mutect2.realignment_index_bundle": "/datasettestinputs/dataset/gatk4-somatic-snvs-indels/Homo_sapiens_assembly38.index_bundle", + + "Mutect2.emergency_extra_disk": 20 +} + + diff --git a/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json b/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json new file mode 100644 index 00000000..8e32b7d6 --- /dev/null +++ b/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json @@ -0,0 +1,6 @@ +{ + "WorkflowUrl": "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/az2.7.0/mutect2.wdl", + "WorkflowInputsUrl": "https://raw.githubusercontent.com/microsoft/CromwellOnAzure/mattmcl4475/long-running-int-test/src/TriggerService.Tests/test-wdls/mutect2/mutect2.inputs.json", + "WorkflowOptionsUrl": null, + "WorkflowDependenciesUrl": null +} \ No newline at end of file From 98c7bbc0583290b103d908fc3d124aee5d61a65e Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Thu, 13 Jul 2023 21:18:41 -0700 Subject: [PATCH 24/48] update gatk to latest --- src/TriggerService.Tests/test-wdls/mutect2/mutect2.inputs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerService.Tests/test-wdls/mutect2/mutect2.inputs.json b/src/TriggerService.Tests/test-wdls/mutect2/mutect2.inputs.json index 4abb8926..6d8a54dc 100644 --- a/src/TriggerService.Tests/test-wdls/mutect2/mutect2.inputs.json +++ b/src/TriggerService.Tests/test-wdls/mutect2/mutect2.inputs.json @@ -1,5 +1,5 @@ { - "Mutect2.gatk_docker": "broadinstitute/gatk:4.1.4.1", + "Mutect2.gatk_docker": "broadinstitute/gatk:latest", "Mutect2.intervals": "/datasettestinputs/dataset/references/hg19/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.baits.interval_list", "Mutect2.scatter_count": 50, From 33d0fc8d75e7f7d8d19293d874d1aa545f48e195 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 11:01:58 -0700 Subject: [PATCH 25/48] update submodule and minor formatting --- src/TriggerService.Tests/IntegrationTests.cs | 2 +- src/ga4gh-tes | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 7a832a37..329cf127 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -75,7 +75,7 @@ public async Task RunScaleTestWithMutect2Async() } /// - /// To run this test, specify a testStorageAccountName, a workflowsContainerSasToken, and remove the [Ignore] attribute + /// To run this test, specify a testStorageAccountName, a workflowsContainerSasToken /// /// [TestCategory("Integration")] diff --git a/src/ga4gh-tes b/src/ga4gh-tes index 8693d86a..a28d00e1 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit 8693d86afee4e30a95751be33f0b2065edc98618 +Subproject commit a28d00e13c752e5aadc4f234c87736f2acc01b14 From 3aa0d8bfd342aec17290e8c4085716e11f078315 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 11:06:49 -0700 Subject: [PATCH 26/48] add debug --- src/TriggerService.Tests/IntegrationTests.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 329cf127..d7142ef8 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -252,6 +252,17 @@ private static async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWor existingBlobNames.Add(blobName); } + // TODO remove - debug + foreach (var original in originalBlobNames) + { + Console.WriteLine($"Original: {original}"); + } + + foreach (var existing in existingBlobNames) + { + Console.WriteLine($"Existing: {existing}"); + } + succeededCount = existingBlobNames .Where(b => b.StartsWith("succeeded/")) .Count(existingBlobName => originalBlobNames.Any(b => b From 3d4f8a28a1e37266aa8277c6d7281cb339f69f9b Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 18 Jul 2023 17:06:59 -0700 Subject: [PATCH 27/48] add timeout and assertion --- src/TriggerService.Tests/IntegrationTests.cs | 36 +++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index d7142ef8..3aef00ef 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Net.Http; @@ -178,7 +179,7 @@ public async Task DeleteOldBatchPoolsAsync() Console.WriteLine($"Deleted {count} pools."); } - private static async Task StartWorkflowsAsync( + private async Task StartWorkflowsAsync( int countOfWorkflowsToRun, string triggerFile, string workflowFriendlyName, @@ -233,24 +234,33 @@ private static async Task StartWorkflowsAsync( } } - private static async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsToRun, DateTime startTime, BlobContainerClient container, List originalBlobNames) + private async Task> GetBlobsAsync(BlobContainerClient container) + { + var enumerator = container.GetBlobsAsync().GetAsyncEnumerator(); + var existingBlobNames = new List(); + + while (await enumerator.MoveNextAsync()) + { + // example: inprogress/mutect2-001-of-100-2023-4-7-3-9.0fb0858a-3166-4a22-85b6-4337df2f53c5.json + var blobName = enumerator.Current.Name; + existingBlobNames.Add(blobName); + } + + return existingBlobNames; + } + + private async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsToRun, DateTime startTime, BlobContainerClient container, List originalBlobNames) { int succeededCount = 0; int failedCount = 0; - while (succeededCount + failedCount < countOfWorkflowsToRun) + var sw = Stopwatch.StartNew(); + + while (succeededCount + failedCount < countOfWorkflowsToRun && sw.Elapsed.TotalHours < 5) { try { - var enumerator = container.GetBlobsAsync().GetAsyncEnumerator(); - var existingBlobNames = new List(); - - while (await enumerator.MoveNextAsync()) - { - // example: inprogress/mutect2-001-of-100-2023-4-7-3-9.0fb0858a-3166-4a22-85b6-4337df2f53c5.json - var blobName = enumerator.Current.Name; - existingBlobNames.Add(blobName); - } + var existingBlobNames = await GetBlobsAsync(container); // TODO remove - debug foreach (var original in originalBlobNames) @@ -294,7 +304,7 @@ private static async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWor Console.WriteLine($"Succeeded count: {succeededCount}"); Console.WriteLine($"Failed count: {failedCount}"); - Assert.IsTrue(failedCount == 0); + Assert.IsTrue(succeededCount + failedCount >= countOfWorkflowsToRun); } } } From e5e6f4c0f14a0e62a2dc2be998b41f94add28663 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 10:36:01 -0700 Subject: [PATCH 28/48] fix status check --- src/TriggerService.Tests/IntegrationTests.cs | 64 +++++++++++--------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 3aef00ef..0434b23f 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -100,7 +100,13 @@ public async Task RunScaleTestWithMutect2WaitTilDoneAsync() string storageAccountName = lines[0].Trim(); string workflowsContainerSasToken = lines[1].Trim('"'); - const int countOfWorkflowsToRun = 1; + int countOfWorkflowsToRun = 1; + + if (lines.Length > 2) + { + int.TryParse(lines[2].Trim('"'), out countOfWorkflowsToRun); + } + const string triggerFile = "https://raw.githubusercontent.com/microsoft/CromwellOnAzure/mattmcl4475/long-running-int-test/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; @@ -219,6 +225,8 @@ private async Task StartWorkflowsAsync( var blobNames = new List(); var date = $"{startTime.Year}-{startTime.Month}-{startTime.Day}-{startTime.Hour}-{startTime.Minute}"; + Console.WriteLine($"Starting {countOfWorkflowsToRun} workflows...}"); + for (var i = 1; i <= countOfWorkflowsToRun; i++) { // example: new/mutect2-001-of-100-2023-4-7-3-9.json @@ -249,6 +257,29 @@ private async Task> GetBlobsAsync(BlobContainerClient container) return existingBlobNames; } + public int CountWorkflowsByState(List originalBlobNames, List currentBlobNames, WorkflowState state) + { + var stateString = state.ToString().ToLowerInvariant(); + + return originalBlobNames + .Select(originalBlob => originalBlob + .Replace("new/", $"{stateString}/", StringComparison.OrdinalIgnoreCase) + .Replace(".json", "", StringComparison.OrdinalIgnoreCase)) + .Count(originalBlobModified => currentBlobNames + .Any(currentBlob => currentBlob.StartsWith(originalBlobModified, StringComparison.OrdinalIgnoreCase))); + } + + [TestMethod] + public void CountCompletedWorkflowsTest() + { + var originalBlobNames = new List { "new/mutect2-0001-of-0001-2023-7-19-2-49.json", "new/mutect2-0001-of-0001-2023-7-19-2-50.json" }; + var currentBlobNames = new List { "failed/mutect2-0001-of-0001-2023-7-19-2-49.817f052c-81f8-45ff-863b-03f9655eee5c.json", + "succeeded/mutect2-0001-of-0001-2023-7-19-2-49.817f052c-81f8-45ff-863b-03f9655eee5c.json"}; + + Assert.IsTrue(CountWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Failed) == 1); + Assert.IsTrue(CountWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Succeeded) == 1); + } + private async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsToRun, DateTime startTime, BlobContainerClient container, List originalBlobNames) { int succeededCount = 0; @@ -256,36 +287,15 @@ private async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsT var sw = Stopwatch.StartNew(); - while (succeededCount + failedCount < countOfWorkflowsToRun && sw.Elapsed.TotalHours < 5) + while (succeededCount + failedCount < countOfWorkflowsToRun) // && sw.Elapsed.TotalHours < 5) { try { var existingBlobNames = await GetBlobsAsync(container); - - // TODO remove - debug - foreach (var original in originalBlobNames) - { - Console.WriteLine($"Original: {original}"); - } - - foreach (var existing in existingBlobNames) - { - Console.WriteLine($"Existing: {existing}"); - } - - succeededCount = existingBlobNames - .Where(b => b.StartsWith("succeeded/")) - .Count(existingBlobName => originalBlobNames.Any(b => b - .Equals(existingBlobName.Replace("succeeded/", "new/"), StringComparison.OrdinalIgnoreCase))); - - failedCount = existingBlobNames - .Where(b => b.StartsWith("failed/")) - .Count(existingBlobName => originalBlobNames.Any(b => b - .Equals(existingBlobName.Replace("failed/", "new/"), StringComparison.OrdinalIgnoreCase))); - - var elapsed = DateTime.UtcNow - startTime; - Console.WriteLine($"[{elapsed.TotalMinutes:n0}m] Succeeded count: {succeededCount}"); - Console.WriteLine($"[{elapsed.TotalMinutes:n0}m] Failed count: {failedCount}"); + succeededCount = CountWorkflowsByState(originalBlobNames, existingBlobNames, WorkflowState.Succeeded); + failedCount = CountWorkflowsByState(originalBlobNames, existingBlobNames, WorkflowState.Failed); + Console.WriteLine($"[{sw.Elapsed.TotalMinutes:n0}m] Succeeded count: {succeededCount}"); + Console.WriteLine($"[{sw.Elapsed.TotalMinutes:n0}m] Failed count: {failedCount}"); } catch (Exception exc) { From 702ca60543e563a6dc7108cf05068675f04a3ef9 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 10:41:00 -0700 Subject: [PATCH 29/48] fail the test if any failed workflows --- src/TriggerService.Tests/IntegrationTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 0434b23f..3f531dba 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -315,6 +315,7 @@ private async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsT Console.WriteLine($"Failed count: {failedCount}"); Assert.IsTrue(succeededCount + failedCount >= countOfWorkflowsToRun); + Assert.IsTrue(failedCount == 0); } } } From 2b03bc65c7d73a2869aac6affa9a76a748afaea1 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 10:41:39 -0700 Subject: [PATCH 30/48] change logic to exact equality --- src/TriggerService.Tests/IntegrationTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 3f531dba..acffb5ba 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -302,7 +302,7 @@ private async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsT Console.WriteLine(exc); } - if (succeededCount + failedCount >= countOfWorkflowsToRun) + if (succeededCount + failedCount == countOfWorkflowsToRun) { break; } @@ -314,7 +314,7 @@ private async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsT Console.WriteLine($"Succeeded count: {succeededCount}"); Console.WriteLine($"Failed count: {failedCount}"); - Assert.IsTrue(succeededCount + failedCount >= countOfWorkflowsToRun); + Assert.IsTrue(succeededCount + failedCount == countOfWorkflowsToRun); Assert.IsTrue(failedCount == 0); } } From 12ddf4ea6df034f29f33365f683de114f0b7204e Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 10:51:08 -0700 Subject: [PATCH 31/48] fix bug --- src/TriggerService.Tests/IntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index acffb5ba..0c5ca9cf 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -225,7 +225,7 @@ private async Task StartWorkflowsAsync( var blobNames = new List(); var date = $"{startTime.Year}-{startTime.Month}-{startTime.Day}-{startTime.Hour}-{startTime.Minute}"; - Console.WriteLine($"Starting {countOfWorkflowsToRun} workflows...}"); + Console.WriteLine($"Starting {countOfWorkflowsToRun} workflows..."); for (var i = 1; i <= countOfWorkflowsToRun; i++) { From 13706ceadc6b5b4ffca150b17e481631258b394c Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 10:53:02 -0700 Subject: [PATCH 32/48] minor formatting --- src/TriggerService.Tests/IntegrationTests.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 0c5ca9cf..a1d8fb0c 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -257,7 +257,7 @@ private async Task> GetBlobsAsync(BlobContainerClient container) return existingBlobNames; } - public int CountWorkflowsByState(List originalBlobNames, List currentBlobNames, WorkflowState state) + private int CountWorkflowsByState(List originalBlobNames, List currentBlobNames, WorkflowState state) { var stateString = state.ToString().ToLowerInvariant(); @@ -272,9 +272,15 @@ public int CountWorkflowsByState(List originalBlobNames, List cu [TestMethod] public void CountCompletedWorkflowsTest() { - var originalBlobNames = new List { "new/mutect2-0001-of-0001-2023-7-19-2-49.json", "new/mutect2-0001-of-0001-2023-7-19-2-50.json" }; - var currentBlobNames = new List { "failed/mutect2-0001-of-0001-2023-7-19-2-49.817f052c-81f8-45ff-863b-03f9655eee5c.json", - "succeeded/mutect2-0001-of-0001-2023-7-19-2-49.817f052c-81f8-45ff-863b-03f9655eee5c.json"}; + var originalBlobNames = new List { + "new/mutect2-0001-of-0001-2023-7-19-2-49.json", + "new/mutect2-0001-of-0001-2023-7-19-2-50.json" + }; + + var currentBlobNames = new List { + "failed/mutect2-0001-of-0001-2023-7-19-2-49.817f052c-81f8-45ff-863b-03f9655eee5c.json", + "succeeded/mutect2-0001-of-0001-2023-7-19-2-49.817f052c-81f8-45ff-863b-03f9655eee5c.json" + }; Assert.IsTrue(CountWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Failed) == 1); Assert.IsTrue(CountWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Succeeded) == 1); From 650a518a94149a2ca2bb90fc6f3e70324f427103 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 10:54:18 -0700 Subject: [PATCH 33/48] refactor --- src/TriggerService.Tests/IntegrationTests.cs | 111 +++++++++---------- 1 file changed, 55 insertions(+), 56 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index a1d8fb0c..ed67da13 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -22,6 +22,44 @@ public class IntegrationTests { private const string testStorageAccountName = ""; + /// + /// To run this test, specify a testStorageAccountName, a workflowsContainerSasToken + /// + /// + [TestCategory("Integration")] + [TestMethod] + public async Task RunScaleTestWithMutect2WaitTilDoneAsync() + { + // This is set in the Azure Devops pipeline, which writes the file to the .csproj directory + // The current working directory is this: /mnt/vss/_work/r1/a/CoaArtifacts/AllSource/TriggerService.Tests/bin/Debug/net7.0/ + // And the file is available here: /mnt/vss/_work/r1/a/CoaArtifacts/AllSource/TriggerService.Tests/temp_storage_account_name.txt + const string storageAccountNamePath = "../../../temp_storage_account_name.txt"; + var path = storageAccountNamePath; + + if (!File.Exists(path)) + { + Console.WriteLine($"Path not found - exiting integration test: {path}"); + return; + } + + Console.WriteLine($"Found path: {path}"); + var lines = await File.ReadAllLinesAsync(path); + string storageAccountName = lines[0].Trim(); + string workflowsContainerSasToken = lines[1].Trim('"'); + + int countOfWorkflowsToRun = 1; + + if (lines.Length > 2) + { + int.TryParse(lines[2].Trim('"'), out countOfWorkflowsToRun); + } + + const string triggerFile = "https://raw.githubusercontent.com/microsoft/CromwellOnAzure/mattmcl4475/long-running-int-test/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json"; + const string workflowFriendlyName = $"mutect2"; + + await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, storageAccountName, waitTilDone: true, workflowsContainerSasToken); + } + /// /// To run this test, specify a testStorageAccountName, a workflowsContainerSasToken, and remove the [Ignore] attribute /// @@ -58,7 +96,6 @@ public async Task RunGlobTestWdlAsync() await container.GetBlobClient(triggerFileBlobName).UploadAsync(BinaryData.FromString(triggerJson), true); } - /// /// To run this test, specify a testStorageAccountName, a workflowsContainerSasToken, and remove the [Ignore] attribute /// @@ -75,44 +112,6 @@ public async Task RunScaleTestWithMutect2Async() await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, testStorageAccountName); } - /// - /// To run this test, specify a testStorageAccountName, a workflowsContainerSasToken - /// - /// - [TestCategory("Integration")] - [TestMethod] - public async Task RunScaleTestWithMutect2WaitTilDoneAsync() - { - // This is set in the Azure Devops pipeline, which writes the file to the .csproj directory - // The current working directory is this: /mnt/vss/_work/r1/a/CoaArtifacts/AllSource/TriggerService.Tests/bin/Debug/net7.0/ - // And the file is available here: /mnt/vss/_work/r1/a/CoaArtifacts/AllSource/TriggerService.Tests/temp_storage_account_name.txt - const string storageAccountNamePath = "../../../temp_storage_account_name.txt"; - var path = storageAccountNamePath; - - if (!File.Exists(path)) - { - Console.WriteLine($"Path not found - exiting integration test: {path}"); - return; - } - - Console.WriteLine($"Found path: {path}"); - var lines = await File.ReadAllLinesAsync(path); - string storageAccountName = lines[0].Trim(); - string workflowsContainerSasToken = lines[1].Trim('"'); - - int countOfWorkflowsToRun = 1; - - if (lines.Length > 2) - { - int.TryParse(lines[2].Trim('"'), out countOfWorkflowsToRun); - } - - const string triggerFile = "https://raw.githubusercontent.com/microsoft/CromwellOnAzure/mattmcl4475/long-running-int-test/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json"; - const string workflowFriendlyName = $"mutect2"; - - await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, storageAccountName, waitTilDone: true, workflowsContainerSasToken); - } - /// /// To run this test, specify a testStorageAccountName, a workflowsContainerSasToken, and remove the [Ignore] attribute /// @@ -185,6 +184,23 @@ public async Task DeleteOldBatchPoolsAsync() Console.WriteLine($"Deleted {count} pools."); } + [TestMethod] + public void CountCompletedWorkflowsTest() + { + var originalBlobNames = new List { + "new/mutect2-0001-of-0001-2023-7-19-2-49.json", + "new/mutect2-0001-of-0001-2023-7-19-2-50.json" + }; + + var currentBlobNames = new List { + "failed/mutect2-0001-of-0001-2023-7-19-2-49.817f052c-81f8-45ff-863b-03f9655eee5c.json", + "succeeded/mutect2-0001-of-0001-2023-7-19-2-49.817f052c-81f8-45ff-863b-03f9655eee5c.json" + }; + + Assert.IsTrue(CountWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Failed) == 1); + Assert.IsTrue(CountWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Succeeded) == 1); + } + private async Task StartWorkflowsAsync( int countOfWorkflowsToRun, string triggerFile, @@ -269,23 +285,6 @@ private int CountWorkflowsByState(List originalBlobNames, List c .Any(currentBlob => currentBlob.StartsWith(originalBlobModified, StringComparison.OrdinalIgnoreCase))); } - [TestMethod] - public void CountCompletedWorkflowsTest() - { - var originalBlobNames = new List { - "new/mutect2-0001-of-0001-2023-7-19-2-49.json", - "new/mutect2-0001-of-0001-2023-7-19-2-50.json" - }; - - var currentBlobNames = new List { - "failed/mutect2-0001-of-0001-2023-7-19-2-49.817f052c-81f8-45ff-863b-03f9655eee5c.json", - "succeeded/mutect2-0001-of-0001-2023-7-19-2-49.817f052c-81f8-45ff-863b-03f9655eee5c.json" - }; - - Assert.IsTrue(CountWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Failed) == 1); - Assert.IsTrue(CountWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Succeeded) == 1); - } - private async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsToRun, DateTime startTime, BlobContainerClient container, List originalBlobNames) { int succeededCount = 0; From 28d1bcc75a662e19bbdf842d62615caa114addb9 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 10:56:53 -0700 Subject: [PATCH 34/48] refactor --- src/TriggerService.Tests/IntegrationTests.cs | 55 +++++++++++--------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index ed67da13..bbe19527 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -30,34 +30,10 @@ public class IntegrationTests [TestMethod] public async Task RunScaleTestWithMutect2WaitTilDoneAsync() { - // This is set in the Azure Devops pipeline, which writes the file to the .csproj directory - // The current working directory is this: /mnt/vss/_work/r1/a/CoaArtifacts/AllSource/TriggerService.Tests/bin/Debug/net7.0/ - // And the file is available here: /mnt/vss/_work/r1/a/CoaArtifacts/AllSource/TriggerService.Tests/temp_storage_account_name.txt - const string storageAccountNamePath = "../../../temp_storage_account_name.txt"; - var path = storageAccountNamePath; - - if (!File.Exists(path)) - { - Console.WriteLine($"Path not found - exiting integration test: {path}"); - return; - } - - Console.WriteLine($"Found path: {path}"); - var lines = await File.ReadAllLinesAsync(path); - string storageAccountName = lines[0].Trim(); - string workflowsContainerSasToken = lines[1].Trim('"'); - - int countOfWorkflowsToRun = 1; - - if (lines.Length > 2) - { - int.TryParse(lines[2].Trim('"'), out countOfWorkflowsToRun); - } - const string triggerFile = "https://raw.githubusercontent.com/microsoft/CromwellOnAzure/mattmcl4475/long-running-int-test/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; - await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, storageAccountName, waitTilDone: true, workflowsContainerSasToken); + await RunIntegrationTestAsync(triggerFile, workflowFriendlyName); } /// @@ -201,6 +177,35 @@ public void CountCompletedWorkflowsTest() Assert.IsTrue(CountWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Succeeded) == 1); } + private async Task RunIntegrationTestAsync(string triggerFile, string workflowFriendlyName) + { + // This is set in the Azure Devops pipeline, which writes the file to the .csproj directory + // The current working directory is this: /mnt/vss/_work/r1/a/CoaArtifacts/AllSource/TriggerService.Tests/bin/Debug/net7.0/ + // And the file is available here: /mnt/vss/_work/r1/a/CoaArtifacts/AllSource/TriggerService.Tests/temp_storage_account_name.txt + const string storageAccountNamePath = "../../../temp_storage_account_name.txt"; + var path = storageAccountNamePath; + + if (!File.Exists(path)) + { + Console.WriteLine($"Path not found - exiting integration test: {path}"); + return; + } + + Console.WriteLine($"Found path: {path}"); + var lines = await File.ReadAllLinesAsync(path); + string storageAccountName = lines[0].Trim(); + string workflowsContainerSasToken = lines[1].Trim('"'); + + int countOfWorkflowsToRun = 1; + + if (lines.Length > 2) + { + int.TryParse(lines[2].Trim('"'), out countOfWorkflowsToRun); + } + + await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, storageAccountName, waitTilDone: true, workflowsContainerSasToken); + } + private async Task StartWorkflowsAsync( int countOfWorkflowsToRun, string triggerFile, From 23481fb7a3f10f6dbf36996692a557bb1b3b10cb Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 11:06:09 -0700 Subject: [PATCH 35/48] minor rename --- src/TriggerService.Tests/IntegrationTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index bbe19527..623d4bf8 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -263,7 +263,7 @@ private async Task StartWorkflowsAsync( } } - private async Task> GetBlobsAsync(BlobContainerClient container) + private async Task> ListContainerBlobNamesAsync(BlobContainerClient container) { var enumerator = container.GetBlobsAsync().GetAsyncEnumerator(); var existingBlobNames = new List(); @@ -301,7 +301,7 @@ private async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsT { try { - var existingBlobNames = await GetBlobsAsync(container); + var existingBlobNames = await ListContainerBlobNamesAsync(container); succeededCount = CountWorkflowsByState(originalBlobNames, existingBlobNames, WorkflowState.Succeeded); failedCount = CountWorkflowsByState(originalBlobNames, existingBlobNames, WorkflowState.Failed); Console.WriteLine($"[{sw.Elapsed.TotalMinutes:n0}m] Succeeded count: {succeededCount}"); From 1efeeb41151b547561b86c41d7144f3e94f9edbd Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 11:12:12 -0700 Subject: [PATCH 36/48] remove whitespace --- src/TriggerService.Tests/IntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 623d4bf8..d51667f7 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -225,7 +225,7 @@ private async Task StartWorkflowsAsync( blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/"), new AzureCliCredential()); } else - { + { blobServiceClient = new BlobServiceClient(new Uri($"https://{storageAccountName}.blob.core.windows.net/")); } From f6ab50f87b56a4029af78b61fac8d5ef7827df8c Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 14:23:15 -0700 Subject: [PATCH 37/48] add failure info --- src/TriggerService.Tests/IntegrationTests.cs | 63 ++++++++++++++++---- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index d51667f7..4e2581d7 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -170,7 +170,7 @@ public void CountCompletedWorkflowsTest() var currentBlobNames = new List { "failed/mutect2-0001-of-0001-2023-7-19-2-49.817f052c-81f8-45ff-863b-03f9655eee5c.json", - "succeeded/mutect2-0001-of-0001-2023-7-19-2-49.817f052c-81f8-45ff-863b-03f9655eee5c.json" + "succeeded/mutect2-0001-of-0001-2023-7-19-2-50.817f052c-81f8-45ff-863b-03f9655eee5c.json" }; Assert.IsTrue(CountWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Failed) == 1); @@ -279,17 +279,44 @@ private async Task> ListContainerBlobNamesAsync(BlobContainerClient } private int CountWorkflowsByState(List originalBlobNames, List currentBlobNames, WorkflowState state) + { + return GetWorkflowsByState(originalBlobNames, currentBlobNames, state).Count(); + } + + private List GetWorkflowsByState(List originalBlobNames, List currentBlobNames, WorkflowState state) { var stateString = state.ToString().ToLowerInvariant(); - return originalBlobNames - .Select(originalBlob => originalBlob - .Replace("new/", $"{stateString}/", StringComparison.OrdinalIgnoreCase) - .Replace(".json", "", StringComparison.OrdinalIgnoreCase)) - .Count(originalBlobModified => currentBlobNames - .Any(currentBlob => currentBlob.StartsWith(originalBlobModified, StringComparison.OrdinalIgnoreCase))); + return currentBlobNames + .Where(currentBlob => originalBlobNames + .Any(originalBlob => currentBlob.StartsWith( + originalBlob + .Replace("new/", $"{stateString}/", StringComparison.OrdinalIgnoreCase) + .Replace(".json", "", StringComparison.OrdinalIgnoreCase), StringComparison.OrdinalIgnoreCase))) + .ToList(); + } + + [TestMethod] + public void GetWorkflowsByStateTest() + { + var originalBlobNames = new List { + "new/mutect2-0001-of-0001-2023-7-19-2-49.json", + "new/mutect2-0001-of-0001-2023-7-19-2-50.json" + }; + + var currentBlobNames = new List { + "failed/mutect2-0001-of-0001-2023-7-19-2-49.817f052c-81f8-45ff-863b-03f9655eee5c.json", + "succeeded/mutect2-0001-of-0001-2023-7-19-2-50.817f052c-81f8-45ff-863b-03f9655eee5c.json" + }; + + var failed = GetWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Failed); + var succeeded = GetWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Succeeded); + Assert.IsTrue(failed.Single() == currentBlobNames.First()); + Assert.IsTrue(succeeded.Single() == currentBlobNames.Skip(1).First()); } + + private async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsToRun, DateTime startTime, BlobContainerClient container, List originalBlobNames) { int succeededCount = 0; @@ -304,8 +331,7 @@ private async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsT var existingBlobNames = await ListContainerBlobNamesAsync(container); succeededCount = CountWorkflowsByState(originalBlobNames, existingBlobNames, WorkflowState.Succeeded); failedCount = CountWorkflowsByState(originalBlobNames, existingBlobNames, WorkflowState.Failed); - Console.WriteLine($"[{sw.Elapsed.TotalMinutes:n0}m] Succeeded count: {succeededCount}"); - Console.WriteLine($"[{sw.Elapsed.TotalMinutes:n0}m] Failed count: {failedCount}"); + Console.WriteLine($"[{sw.Elapsed.TotalMinutes:n0}m] Succeeded count: {succeededCount}, Failed count: {failedCount}"); } catch (Exception exc) { @@ -325,7 +351,24 @@ private async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsT Console.WriteLine($"Failed count: {failedCount}"); Assert.IsTrue(succeededCount + failedCount == countOfWorkflowsToRun); - Assert.IsTrue(failedCount == 0); + + if (failedCount > 0) + { + Console.WriteLine("Failed workflow details:"); + + var existingBlobNames = await ListContainerBlobNamesAsync(container); + var failedWorkflowBlobNames = GetWorkflowsByState(originalBlobNames, existingBlobNames, WorkflowState.Failed); + + foreach (var workflowBlobName in failedWorkflowBlobNames) + { + var content = (await container.GetBlobClient(workflowBlobName).DownloadContentAsync()).Value.Content.ToString(); + Console.WriteLine($"Failed workflow blob name: {workflowBlobName}"); + Console.WriteLine($"Failed workflow triggerfile content: {content}"); + Console.WriteLine(content); + } + } + + Assert.IsTrue(failedCount == 0, $"{failedCount} workflows FAILED."); } } } From c84f70bbe282afaeeb42f89f9e51f22ad134dee6 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 15:44:03 -0700 Subject: [PATCH 38/48] add failure info to console --- src/deploy-cromwell-on-azure/Deployer.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/deploy-cromwell-on-azure/Deployer.cs b/src/deploy-cromwell-on-azure/Deployer.cs index 6fd19087..6e25492b 100644 --- a/src/deploy-cromwell-on-azure/Deployer.cs +++ b/src/deploy-cromwell-on-azure/Deployer.cs @@ -2109,10 +2109,17 @@ private static async Task IsWorkflowSuccessfulAfterLongPollingAsync(IStora try { var succeeded = container.GetBlobs(prefix: $"succeeded/{id}").Count() == 1; - var failed = container.GetBlobs(prefix: $"failed/{id}").Count() == 1; + var failedWorkflows = container.GetBlobs(prefix: $"failed/{id}").ToList(); + var failed = failedWorkflows.Count == 1; if (succeeded || failed) { + if (failed) + { + var failedContent = (await container.GetBlobClient(failedWorkflows.First().Name).DownloadContentAsync()).Value.Content.ToString(); + ConsoleEx.WriteLine($"Failed workflow trigger JSON: {failedContent}"); + } + return succeeded && !failed; } } From 17a346a8d9aa9dd81ea339e854e027b462c20138 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 16:54:58 -0700 Subject: [PATCH 39/48] minor rename --- src/deploy-cromwell-on-azure/Deployer.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/deploy-cromwell-on-azure/Deployer.cs b/src/deploy-cromwell-on-azure/Deployer.cs index 6e25492b..a17a13ed 100644 --- a/src/deploy-cromwell-on-azure/Deployer.cs +++ b/src/deploy-cromwell-on-azure/Deployer.cs @@ -2108,19 +2108,19 @@ private static async Task IsWorkflowSuccessfulAfterLongPollingAsync(IStora { try { - var succeeded = container.GetBlobs(prefix: $"succeeded/{id}").Count() == 1; - var failedWorkflows = container.GetBlobs(prefix: $"failed/{id}").ToList(); - var failed = failedWorkflows.Count == 1; + var hasSucceeded = container.GetBlobs(prefix: $"succeeded/{id}").Count() == 1; + var failedWorkflowTriggerFileBlobs = container.GetBlobs(prefix: $"failed/{id}").ToList(); + var hasFailed = failedWorkflowTriggerFileBlobs.Count == 1; - if (succeeded || failed) + if (hasSucceeded || hasFailed) { - if (failed) + if (hasFailed) { - var failedContent = (await container.GetBlobClient(failedWorkflows.First().Name).DownloadContentAsync()).Value.Content.ToString(); + var failedContent = (await container.GetBlobClient(failedWorkflowTriggerFileBlobs.First().Name).DownloadContentAsync()).Value.Content.ToString(); ConsoleEx.WriteLine($"Failed workflow trigger JSON: {failedContent}"); } - return succeeded && !failed; + return hasSucceeded && !hasFailed; } } catch (Exception exc) From 4fc20975ed3cb9521c3faba5be88f7a2594f9b0b Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 17:26:08 -0700 Subject: [PATCH 40/48] add support for all common workflows --- src/TriggerService.Tests/IntegrationTests.cs | 64 ++++++++++++++------ 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 4e2581d7..dc426eb4 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -33,7 +33,29 @@ public async Task RunScaleTestWithMutect2WaitTilDoneAsync() const string triggerFile = "https://raw.githubusercontent.com/microsoft/CromwellOnAzure/mattmcl4475/long-running-int-test/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; - await RunIntegrationTestAsync(triggerFile, workflowFriendlyName); + await RunIntegrationTestAsync(new List<(string triggerFileBlobUrl, string workflowFriendlyName)> { (triggerFile, workflowFriendlyName) }); + } + + /// + /// To run this test, specify a testStorageAccountName, a workflowsContainerSasToken + /// + /// + [TestCategory("Integration")] + [TestMethod] + public async Task RunAllCommonWorkflowsWaitTilDoneAsync() + { + var workflowTriggerFiles = new List<(string triggerFileBlobUrl, string workflowFriendlyName)> { + ("https://raw.githubusercontent.com/microsoft/gatk4-data-processing-azure/main-azure/processing-for-variant-discovery-gatk4.b37.trigger.json", "preprocessing-b37"), + ("https://raw.githubusercontent.com/microsoft/gatk4-data-processing-azure/main-azure/processing-for-variant-discovery-gatk4.hg38.trigger.json", "preprocessing-hg38"), + ("https://raw.githubusercontent.com/microsoft/gatk4-genome-processing-pipeline-azure/main-azure/WholeGenomeGermlineSingleSample.trigger.json", "germline"), + ("https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.trigger.json", "mutect2"), + ("https://raw.githubusercontent.com/microsoft/gatk4-rnaseq-germline-snps-indels-azure/jsaun/gatk4-rna-germline-variant-calling.trigger.json", "rna-germline"), + ("https://raw.githubusercontent.com/microsoft/gatk4-cnn-variant-filter-azure/main-azure/cram2filtered.trigger.json", "cram-to-filtered"), + ("https://raw.githubusercontent.com/microsoft/seq-format-conversion-azure/main-azure/interleaved-fastq-to-paired-fastq.trigger.json", "fastq-to-paired"), + ("https://raw.githubusercontent.com/microsoft/seq-format-conversion-azure/main-azure/paired-fastq-to-unmapped-bam.trigger.json", "paired-fastq-to-unmapped-bam"), + ("https://raw.githubusercontent.com/microsoft/seq-format-conversion-azure/main-azure/cram-to-bam.trigger.json", "cram-to-bam") }; + + await RunIntegrationTestAsync(workflowTriggerFiles); } /// @@ -84,8 +106,8 @@ public async Task RunScaleTestWithMutect2Async() const int countOfWorkflowsToRun = 100; const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/main-azure/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; - - await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, testStorageAccountName); + var triggerFiles = new List<(string triggerFileBlobUrl, string workflowFriendlyName)> { (triggerFile, workflowFriendlyName) }; + await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFiles, testStorageAccountName); } /// @@ -101,7 +123,8 @@ public async Task RunScaleTestWithWholeGenomeGermlineSingleSampleAsync() const string triggerFile = "https://raw.githubusercontent.com/microsoft/gatk4-genome-processing-pipeline-azure/main-azure/WholeGenomeGermlineSingleSample.trigger.json"; const string workflowFriendlyName = $"wgs-germline"; - await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, testStorageAccountName); + var triggerFiles = new List<(string triggerFileBlobUrl, string workflowFriendlyName)> { (triggerFile, workflowFriendlyName) }; + await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFiles, testStorageAccountName); } /// @@ -177,7 +200,7 @@ public void CountCompletedWorkflowsTest() Assert.IsTrue(CountWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Succeeded) == 1); } - private async Task RunIntegrationTestAsync(string triggerFile, string workflowFriendlyName) + private async Task RunIntegrationTestAsync(List<(string triggerFileBlobUrl, string workflowFriendlyName)> triggerFiles) { // This is set in the Azure Devops pipeline, which writes the file to the .csproj directory // The current working directory is this: /mnt/vss/_work/r1/a/CoaArtifacts/AllSource/TriggerService.Tests/bin/Debug/net7.0/ @@ -203,13 +226,12 @@ private async Task RunIntegrationTestAsync(string triggerFile, string workflowFr int.TryParse(lines[2].Trim('"'), out countOfWorkflowsToRun); } - await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFile, workflowFriendlyName, storageAccountName, waitTilDone: true, workflowsContainerSasToken); + await StartWorkflowsAsync(countOfWorkflowsToRun, triggerFiles, storageAccountName, waitTilDone: true, workflowsContainerSasToken); } private async Task StartWorkflowsAsync( int countOfWorkflowsToRun, - string triggerFile, - string workflowFriendlyName, + List<(string triggerFileBlobUrl, string workflowFriendlyName)> triggerFiles, string storageAccountName, bool waitTilDone = false, string workflowsContainerSasToken = null) @@ -240,20 +262,24 @@ private async Task StartWorkflowsAsync( // 1. Get the publically available trigger file using var httpClient = new HttpClient(); - var triggerFileJson = await (await httpClient.GetAsync(triggerFile)).Content.ReadAsStringAsync(); - - // 2. Start the workflows by uploading new trigger files var blobNames = new List(); - var date = $"{startTime.Year}-{startTime.Month}-{startTime.Day}-{startTime.Hour}-{startTime.Minute}"; - - Console.WriteLine($"Starting {countOfWorkflowsToRun} workflows..."); - for (var i = 1; i <= countOfWorkflowsToRun; i++) + foreach (var triggerFile in triggerFiles) { - // example: new/mutect2-001-of-100-2023-4-7-3-9.json - var blobName = $"new/{workflowFriendlyName}-{i:D4}-of-{countOfWorkflowsToRun:D4}-{date}.json"; - blobNames.Add(blobName); - await workflowsContainer.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(triggerFileJson), true); + var triggerFileJson = await (await httpClient.GetAsync(triggerFile.triggerFileBlobUrl)).Content.ReadAsStringAsync(); + + // 2. Start the workflows by uploading new trigger files + + var date = $"{startTime.Year}-{startTime.Month}-{startTime.Day}-{startTime.Hour}-{startTime.Minute}"; + Console.WriteLine($"Starting {countOfWorkflowsToRun} workflows..."); + + for (var i = 1; i <= countOfWorkflowsToRun; i++) + { + // example: new/mutect2-001-of-100-2023-4-7-3-9.json + var blobName = $"new/{triggerFile.workflowFriendlyName}-{i:D4}-of-{countOfWorkflowsToRun:D4}-{date}.json"; + blobNames.Add(blobName); + await workflowsContainer.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(triggerFileJson), true); + } } // 3. Loop forever until they are all in a terminal state (succeeded or failed) From e18b8fbefb17b45f707ccf34deaea22567b4c01d Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 19 Jul 2023 17:47:44 -0700 Subject: [PATCH 41/48] minor fix --- src/TriggerService.Tests/IntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index dc426eb4..9e8038a0 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -389,7 +389,7 @@ private async Task WaitTilAllWorkflowsInTerminalStateAsync(int countOfWorkflowsT { var content = (await container.GetBlobClient(workflowBlobName).DownloadContentAsync()).Value.Content.ToString(); Console.WriteLine($"Failed workflow blob name: {workflowBlobName}"); - Console.WriteLine($"Failed workflow triggerfile content: {content}"); + Console.WriteLine("Failed workflow triggerfile content:"); Console.WriteLine(content); } } From e175c0264049fb9fa72c922f5c6d52c5d4150f02 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Fri, 21 Jul 2023 15:46:30 -0700 Subject: [PATCH 42/48] update tes submodule --- src/ga4gh-tes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ga4gh-tes b/src/ga4gh-tes index a28d00e1..5bf26417 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit a28d00e13c752e5aadc4f234c87736f2acc01b14 +Subproject commit 5bf264177b61cf6f3c6a14d017ca4d95117dcbb9 From a06eec503f47274aaf481259218f1a608e274c64 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Fri, 21 Jul 2023 21:53:05 -0700 Subject: [PATCH 43/48] update submodule --- src/ga4gh-tes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ga4gh-tes b/src/ga4gh-tes index 5bf26417..58c1335b 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit 5bf264177b61cf6f3c6a14d017ca4d95117dcbb9 +Subproject commit 58c1335b1b3d1a3a5c63064b219966c88b61d80c From 78cdb2149a8232e5e44f6fce10eab29b84bf9a92 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Tue, 25 Jul 2023 22:26:56 -0700 Subject: [PATCH 44/48] update subm --- src/ga4gh-tes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ga4gh-tes b/src/ga4gh-tes index 58c1335b..077d76bd 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit 58c1335b1b3d1a3a5c63064b219966c88b61d80c +Subproject commit 077d76bd16acf93619ed2c1c18519e77fc6c590c From 02f9064904ebc34f210699a970e4ac9625189f7f Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 26 Jul 2023 07:11:13 -0700 Subject: [PATCH 45/48] update submodule --- src/ga4gh-tes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ga4gh-tes b/src/ga4gh-tes index 077d76bd..03ce1c52 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit 077d76bd16acf93619ed2c1c18519e77fc6c590c +Subproject commit 03ce1c522b53f826e3a0e5da7c5b800c9abe0a18 From 465244a0c1dec5f2d769d03224b80e6f231f986b Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 26 Jul 2023 08:54:03 -0700 Subject: [PATCH 46/48] update submodule --- src/ga4gh-tes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ga4gh-tes b/src/ga4gh-tes index 03ce1c52..5cf89b89 160000 --- a/src/ga4gh-tes +++ b/src/ga4gh-tes @@ -1 +1 @@ -Subproject commit 03ce1c522b53f826e3a0e5da7c5b800c9abe0a18 +Subproject commit 5cf89b896264b8496fb4eabd15905d20d0c97399 From 84de5d96d81dfd57baabe570f41e719ed09b904b Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 26 Jul 2023 08:54:36 -0700 Subject: [PATCH 47/48] update url --- src/TriggerService.Tests/IntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerService.Tests/IntegrationTests.cs b/src/TriggerService.Tests/IntegrationTests.cs index 9e8038a0..7e67be51 100644 --- a/src/TriggerService.Tests/IntegrationTests.cs +++ b/src/TriggerService.Tests/IntegrationTests.cs @@ -30,7 +30,7 @@ public class IntegrationTests [TestMethod] public async Task RunScaleTestWithMutect2WaitTilDoneAsync() { - const string triggerFile = "https://raw.githubusercontent.com/microsoft/CromwellOnAzure/mattmcl4475/long-running-int-test/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json"; + const string triggerFile = "https://raw.githubusercontent.com/microsoft/CromwellOnAzure/main/long-running-int-test/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json"; const string workflowFriendlyName = $"mutect2"; await RunIntegrationTestAsync(new List<(string triggerFileBlobUrl, string workflowFriendlyName)> { (triggerFile, workflowFriendlyName) }); From 7ee7eedd4b9be1715cb975b63b2245f8685f5967 Mon Sep 17 00:00:00 2001 From: Matt McLoughlin Date: Wed, 26 Jul 2023 08:55:43 -0700 Subject: [PATCH 48/48] update path --- src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json b/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json index 8e32b7d6..0009a016 100644 --- a/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json +++ b/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json @@ -1,6 +1,6 @@ { "WorkflowUrl": "https://raw.githubusercontent.com/microsoft/gatk4-somatic-snvs-indels-azure/az2.7.0/mutect2.wdl", - "WorkflowInputsUrl": "https://raw.githubusercontent.com/microsoft/CromwellOnAzure/mattmcl4475/long-running-int-test/src/TriggerService.Tests/test-wdls/mutect2/mutect2.inputs.json", + "WorkflowInputsUrl": "https://raw.githubusercontent.com/microsoft/CromwellOnAzure/main/long-running-int-test/src/TriggerService.Tests/test-wdls/mutect2/mutect2.inputs.json", "WorkflowOptionsUrl": null, "WorkflowDependenciesUrl": null } \ No newline at end of file