Retry app crash tests and consider then non-fatal if they pass (#456)

Introducing a flag `retry-app-crash-tests` to retry tests that
might have caused app crashes. When this flag is turned on, the
execution will not fail if all app crash tests pass on retry.

The retries will honor the `error-retries` count, restarts the
sim before retrying the crashed test and makes the crash logs
available, like before.

README.md

@@ -89,6 +89,7 @@ A full list supported options are listed here.
 |       video-paths      |           -V           | A list of videos that will be saved in the simulators.                              |     N    | n/a              |
 |       image-paths      |           -I           | A list of images that will be saved in the simulators.                              |     N    | n/a              |
 | unsafe-skip-xcode-version-check |               | Skip Xcode version check                                                            |     N    | NO               |
+|  retry-app-crash-tests |                        | Retry tests that crashed app and consider it non-fatal if it passes on retry.       |     N    | false            |
 
 
 ## Exit Status

bp/src/BPConfiguration.h

@@ -79,6 +79,7 @@ typedef NS_ENUM(NSInteger, BPProgram) {
 @property (nonatomic) BOOL saveDiagnosticsOnError;
 @property (nonatomic, strong) NSNumber *failureTolerance;
 @property (nonatomic) BOOL onlyRetryFailed;
+@property (nonatomic) BOOL retryAppCrashTests;
 @property (nonatomic, strong) NSArray *testCasesToSkip;
 @property (nonatomic, strong) NSArray *testCasesToRun;
 @property (nonatomic, strong) NSArray *allTestCases;

bp/src/BPConfiguration.m

@@ -103,7 +103,7 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) {
     {'q', "quiet", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "quiet",
         "Turn off all output except fatal errors."},
     {'F', "only-retry-failed", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "onlyRetryFailed",
-        "Only retry failed tests instead of all. Also retry test that timed-out/crashed. Note that app crashes are fatal even if the test passes on retry."},
+        "Only retry failed tests instead of all. Also retry test that timed-out/crashed."},
     {'l', "list-tests", BP_MASTER, NO, NO, no_argument, NULL, BP_VALUE | BP_BOOL, "listTestsOnly",
         "Only list tests and exit without executing tests."},
     {'v', "verbose", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "verboseLogging",
@@ -143,7 +143,9 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) {
     {364, "test-plan-path", BP_MASTER | BP_SLAVE, NO, NO, required_argument, NULL, BP_VALUE | BP_PATH, "testPlanPath",
         "The path of a json file which describes the test plan. It is equivalent to the .xctestrun file generated by Xcode, but it can be generated by a different build system, e.g. Bazel"},
     {365, "unsafe-skip-xcode-version-check", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL , "unsafeSkipXcodeVersionCheck",
-    "   "},
+        "Skip Xcode version check if using an Xcode version that is not officially supported the Bluepill version being used. Not safe/recommended and has a limited support."},
+    {366, "retry-app-crash-tests", BP_MASTER | BP_SLAVE, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "retryAppCrashTests",
+        "Retry the tests after an app crash and if it passes on retry, consider them non-fatal."},
 
     {0, 0, 0, 0, 0, 0, 0}
 };

bp/src/Bluepill.m

@@ -631,8 +631,10 @@ - (void)finishWithContext:(BPExecutionContext *)context {
             return;
 
         case BPExitStatusAppCrashed:
-            // Crashed test is considered fatal and shall not be retried
-            self.finalExitStatus |= context.exitStatus;
+            if (!self.config.retryAppCrashTests) {
+                // Crashed test is considered fatal when retry is disabled
+                self.finalExitStatus |= context.exitStatus;
+            }
             NEXT([self proceed]);
             return;
 

bp/src/SimulatorMonitor.m

@@ -195,8 +195,12 @@ - (void)onOutputReceived:(NSString *)output {
             NSString *testClass = (__self.currentClassName ?: __self.previousClassName);
             NSString *testName = (__self.currentTestName ?: __self.previousTestName);
             if (__self.testsState == Running) {
-                [self updateExecutedTestCaseList:testName inClass:testClass];
-                [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Not retrying it.", testClass, testName];
+                if (self.config.retryAppCrashTests) {
+                    [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Configured to retry.", testClass, testName];
+                } else {
+                    [self updateExecutedTestCaseList:testName inClass:testClass];
+                    [BPUtils printInfo:CRASH withString:@"%@/%@ crashed app. Retry disabled.", testClass, testName];
+                }
                 [[BPStats sharedStats] endTimer:[NSString stringWithFormat:TEST_CASE_FORMAT, [BPStats sharedStats].attemptNumber, testClass, testName] withResult:@"CRASHED"];
             } else {
                 assert(__self.testsState == Idle);

bp/tests/BluepillTests.m

@@ -35,7 +35,7 @@ @implementation BluepillTests
 
 - (void)setUp {
     [super setUp];
-    
+
     self.continueAfterFailure = NO;
     NSString *hostApplicationPath = [BPTestHelper sampleAppPath];
     NSString *testBundlePath = [BPTestHelper sampleAppNegativeTestsBundlePath];
@@ -222,11 +222,11 @@ - (void)testReportWithAppCrashingAndRetryOnlyFailedTestsSet {
     self.config.outputDirectory = outputDir;
     self.config.errorRetriesCount = @1;
     self.config.failureTolerance = @1;
-    self.config.onlyRetryFailed = YES;
-    
+    self.config.onlyRetryFailed = TRUE;
+
     BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
     XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
-    
+
     NSString *junitReportPath = [outputDir stringByAppendingPathComponent:@"TEST-BPSampleAppCrashingTests-1-results.xml"];
     NSLog(@"JUnit file: %@", junitReportPath);
     NSString *expectedFilePath = [[[NSBundle bundleForClass:[self class]] resourcePath] stringByAppendingPathComponent:@"crash_tests_with_retry_attempt_1.xml"];
@@ -248,7 +248,7 @@ - (void)DISABLE_testAppCrashingAndRetryReportsCorrectExitCode {
     self.config.testing_crashOnAttempt = @1;
     self.config.errorRetriesCount = @2;
     self.config.failureTolerance = @1;
-    self.config.onlyRetryFailed = YES;
+    self.config.onlyRetryFailed = TRUE;
 
     BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
     XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed);
@@ -326,11 +326,11 @@ - (void)testReportWithAppHangingTestsShouldReturnFailure {
 }
 
 /**
- Execution plan: TIMEOUT, CRASH, PASS
+ Execution plan: TIMEOUT, CRASH (not retried)
  */
 - (void)testReportFailureOnTimeoutCrashAndPass {
     self.config.stuckTimeout = @6;
-    self.config.testing_ExecutionPlan = @"TIMEOUT CRASH PASS";
+    self.config.testing_ExecutionPlan = @"TIMEOUT CRASH";
     self.config.errorRetriesCount = @4;
     self.config.onlyRetryFailed = TRUE;
     NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
@@ -345,6 +345,48 @@ - (void)testReportFailureOnTimeoutCrashAndPass {
     XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
 }
 
+/**
+ Execution plan: TIMEOUT, CRASH, CRASH w/ flag to retry crashes and consider them non-fatal
+ */
+- (void)testReportFailureOnTimeoutCrashAndCrashOnRetry {
+    self.config.stuckTimeout = @6;
+    self.config.retryAppCrashTests = TRUE;
+    self.config.testing_ExecutionPlan = @"TIMEOUT CRASH CRASH";
+    self.config.errorRetriesCount = @2;
+    self.config.onlyRetryFailed = TRUE;
+    NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
+    self.config.testBundlePath = testBundlePath;
+    NSString *tempDir = NSTemporaryDirectory();
+    NSError *error;
+    NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
+    NSLog(@"output directory is %@", outputDir);
+    self.config.outputDirectory = outputDir;
+
+    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+    XCTAssertTrue(exitCode == (BPExitStatusTestTimeout | BPExitStatusAppCrashed));
+}
+
+/**
+ Execution plan: TIMEOUT, CRASH, PASS w/ flag to retry crashes and consider them non-fatal
+ */
+- (void)testReportSuccessOnTimeoutCrashAndPassOnRetry {
+    self.config.stuckTimeout = @6;
+    self.config.retryAppCrashTests = TRUE;
+    self.config.testing_ExecutionPlan = @"TIMEOUT CRASH PASS";
+    self.config.errorRetriesCount = @4;
+    self.config.onlyRetryFailed = TRUE;
+    NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
+    self.config.testBundlePath = testBundlePath;
+    NSString *tempDir = NSTemporaryDirectory();
+    NSError *error;
+    NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
+    NSLog(@"output directory is %@", outputDir);
+    self.config.outputDirectory = outputDir;
+
+    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+    XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed);
+}
+
 /**
  Execution plan: CRASH
  */
@@ -387,6 +429,28 @@ - (void)testReportFailureOnCrashAndTimeoutTests {
     XCTAssertTrue(exitCode == BPExitStatusAppCrashed);
 }
 
+/**
+ Execution plan: Test crashes but passes on retry w/ retry app crash tests flag set
+ */
+- (void)testReportSuccessOnAppCrashTestPassesOnRetry {
+    self.config.stuckTimeout = @6;
+    self.config.retryAppCrashTests = TRUE;
+    self.config.testing_ExecutionPlan = @"CRASH PASS; SKIP PASS";
+    self.config.onlyRetryFailed = TRUE;
+    self.config.failureTolerance = @1;
+    self.config.errorRetriesCount = @2;
+    NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
+    self.config.testBundlePath = testBundlePath;
+    NSString *tempDir = NSTemporaryDirectory();
+    NSError *error;
+    NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error];
+    NSLog(@"output directory is %@", outputDir);
+    self.config.outputDirectory = outputDir;
+
+    BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
+    XCTAssertTrue(exitCode == BPExitStatusAllTestsPassed);
+}
+
 /**
  Execution plan: One test CRASHes and another one keeps timing out
  */
@@ -457,7 +521,7 @@ - (void)testReportSuccessOnTimeoutAndPassOnRetry {
     self.config.stuckTimeout = @6;
     self.config.testing_ExecutionPlan = @"TIMEOUT PASS";
     self.config.errorRetriesCount = @4;
-    self.config.onlyRetryFailed = YES;
+    self.config.onlyRetryFailed = TRUE;
     self.config.failureTolerance = @0;  // Not relevant
     NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;
@@ -478,7 +542,7 @@ - (void)testReportFailureOnTimeoutAndNoRetry {
     self.config.stuckTimeout = @6;
     self.config.testing_ExecutionPlan = @"TIMEOUT";
     self.config.errorRetriesCount = @2;
-    self.config.onlyRetryFailed = NO;
+    self.config.onlyRetryFailed = FALSE;
     self.config.failureTolerance = @1;  // Not relevant since it's not a test failure
     NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;
@@ -500,7 +564,7 @@ - (void)testReportSuccessOnFailedTestAndPassOnRetryAll {
     self.config.testing_ExecutionPlan = @"FAIL PASS";
     self.config.errorRetriesCount = @4;
     self.config.onlyRetryFailed = NO;  // Indicates to retry all tests when a test fails
-    self.config.failureTolerance = @1;  // Even though failureTolerance is non-zero it wouldn't retry because onlyRetryFailed = NO
+    self.config.failureTolerance = @1;
     NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;
     NSString *tempDir = NSTemporaryDirectory();
@@ -578,7 +642,7 @@ - (void)testRetryOnlyFailures {
     self.config.outputDirectory = outputDir;
     self.config.errorRetriesCount = @100;
     self.config.failureTolerance = @1;
-    self.config.onlyRetryFailed = YES;
+    self.config.onlyRetryFailed = TRUE;
     BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run];
     XCTAssert(exitCode == BPExitStatusTestsFailed);
     // Make sure all tests started on the first run
@@ -626,7 +690,7 @@ - (void)testKeepSimulatorWithAppCrashingTestsSet  {
     NSString *testBundlePath = [BPTestHelper sampleAppCrashingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;
     self.config.keepSimulator = YES;
-    
+
     Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config];
     BPExitStatus exitCode = [bp run];
     XCTAssert(exitCode == BPExitStatusAppCrashed);
@@ -639,7 +703,7 @@ - (void)testKeepSimulatorWithAppHangingTestsSet  {
     self.config.testBundlePath = testBundlePath;
     self.config.keepSimulator = YES;
     self.config.testing_ExecutionPlan = @"TIMEOUT";
-    
+
     Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config];
     BPExitStatus exitCode = [bp run];
     XCTAssert(exitCode == BPExitStatusTestTimeout);
@@ -649,15 +713,15 @@ - (void)testDeleteSimulatorOnly {
     NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath];
     self.config.testBundlePath = testBundlePath;
     self.config.keepSimulator = YES;
-    
+
     Bluepill *bp = [[Bluepill alloc ] initWithConfiguration:self.config];
     BPExitStatus exitCode = [bp run];
     XCTAssert(exitCode == BPExitStatusAllTestsPassed);
     XCTAssertNotNil(bp.test_simulatorUDID);
-    
+
     self.config.deleteSimUDID = bp.test_simulatorUDID;
     XCTAssertNotNil(self.config.deleteSimUDID);
-    
+
     Bluepill *bp2 = [[Bluepill alloc ] initWithConfiguration:self.config];
     BPExitStatus exitCode2 = [bp2 run];
     XCTAssert(exitCode2 == BPExitStatusSimulatorDeleted);