Support running UI tests.

facebookarchive · May 18, 2019 · 543348a · 543348a
1 parent 9219e0d
commit 543348a
Show file tree

Hide file tree

Showing 8 changed files with 399 additions and 70 deletions.
diff --git a/otest-shim/otest-shim/otest-shim.m b/otest-shim/otest-shim/otest-shim.m
@@ -456,6 +456,13 @@ static void XCTestSuite_performTest(id self, SEL sel, id arg1)
   XCPerformTestWithSuppressedExpectedAssertionFailures(self, originalSelector, arg1);
 }
 
+#pragma mark - UI Tests
+static BOOL XCUIApplication_prefersPlatformLauncher(id self, SEL sel)
+{
+  // Override to force XCUIApplication to not rely on Xcode when running UI tests
+  return YES;
+}
+
 #pragma mark - _enableSymbolication
 static BOOL XCTestCase__enableSymbolication(id self, SEL sel)
 {
@@ -568,6 +575,9 @@ static void SwizzleXCTestMethodsIfAvailable()
     XTSwizzleSelectorForFunction(NSClassFromString(@"XCTestSuite"),
                                  @selector(performTest:),
                                  (IMP)XCTestSuite_performTest);
+    XTSwizzleSelectorForFunction(NSClassFromString(@"XCUIApplication"),
+                                 @selector(prefersPlatformLauncher),
+                                 (IMP)XCUIApplication_prefersPlatformLauncher);
     if ([NSClassFromString(@"XCTestCase") respondsToSelector:@selector(_enableSymbolication)]) {
       // Disable symbolication thing on xctest 7 because it sometimes takes forever.
       XTSwizzleClassSelectorForFunction(NSClassFromString(@"XCTestCase"),

diff --git a/xctool/xctool-tests/RunTestsActionTests.m b/xctool/xctool-tests/RunTestsActionTests.m
@@ -1153,6 +1153,28 @@ - (void)testActionOptionMultipleAppTests
   }];
 }
 
+- (void)testActionOptionUITest
+{
+  [[FakeTaskManager sharedManager] runBlockWithFakeTasks:^{
+    Options *options = [[Options optionsFrom:
+  @[
+    @"-sdk", @"macosx10.7",
+    @"run-tests",
+    @"-uiTest",
+    TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/PlugIns/TestProject-UITestsUITests.xctest:"
+    TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/TestProject-UITestsUITests-Runner:"
+    TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITests.app/TestProject-UITests"
+    ]] assertOptionsValidate];
+    RunTestsAction *action = options.actions[0];
+    RunTestsActionUITest *config =
+    [[RunTestsActionUITest alloc] initWithHostApp:TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITests.app/TestProject-UITests"
+                                        runnerApp:TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/TestProject-UITestsUITests-Runner"];
+    assertThat(action.uiTests, equalTo(
+    @{TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/PlugIns/TestProject-UITestsUITests.xctest": config,
+      }));
+  }];
+}
+
 - (void)testActionOptionMixedLogicAndAppTests
 {
   [[FakeTaskManager sharedManager] runBlockWithFakeTasks:^{
@@ -1210,7 +1232,7 @@ - (void)testWillComplainWhenPassingAppTestThatDoesntExist
                               @"-appTest", TEST_DATA @"path/to/this-does-not-exist.xctest:path/to/HostApp.app/HostApp",
                               ]]
        assertOptionsFailToValidateWithError:
-           @"run-tests: Application test at path '" TEST_DATA @"path/to/this-does-not-exist.xctest' does not exist or is not a directory"];
+           @"run-tests: option -appTest has invalid argument: path '" TEST_DATA @"path/to/this-does-not-exist.xctest' doesn't exist"];
 
   }];
 }
@@ -1225,8 +1247,51 @@ - (void)testWillComplainWhenPassingHostAppBinaryThatDoesntExist
                                            TEST_DATA @"path/to/NonExistentHostApp.app/HostApp",
                               ]]
        assertOptionsFailToValidateWithError:
-           @"run-tests: Application test host binary at path '" TEST_DATA "path/to/NonExistentHostApp.app/HostApp' does not exist or is not a file"];
+           @"run-tests: option -appTest has invalid argument: path '" TEST_DATA "path/to/NonExistentHostApp.app/HostApp' doesn't exist"];
+
+  }];
+}
+
+- (void)testWillComplainWhenPassingUITestBundleThatDoesntExist
+{
+  [[FakeTaskManager sharedManager] runBlockWithFakeTasks:^{
+    [[Options optionsFrom:
+      @[@"-sdk", @"iphonesimulator",
+        @"run-tests",
+        @"-uiTest",
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/PlugIns/TestProject-UITestsUITestsFAKE.xctest:"
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/TestProject-UITestsUITests-Runner:"
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITests.app/TestProject-UITests"]]
+     assertOptionsFailToValidateWithError:
+     @"run-tests: option -uiTest has invalid argument: path '"TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/PlugIns/TestProject-UITestsUITestsFAKE.xctest' doesn't exist"];
+  }];
+}
+
+- (void)testWillComplainWhenPassingUITestRunnerAppThatDoesntExist {
+  [[FakeTaskManager sharedManager] runBlockWithFakeTasks:^{
+    [[Options optionsFrom:
+      @[@"-sdk", @"iphonesimulator",
+        @"run-tests",
+        @"-uiTest",
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/PlugIns/TestProject-UITestsUITests.xctest:"
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/TestProject-UITestsUITests-RunnerFAKE:"
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITests.app/TestProject-UITests"]]
+     assertOptionsFailToValidateWithError:
+     @"run-tests: option -uiTest has invalid argument: path '"TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/TestProject-UITestsUITests-RunnerFAKE' doesn't exist"];
+  }];
+}
 
+- (void)testWillComplainWhenPassingUITestHostAppThatDoesntExist {
+  [[FakeTaskManager sharedManager] runBlockWithFakeTasks:^{
+    [[Options optionsFrom:
+      @[@"-sdk", @"iphonesimulator",
+        @"run-tests",
+        @"-uiTest",
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/PlugIns/TestProject-UITestsUITests.xctest:"
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/TestProject-UITestsUITests-Runner:"
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITests.app/TestProject-UITestsFAKE"]]
+     assertOptionsFailToValidateWithError:
+     @"run-tests: option -uiTest has invalid argument: path '"TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITests.app/TestProject-UITestsFAKE' doesn't exist"];
   }];
 }
 
@@ -1247,6 +1312,26 @@ - (void)testWillComplainWhenPassingSameLogicTestForMultipleTestHostApps
   }];
 }
 
+- (void)testWillComplainWhenPassingSameLogicTestForMultipleUITests
+{
+  [[FakeTaskManager sharedManager] runBlockWithFakeTasks:^{
+    [[Options optionsFrom:
+      @[@"-sdk", @"iphonesimulator",
+        @"run-tests",
+        @"-uiTest",
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/PlugIns/TestProject-UITestsUITests.xctest:"
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/TestProject-UITestsUITests-Runner:"
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITests.app/TestProject-UITests",
+        @"-uiTest",
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/PlugIns/TestProject-UITestsUITests.xctest:"
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/TestProject-UITestsUITests-Runner:"
+        TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITests.app/TestProject-UITests",
+        ]]
+     assertOptionsFailToValidateWithError:
+     @"run-tests: The same test bundle '"TEST_DATA @"TestProject-UITests/Build/Products/Debug-iphonesimulator/TestProject-UITestsUITests-Runner.app/PlugIns/TestProject-UITestsUITests.xctest' cannot have more than one test configuration of host app and runner"];
+  }];
+}
+
 - (void)testPassingLogicTestViaCommandLine
 {
   [[FakeTaskManager sharedManager] runBlockWithFakeTasks:^{

diff --git a/xctool/xctool/OCUnitIOSAppTestRunner.m b/xctool/xctool/OCUnitIOSAppTestRunner.m
@@ -44,6 +44,22 @@ - (void)runTestsAndFeedOutputTo:(FdOutputLineFeedBlock)outputLineBlock
     return;
   }
 
+  NSString *testRunnerBundlePath;
+  NSString *testRunnerBundleID;
+  if ([_buildSettings[Xcode_USES_XCTRUNNER] boolValue]) {
+    // manually specified
+    if (_buildSettings[Xcode_UI_RUNNER_APP] != nil) {
+      testRunnerBundleID =
+      [self bundleIDForAppAtPath:_buildSettings[Xcode_UI_RUNNER_APP]
+                      bundlePath:&testRunnerBundlePath
+                           error:startupError];
+    } else {
+      // extract from build settings
+      testRunnerBundlePath = [_buildSettings[Xcode_TARGET_BUILD_DIR] stringByDeletingLastPathComponent];
+      testRunnerBundleID = AppBundleIDForAppAtPath(testRunnerBundlePath);
+    }
+  }
+
   BOOL (^prepareSimulator)(BOOL freshSimulator, BOOL resetSimulator, NSString **error) =
   ^(BOOL freshSimulator, BOOL resetSimulator, NSString **error) {
     if (freshSimulator || resetSimulator) {
@@ -120,6 +136,14 @@ - (void)runTestsAndFeedOutputTo:(FdOutputLineFeedBlock)outputLineBlock
                                                  error:error]) {
         return NO;
       }
+
+      if (testRunnerBundleID != nil &&
+          ![SimulatorWrapper uninstallTestHostBundleID:testRunnerBundleID
+                                                device:[_simulatorInfo simulatedDevice]
+                                             reporters:_reporters
+                                                 error:error]) {
+            return NO;
+          }
     }
 
     // Always install the app before running it.  We've observed that
@@ -139,6 +163,14 @@ - (void)runTestsAndFeedOutputTo:(FdOutputLineFeedBlock)outputLineBlock
                                              error:error]) {
       return NO;
     }
+    if (testRunnerBundleID != nil && testRunnerBundlePath != nil &&
+        ![SimulatorWrapper installTestHostBundleID:testRunnerBundleID
+                                    fromBundlePath:testRunnerBundlePath
+                                            device:[_simulatorInfo simulatedDevice]
+                                         reporters:_reporters
+                                             error:error]) {
+      return NO;
+    }
     return YES;
   };
 
@@ -191,7 +223,7 @@ - (void)runTestsAndFeedOutputTo:(FdOutputLineFeedBlock)outputLineBlock
   // Let's try several times to run before reporting about failure to callers.
   for (NSInteger remainingAttempts = kMaxRunTestsAttempts - 1; remainingAttempts >= 0; --remainingAttempts) {
     NSError *error = nil;
-    BOOL infraSucceeded = [SimulatorWrapper runHostAppTests:testHostBundleID
+    BOOL infraSucceeded = [SimulatorWrapper runHostAppTests:testRunnerBundleID ?: testHostBundleID
                                                      device:[_simulatorInfo simulatedDevice]
                                                   arguments:appLaunchArgs
                                                 environment:appLaunchEnvironment

diff --git a/xctool/xctool/OCUnitTestRunner.m b/xctool/xctool/OCUnitTestRunner.m
@@ -345,18 +345,47 @@ - (NSArray *)testArgumentsWithSpecifiedTestsToRun
 
 - (NSDictionary *)testEnvironmentWithSpecifiedTestConfiguration
 {
-  NSArray *testCasesToSkip = [self testCasesToSkip];
-
   Class XCTestConfigurationClass = NSClassFromString(@"XCTestConfiguration");
   NSAssert(XCTestConfigurationClass, @"XCTestConfiguration isn't available");
 
-  XCTestConfiguration *configuration = [[XCTestConfigurationClass alloc] init];
-  [configuration setProductModuleName:_buildSettings[Xcode_PRODUCT_MODULE_NAME]];
-  [configuration setTestBundleURL:[NSURL fileURLWithPath:[_simulatorInfo productBundlePath]]];
-  [configuration setTestsToSkip:[NSSet setWithArray:testCasesToSkip]];
-  [configuration setReportResultsToIDE:NO];
+  NSArray *testCasesToSkip = [self testCasesToSkip];
+  NSString *testHostAppPath = FixedAppPathFromAppPath(_simulatorInfo.testHostPath);
+  NSString *testHostBundleID = AppBundleIDForAppAtPath(testHostAppPath);
+
+  XCTestConfiguration *configuration = [XCTestConfigurationClass new];
+  configuration.testBundleURL = [NSURL fileURLWithPath:_simulatorInfo.productBundlePath];
+  configuration.productModuleName = _buildSettings[Xcode_PRODUCT_MODULE_NAME];
+  configuration.testsToSkip = [NSSet setWithArray:testCasesToSkip];
+  configuration.targetApplicationPath = testHostAppPath;
+  configuration.targetApplicationBundleID = testHostBundleID;
+  configuration.reportActivities = YES;
+
+  // UI tests require special treatment
+  if ([_buildSettings[Xcode_USES_XCTRUNNER] boolValue]) {
+    configuration.testsMustRunOnMainThread = YES;
+    configuration.initializeForUITesting = YES;
+  }
 
-  NSString *XCTestConfigurationFilename = [NSString stringWithFormat:@"%@-%@", _buildSettings[Xcode_PRODUCT_NAME], [configuration.sessionIdentifier UUIDString]];
+  // Xcode also sets these properties for UI (and may be other) tests:
+  // - testBundleRelativePath:
+  //   uses __TESTHOST__ placeholder in the path to xctest bundle
+  // - sessionIdentifier:
+  //   needs to be reset, so xctest doesn't expect any session to be running
+  // - testApplicationDependencies:
+  //   @{@"runner bundle id" -> "runner app bundle path",
+  //     @"target app bundle id" -> "target app bundle path"}
+  // - aggregateStatisticsBeforeCrash:
+  //   @{@"XCSuiteRecordsKey": @{}}
+  // - automationFrameworkPath:
+  //   set to XCTAutomationSupport.framework path inside Xcode app bundle
+  // - automationFrameworkPath:
+  //   set to 1
+  // - systemAttachmentLifetime:
+  //   set to 1
+
+  NSString *XCTestConfigurationFilename = [NSString stringWithFormat:@"%@-%@",
+                                           _buildSettings[Xcode_PRODUCT_MODULE_NAME],
+                                           configuration.sessionIdentifier.UUIDString];
   NSString *XCTestConfigurationFilePath = [MakeTempFileWithPrefix(XCTestConfigurationFilename) stringByAppendingPathExtension:@"xctestconfiguration"];
   if ([[NSFileManager defaultManager] fileExistsAtPath:XCTestConfigurationFilePath]) {
     [[NSFileManager defaultManager] removeItemAtPath:XCTestConfigurationFilePath error:nil];

diff --git a/xctool/xctool/RunTestsAction.h b/xctool/xctool/RunTestsAction.h
@@ -48,7 +48,9 @@ typedef NS_ENUM(NSInteger, BucketBy) {
   // be in the same bucket.
   BucketByClass,
 
-} ;
+};
+
+@class RunTestsActionUITest;
 
 @interface RunTestsAction : Action<TestRunning>
 
@@ -65,7 +67,8 @@ typedef NS_ENUM(NSInteger, BucketBy) {
 @property (nonatomic, strong) NSMutableArray *onlyList;
 @property (nonatomic, strong) NSMutableArray *omitList;
 @property (nonatomic, strong) NSMutableArray *logicTests;
-@property (nonatomic, strong) NSMutableDictionary *appTests;
+@property (nonatomic, strong) NSMutableDictionary<NSString *, NSString *> *appTests; // testBundle -> hostApp
+@property (nonatomic, strong) NSMutableDictionary<NSString *, RunTestsActionUITest *> *uiTests; // testBundle -> (host app, runner app)
 @property (nonatomic, copy) NSString *targetedDeviceFamily;
 
 - (void)setLogicTestBucketSizeValue:(NSString *)str;
@@ -74,3 +77,9 @@ typedef NS_ENUM(NSInteger, BucketBy) {
 - (void)setTestTimeoutValue:(NSString *)str;
 
 @end
+
+@interface RunTestsActionUITest: NSObject
+@property (nonatomic, copy, readonly) NSString *hostApp;
+@property (nonatomic, copy, readonly) NSString *runnerApp;
+- (instancetype)initWithHostApp:(NSString *)hostApp runnerApp:(NSString *)runnerApp;
+@end