TINY-10006: Add --remote flag for remote testing (#131)

* TINY-10006: Add `--remote` option --------- Co-authored-by: James Johnson <[email protected]> Co-authored-by: jscasca <[email protected]>
tinymce · Nov 23, 2023 · 62fb0c7 · 62fb0c7
1 parent 9e50d5e
commit 62fb0c7
Show file tree

Hide file tree

Showing 23 changed files with 4,108 additions and 1,880 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+- Add `--remote <webdriver>` to support remote webdrivers: aws device farm and lambdatest #TINY-10006
+- Tunneling configuration: Ssh Tunneling and LambdaTest Tunneling #TINY-10006
+
 ## 13.6.0 - 2023-10-16
 
 ## Improved

diff --git a/modules/sample/src/test/ts/client/pass/CustomRouteTest.ts b/modules/sample/src/test/ts/client/pass/CustomRouteTest.ts
@@ -20,7 +20,7 @@ describe('CustomRouteTest', () => {
 
   it('undefined route', sendTest('/custom/non-existent', 'json', (xhr) => {
     Assert.eq('Status', 404, xhr.status);
-    Assert.eq('Content Type', 'text/plain', xhr.getResponseHeader('Content-Type'));
+    Assert.eq('Content Type', 'text/plain; charset=utf-8', xhr.getResponseHeader('Content-Type'));
   }));
 
   it('image binary file route', sendTest('/custom/image', 'arraybuffer', (xhr) => {

diff --git a/modules/server/package.json b/modules/server/package.json
@@ -14,13 +14,16 @@
     "buildAndTest": "yarn prepublishOnly && yarn test"
   },
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=16.0.0"
   },
   "dependencies": {
+    "@aws-sdk/client-device-farm": "^3.354.0",
     "@ephox/bedrock-client": "^13.0.0",
     "@ephox/bedrock-common": "^13.0.0",
     "@ephox/bedrock-runner": "^13.6.0",
     "@jsdevtools/coverage-istanbul-loader": "^3.0.5",
+    "@lambdatest/node-tunnel": "^4.0.4",
+    "@wdio/globals": "^8.14.1",
     "async": "^3.0.0",
     "chalk": "^4.1.1",
     "cli-highlight": "^2.1.11",
@@ -29,6 +32,7 @@
     "core-js": "^3.6.4",
     "core-js-bundle": "^3.6.4",
     "cross-spawn": "^7.0.0",
+    "deepmerge": "^4.3.1",
     "etag": "^1.8.1",
     "finalhandler": "^1.1.2",
     "fork-ts-checker-webpack-plugin": "^8.0.0",
@@ -44,9 +48,10 @@
     "rollup-plugin-typescript2": "^0.24.0",
     "serve-static": "^1.10.2",
     "source-map-loader": "^3.0.0",
+    "split2": "^4.2.0",
     "ts-loader": "^9.0.0",
     "tsconfig-paths-webpack-plugin": "^3.2.0",
-    "webdriverio": "^7.31.1",
+    "webdriverio": "^8.0.0",
     "webpack": "^5.40.0",
     "webpack-dev-server": "^4.7.0",
     "which": "^2.0.0",
@@ -63,7 +68,8 @@
     "@types/jquery": "^3.5.11",
     "@types/mime-types": "^2.1.1",
     "@types/mkdirp": "^1.0.1",
-    "@wdio/types": "^7.30.2"
+    "@types/split2": "^4.2.0",
+    "@wdio/types": "^8.0.0"
   },
   "peerDependencies": {
     "typescript": ">=3.8.0"

diff --git a/modules/server/src/main/ts/BedrockAuto.ts b/modules/server/src/main/ts/BedrockAuto.ts
@@ -6,11 +6,14 @@ import * as RunnerRoutes from './bedrock/server/RunnerRoutes';
 import * as Reporter from './bedrock/core/Reporter';
 import * as DriverMaster from './bedrock/server/DriverMaster';
 import * as Driver from './bedrock/auto/Driver';
+import * as Tunnel from './bedrock/auto/Tunnel';
 import * as Lifecycle from './bedrock/core/Lifecycle';
 import { BedrockAutoSettings } from './bedrock/core/Settings';
 import { ExitCodes } from './bedrock/util/ExitCodes';
 import * as ConsoleReporter from './bedrock/core/ConsoleReporter';
 import * as SettingsResolver from './bedrock/core/SettingsResolver';
+import * as portfinder from 'portfinder';
+import { format } from 'node:util';
 
 export const go = (bedrockAutoSettings: BedrockAutoSettings): void => {
   console.log('bedrock-auto ' + Version.get() + ' starting...');
@@ -21,11 +24,36 @@ export const go = (bedrockAutoSettings: BedrockAutoSettings): void => {
   const isPhantom = browserName === 'phantomjs';
   const isHeadless = settings.browser.endsWith('-headless') || isPhantom;
   const basePage = 'src/resources/html/' + (isPhantom ? 'bedrock-phantom.html' : 'bedrock.html');
+  // Remote settings
+  const remoteWebdriver = settings.remote;
+  const sishDomain = settings.sishDomain;
+  const username = settings.username ?? process.env.LT_USERNAME;
+  const accesskey = settings.accesskey ?? process.env.LT_ACCESS_KEY;
 
   const routes = RunnerRoutes.generate('auto', settings.projectdir, settings.basedir, settings.config, settings.bundler, settings.testfiles, settings.chunk, settings.retries, settings.singleTimeout, settings.stopOnFailure, basePage, settings.coverage, settings.polyfills);
 
   routes.then(async (runner) => {
+    const shutdownServices: (() => Promise<any>)[] = [];
 
+    // LambdaTest Tunnel must know dev server port, but tunnel must be created before dev server.
+    const servicePort = await portfinder.getPortPromise({
+      port: 8000,
+      stopPort: 20000
+    });
+
+    const tunnelCredentials = {
+      user: username,
+      key: accesskey
+    };
+
+    const tunnel = await Tunnel.prepareConnection(servicePort, remoteWebdriver, sishDomain, tunnelCredentials);
+    shutdownServices.push(tunnel.shutdown);
+    const location = tunnel.url.href;
+
+    console.log('Creating webdriver...');
+    if (remoteWebdriver == 'aws') {
+        console.log('INFO: Webdriver creation waits for device farm session to activate. Takes 30-45s.');
+    }
     const driver = await Driver.create({
       browser: browserName,
       basedir: settings.basedir,
@@ -35,8 +63,13 @@ export const go = (bedrockAutoSettings: BedrockAutoSettings): void => {
       extraBrowserCapabilities: settings.extraBrowserCapabilities,
       verbose: settings.verbose,
       wipeBrowserCache: settings.wipeBrowserCache,
+      remoteWebdriver,
       webdriverPort: settings.webdriverPort,
-      useSelenium: settings.useSelenium
+      useSelenium: settings.useSelenium,
+      username,
+      accesskey,
+      devicefarmRegion: settings.devicefarmRegion,
+      deviceFarmArn: settings.devicefarmArn
     });
 
     const webdriver = driver.webdriver;
@@ -45,17 +78,20 @@ export const go = (bedrockAutoSettings: BedrockAutoSettings): void => {
       driver: Attempt.passed(webdriver),
       master,
       runner,
-      stickyFirstSession: true
+      stickyFirstSession: true,
+      port: servicePort
     });
+    shutdownServices.push(service.shutdown, driver.shutdown);
 
-    const shutdown = () => Promise.all([ service.shutdown(), driver.shutdown() ]);
+    const shutdown = () => Promise.allSettled(shutdownServices.map((shutdown_fn) => shutdown_fn()));
 
     try {
       if (!isHeadless) {
-        console.log('bedrock-auto ' + Version.get() + ' available at: http://localhost:' + service.port);
+        console.log('bedrock-auto ' + Version.get() + ' available at: ' + location);
       }
 
-      await webdriver.url('http://localhost:' + service.port);
+      console.log('Loading initial page...');
+      await webdriver.url(location);
       console.log(isPhantom ? '\nPhantom tests loading ...\n' : '\nInitial page has loaded ...\n');
       service.markLoaded();
       service.enableHud();
@@ -73,7 +109,8 @@ export const go = (bedrockAutoSettings: BedrockAutoSettings): void => {
       return Lifecycle.error(e, webdriver, shutdown, settings.gruntDone, settings.delayExit);
     }
   }).catch((err) => {
-    console.error(chalk.red(err));
+    // Chalk does not use a formatter. Using node's built-in to expand Objects, etc.
+    console.error(chalk.red('Error creating webdriver', format(err)));
     Lifecycle.exit(settings.gruntDone, ExitCodes.failures.unexpected);
   });
 };

diff --git a/modules/server/src/main/ts/bedrock/auto/Driver.ts b/modules/server/src/main/ts/bedrock/auto/Driver.ts
@@ -1,10 +1,14 @@
+
 import * as path from 'path';
 import * as childProcess from 'child_process';
 import * as os from 'os';
 import * as WebdriverIO from 'webdriverio';
 import * as portfinder from 'portfinder';
 import * as Shutdown from '../util/Shutdown';
 import * as DriverLoader from './DriverLoader';
+import * as RemoteDriver from './RemoteDriver';
+import deepmerge = require('deepmerge');
+import { RemoteOptions } from 'webdriverio';
 
 export interface DriverSettings {
   basedir: string;
@@ -17,11 +21,17 @@ export interface DriverSettings {
   webdriverPort?: number;
   webdriverTimeout?: number;
   wipeBrowserCache?: boolean;
+  servicePort?: number;
+  remoteWebdriver?: string;
   useSelenium?: boolean;
+  username?: string;
+  accesskey?: string;
+  devicefarmRegion?: string;
+  deviceFarmArn?: string;
 }
 
 export interface Driver {
-  webdriver: WebdriverIO.Browser<'async'>;
+  webdriver: WebdriverIO.Browser;
   shutdown: (immediate?: boolean) => Promise<void>;
 }
 
@@ -80,11 +90,8 @@ const getExtraBrowserCapabilities = (settings: DriverSettings): string[] => {
 };
 
 const getOptions = (port: number, browserName: string, settings: DriverSettings, debuggingPort: number): WebdriverIO.RemoteOptions => {
-  const options = {
-    path: settings.useSelenium ? '/wd/hub' : '/',
-    host: '127.0.0.1',
-    port,
-    logLevel: 'silent' as const,
+  const options: WebdriverIO.RemoteOptions = {
+    logLevel: 'warn' as const,
     capabilities: {
       browserName
     }
@@ -105,6 +112,7 @@ const getOptions = (port: number, browserName: string, settings: DriverSettings,
     addArguments(caps, 'moz:firefoxOptions', extraCaps);
   } else if (browserName === 'MicrosoftEdge') {
     addArguments(caps, 'ms:edgeOptions', ['--guest']);
+    caps['ms:edgeChromium'] = true;
   } else if (browserName === 'internet explorer' && settings.wipeBrowserCache) {
     // Setup wiping the browser cache if required, as IE 11 doesn't use a clean session by default
     caps['se:ieOptions'] = {
@@ -130,10 +138,22 @@ const getOptions = (port: number, browserName: string, settings: DriverSettings,
     }
   }
 
-  return options;
+  const driverOpts = deepmerge(
+    options,
+    settings.remoteWebdriver ?
+      RemoteDriver.getOpts(browserName, settings) :
+      {
+        path: settings.useSelenium ? '/wd/hub' : '/',
+        hostname: '127.0.0.1',
+        port
+      }
+  ) as RemoteOptions;
+  console.log('Print Opts: ', driverOpts);
+
+  return driverOpts;
 };
 
-const logDriverDetails = (driver: WebdriverIO.Browser<'async'>, headless: boolean, debuggingPort: number) => {
+const logDriverDetails = (driver: WebdriverIO.Browser, headless: boolean, debuggingPort: number) => {
   const caps: Record<string, any> = driver.capabilities;
   const browserName = caps.browserName;
   const browserVersion = caps.browserVersion || caps.version;
@@ -165,7 +185,7 @@ const focusBrowser = (browserName: string, settings: DriverSettings) => {
   }
 };
 
-const setupShutdown = (driver: WebdriverIO.Browser<'async'>, driverApi: DriverLoader.DriverAPI, shutdownDelay = 0): (immediate?: boolean) => Promise<void> => {
+const setupShutdown = (driver: WebdriverIO.Browser, driverApi: DriverLoader.DriverAPI, shutdownDelay = 0): (immediate?: boolean) => Promise<void> => {
   const driverShutdown = async (immediate?: boolean) => {
     try {
       if (immediate) {
@@ -213,6 +233,26 @@ const getDriverSpec = (settings: DriverSettings, browserName: string): DriverLoa
   };
 };
 
+const driverSetup = async (driver: WebdriverIO.Browser, settings: DriverSettings, debuggingPort: number): Promise<void> => {
+  // Browsers have a habit of reporting via the webdriver that they're ready before they are (particularly FireFox).
+
+  // setTimeout is a temporary solution, VAN-66 has been logged to investigate properly
+  await driver.pause(1500);
+
+  // Log driver details
+  logDriverDetails(driver, settings.headless, debuggingPort);
+
+  // Some tests require large windows, so make it as large as it can be.
+  // Headless modes can't use maximize, so just set the dimensions to 1280x1024
+  if (settings.headless) {
+    await driver.setWindowSize(1280, 1024);
+  } else {
+    await driver.maximizeWindow();
+  }
+
+  return Promise.resolve();
+};
+
 /* Settings:
  *
  * browser: the name of the browser
@@ -221,61 +261,58 @@ const getDriverSpec = (settings: DriverSettings, browserName: string): DriverLoa
  * webdriverTimeout: how long to wait for the webdriver server to start
  */
 export const create = async (settings: DriverSettings): Promise<Driver> => {
-
   const webdriverTimeout = settings.webdriverTimeout || 30000;
 
   const browserName = browserVariants[settings.browser] || settings.browser;
 
-  const driverSpec = getDriverSpec(settings, browserName);
-
-  try {
-    // Find an open port to start the driver on
-    const port = await getPort(settings.webdriverPort, 4444);
-    const debuggingPort = settings.headless ? await getPort(settings.debuggingPort, 9000) : 9000;
+  // Find an open port to start the driver on
+  const port = await getPort(settings.webdriverPort, 4444);
+  const debuggingPort = settings.headless ? await getPort(settings.debuggingPort, 9000) : 9000;
+  const webdriverOptions = getOptions(port, browserName, settings, debuggingPort);
+  console.log('Webdriver options:', webdriverOptions);
 
-    // Wait for the driver to start up and then start the webdriver session
-    await DriverLoader.startAndWaitForAlive(driverSpec, port, webdriverTimeout);
-
-    const webdriverOptions = getOptions(port, browserName, settings, debuggingPort);
-
-    if (settings.verbose) {
-      console.log(
-        `Browser capabilities: ${JSON.stringify(webdriverOptions.capabilities)}`
-      );
-    }
+  if (settings.remoteWebdriver) {
+    const remoteDriver = await RemoteDriver.getApi(settings, browserName, webdriverOptions);
+    await driverSetup(remoteDriver.webdriver, settings, debuggingPort);
+    return remoteDriver;
+  } else {
+    // Local
+    const driverSpec = getDriverSpec(settings, browserName);
+    try {
 
-    const driver = await WebdriverIO.remote(webdriverOptions);
+      if (settings.verbose) {
+        console.log(
+          `Browser capabilities: ${JSON.stringify(webdriverOptions.capabilities)}`
+        );
+      }
 
-    // IEDriverServer ignores a delete session call if done too quickly so it needs a small delay
-    const shutdownDelay = browserName === 'internet explorer' ? 500 : 0;
+      // Wait for the driver to start up and then start the webdriver session
+      await DriverLoader.startAndWaitForAlive(driverSpec, port, webdriverTimeout);
+
+      const driver = await WebdriverIO.remote(webdriverOptions);
 
-    // Ensure the driver gets shutdown correctly if shutdown
-    // by the user instead of the application
-    const driverShutdown = setupShutdown(driver, driverSpec.driverApi, shutdownDelay);
+      // IEDriverServer ignores a delete session call if done too quickly so it needs a small delay
+      const shutdownDelay = browserName === 'internet explorer' ? 500 : 0;
 
-    // Browsers have a habit of reporting via the webdriver that they're ready before they are (particularly FireFox).
-    // setTimeout is a temporary solution, VAN-66 has been logged to investigate properly
-    await driver.pause(1500);
+      // Ensure the driver gets shutdown correctly if shutdown
+      // by the user instead of the application
+      const driverShutdown = setupShutdown(driver, driverSpec.driverApi, shutdownDelay);
 
-    // Log driver details
-    logDriverDetails(driver, settings.headless, debuggingPort);
+      await driverSetup(driver, settings, debuggingPort);
+      await focusBrowser(browserName, settings);
 
-    // Some tests require large windows, so make it as large as it can be.
-    // Headless modes can't use maximize, so just set the dimensions to 1280x1024
-    if (settings.headless) {
-      await driver.setWindowSize(1280, 1024);
-    } else {
-      await driver.maximizeWindow();
+      // Return the public driver api
+      return {
+        webdriver: driver,
+        shutdown: driverShutdown
+      };
+    } catch (e) {
+      try {
+        driverSpec.driverApi.stop();
+      } catch {
+       // Ignore 
+      }
+      return Promise.reject(e);
     }
-    await focusBrowser(browserName, settings);
-
-    // Return the public driver api
-    return {
-      webdriver: driver,
-      shutdown: driverShutdown
-    };
-  } catch (e) {
-    driverSpec.driverApi.stop();
-    return Promise.reject(e);
   }
-};
+};