misc: implement webdriver on top of geckodriver in order to reduce overhead maintenance and code

.circleci/cache-version.txt

@@ -1,3 +1,3 @@
 # Bump this version to force CI to re-create the cache from scratch.
 
-09-30-24
+09-29-24

.circleci/workflows.yml

@@ -76,7 +76,7 @@ windowsWorkflowFilters: &windows-workflow-filters
     - equal: [ develop, << pipeline.git.branch >> ]
     # use the following branch as well to ensure that v8 snapshot cache updates are fully tested
     - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
-    - equal: [ 'misc/remove_marionette_for_geckodriver', << pipeline.git.branch >> ]
+    - equal: [ 'misc/use_webdriver', << pipeline.git.branch >> ]
     - matches:
         pattern: /^release\/\d+\.\d+\.\d+$/
         value: << pipeline.git.branch >>

cli/CHANGELOG.md

@@ -6,6 +6,7 @@ _Released 10/1/2024 (PENDING)_
 **Misc:**
 
 - Cypress now consumes [geckodriver](https://firefox-source-docs.mozilla.org/testing/geckodriver/index.html) to help automate the Firefox browser instead of [marionette-client](https://github.com/cypress-io/marionette-client). Addresses [#30217](https://github.com/cypress-io/cypress/issues/30217).
+- Cypress now consumes [webdriver](https://github.com/webdriverio/webdriverio/tree/main/packages/webdriver) to help automate the Firefox browser and [firefox-profile](https://github.com/saadtazi/firefox-profile-js) to create a firefox profile and convert it to Base64 to save user screen preferences via `xulstore.json`. Addresses [#30300](https://github.com/cypress-io/cypress/issues/30300) and [#30301](https://github.com/cypress-io/cypress/issues/30301).
 - Pass spec information to protocol's `beforeSpec` to improve troubleshooting when reporting on errors. Addressed in [#30316](https://github.com/cypress-io/cypress/pull/30316).
 
 ## 13.15.0

package.json

@@ -245,6 +245,7 @@
       "scripts"
     ],
     "nohoist": [
+      "**/@wdio/*",
       "**/webpack-preprocessor/babel-loader",
       "**/webpack-preprocessor/**/merge-source-map",
       "**/webpack-preprocessor/**/patch-package",
@@ -262,6 +263,7 @@
     "**/@types/cheerio": "0.22.21",
     "**/@types/enzyme": "3.10.5",
     "**/@types/react": "16.9.50",
+    "**/@wdio/logger": "9.0.0",
     "**/jquery": "3.4.1",
     "**/pretty-format": "26.4.0",
     "**/sharp": "0.29.3",

packages/data-context/src/DataContext.ts

@@ -181,7 +181,7 @@ export class DataContext {
   @cached
   get cloud () {
     return new CloudDataSource({
-      fetch: (...args) => this.util.fetch(...args),
+      fetch: (...args: [RequestInfo | URL, (RequestInit | undefined)?]) => this.util.fetch(...args),
       getUser: () => this.coreData.user,
       logout: () => this.actions.auth.logout().catch(this.logTraceError),
       invalidateClientUrqlCache: () => this.graphql.invalidateClientUrqlCache(this),

packages/frontend-shared/src/graphql/urqlFetchSocketAdapter.ts

@@ -3,7 +3,7 @@ import type { SocketShape } from '@packages/socket/lib/types'
 import type { ClientOptions } from '@urql/core'
 
 export const urqlFetchSocketAdapter = (io: SocketShape): ClientOptions['fetch'] => {
-  return (url, fetchOptions = {}) => {
+  return (url, fetchOptions: RequestInit = {}) => {
     return new Promise<Response>((resolve, reject) => {
       // Handle aborted requests
       if (fetchOptions.signal) {

packages/server/lib/browsers/firefox-util.ts

@@ -6,9 +6,9 @@ import Foxdriver from '@benmalka/foxdriver'
 import * as protocol from './protocol'
 import { CdpAutomation } from './cdp_automation'
 import { BrowserCriClient } from './browser-cri-client'
+import type { Client as WebDriverClient } from 'webdriver'
 import type { Automation } from '../automation'
 import type { CypressError } from '@packages/errors'
-import type { WebDriverClassic } from './webdriver-classic'
 
 const debug = Debug('cypress:server:browsers:firefox-util')
 
@@ -20,7 +20,7 @@ let timings = {
   collections: [] as any[],
 }
 
-let webDriverClassic: WebDriverClassic
+let webdriverClient: WebDriverClient
 
 const getTabId = (tab) => {
   return _.get(tab, 'browsingContextID')
@@ -103,11 +103,11 @@ async function connectToNewTabClassic () {
   // For versions 124 and above, a new tab is not created, so @packages/extension creates one for us.
   // Since the tab is always available on our behalf,
   // we can connect to it here and navigate it to about:blank to set it up for CDP connection
-  const handles = await webDriverClassic.getWindowHandles()
+  const handles = await webdriverClient.getWindowHandles()
 
-  await webDriverClassic.switchToWindow(handles[0])
+  await webdriverClient.switchToWindow(handles[0])
 
-  await webDriverClassic.navigate('about:blank')
+  await webdriverClient.navigateTo('about:blank')
 }
 
 async function connectToNewSpec (options, automation: Automation, browserCriClient: BrowserCriClient) {
@@ -140,7 +140,7 @@ async function setupCDP (remotePort: number, automation: Automation, onError?: (
 }
 
 async function navigateToUrlClassic (url: string) {
-  await webDriverClassic.navigate(url)
+  await webdriverClient.navigateTo(url)
 }
 
 const logGcDetails = () => {
@@ -213,17 +213,17 @@ export default {
     url,
     foxdriverPort,
     remotePort,
-    webDriverClassic: wdcInstance,
+    webdriverClient: wdInstance,
   }: {
     automation: Automation
     onError?: (err: Error) => void
     url: string
     foxdriverPort: number
     remotePort: number
-    webDriverClassic: WebDriverClassic
+    webdriverClient: WebDriverClient
   }): Promise<BrowserCriClient> {
     // set the WebDriver classic instance instantiated from geckodriver
-    webDriverClassic = wdcInstance
+    webdriverClient = wdInstance
     const [, browserCriClient] = await Promise.all([
       this.setupFoxdriver(foxdriverPort),
       setupCDP(remotePort, automation, onError),

packages/server/lib/browsers/firefox.ts

@@ -1,4 +1,5 @@
 import _ from 'lodash'
+import EventEmitter from 'events'
 import fs from 'fs-extra'
 import Debug from 'debug'
 import getPort from 'get-port'
@@ -18,12 +19,15 @@ import type { Automation } from '../automation'
 import { getCtx } from '@packages/data-context'
 import { getError } from '@packages/errors'
 import type { BrowserLaunchOpts, BrowserNewTabOpts, RunModeVideoApi } from '@packages/types'
-import { GeckoDriver } from './geckodriver'
-import { WebDriverClassic } from './webdriver-classic'
+import type { RemoteConfig } from 'webdriver'
+import { GeckoDriverOptions, WebDriver } from './webdriver'
 
 const debug = Debug('cypress:server:browsers:firefox')
 const debugVerbose = Debug('cypress-verbose:server:browsers:firefox')
 
+const WEBDRIVER_DEBUG_NAMESPACE_VERBOSE = 'cypress-verbose:server:browsers:webdriver'
+const GECKODRIVER_DEBUG_NAMESPACE_VERBOSE = 'cypress-verbose:server:browsers:geckodriver'
+
 // used to prevent the download prompt for the specified file types.
 // this should cover most/all file types, but if it's necessary to
 // discover more, open Firefox DevTools, download the file yourself
@@ -443,16 +447,15 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc
   const [
     foxdriverPort,
     marionettePort,
-    geckoDriverPort,
     webDriverBiDiPort,
-  ] = await Promise.all([getPort(), getPort(), getPort(), getPort()])
+  ] = await Promise.all([getPort(), getPort(), getPort()])
 
   defaultLaunchOptions.preferences['devtools.debugger.remote-port'] = foxdriverPort
   defaultLaunchOptions.preferences['marionette.port'] = marionettePort
 
   // NOTE: we get the BiDi port and set it inside of geckodriver, but BiDi is not currently enabled (see remote.active-protocols above).
   // this is so the BiDi websocket port does not get set to 0, which is the default for the geckodriver package.
-  debug('available ports: %o', { foxdriverPort, marionettePort, geckoDriverPort, webDriverBiDiPort })
+  debug('available ports: %o', { foxdriverPort, marionettePort, webDriverBiDiPort })
 
   const [
     cacheDir,
@@ -479,15 +482,19 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc
 
   debug('firefox directories %o', { path: profile.path(), cacheDir, extensionDest })
 
-  launchOptions.preferences['browser.cache.disk.parent_directory'] = cacheDir
-  for (const pref in launchOptions.preferences) {
-    const value = launchOptions.preferences[pref]
+  const xulStorePath = path.join(profile.path(), 'xulstore.json')
+
+  // if user has set custom window.sizemode pref or it's the first time launching on this profile, write to xulStore.
+  if (!await fs.pathExists(xulStorePath)) {
+    // this causes the browser to launch maximized, which chrome does by default
+    // otherwise an arbitrary size will be picked for the window size
+    // this will not have an effect after first launch in 'interactive' mode
+    const sizemode = 'maximized'
 
-    profile.setPreference(pref, value)
+    await fs.writeJSON(xulStorePath, { 'chrome://browser/content/browser.xhtml': { 'main-window': { 'width': 1280, 'height': 1024, sizemode } } })
   }
 
-  // TODO: fix this - synchronous FS operation
-  profile.updatePreferences()
+  launchOptions.preferences['browser.cache.disk.parent_directory'] = cacheDir
 
   const userCSSPath = path.join(profileDir, 'chrome')
 
@@ -518,21 +525,22 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc
   const BROWSER_ENVS = {
     MOZ_REMOTE_SETTINGS_DEVTOOLS: '1',
     MOZ_HEADLESS_WIDTH: '1280',
-    MOZ_HEADLESS_HEIGHT: '806',
+    MOZ_HEADLESS_HEIGHT: '722',
     ...launchOptions.env,
   }
 
   debug('launching geckodriver with browser envs %o', BROWSER_ENVS)
 
-  // create the geckodriver process, which we will use WebDriver Classic to open the browser
-  const geckoDriverInstance = await GeckoDriver.create({
+  debug('launch in firefox', { url, args: launchOptions.args })
+
+  const geckoDriverOptions: GeckoDriverOptions = {
     host: '127.0.0.1',
-    port: geckoDriverPort,
+    // geckodriver port is assigned under the hood by @wdio/utils
+    // @see https://github.com/webdriverio/webdriverio/blob/v9.1.1/packages/wdio-utils/src/node/startWebDriver.ts#L65
     marionetteHost: '127.0.0.1',
     marionettePort,
-    webdriverBidiPort: webDriverBiDiPort,
-    profilePath: profile.path(),
-    binaryPath: browser.path,
+    websocketPort: webDriverBiDiPort,
+    profileRoot: profile.path(),
     // To pass env variables into the firefox process, we CANNOT do it through capabilities when starting the browser.
     // Since geckodriver spawns the firefox process, we can pass the env variables directly to geckodriver, which in turn will
     // pass them to the firefox process
@@ -544,93 +552,118 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc
         ...process.env,
       },
     },
-  })
-
-  const wdcInstance = new WebDriverClassic('127.0.0.1', geckoDriverPort)
+    jsdebugger: Debug.enabled(GECKODRIVER_DEBUG_NAMESPACE_VERBOSE) || false,
+    log: Debug.enabled(GECKODRIVER_DEBUG_NAMESPACE_VERBOSE) ? 'debug' : 'error',
+    logNoTruncate: Debug.enabled(GECKODRIVER_DEBUG_NAMESPACE_VERBOSE),
+  }
 
-  debug('launch in firefox', { url, args: launchOptions.args })
+  /**
+   * To set the profile, we use the profile capabilities in firefoxOptions which
+   * requires the profile to be base64 encoded. The profile will be copied over to whatever
+   * profile is created by geckodriver stemming from the root profile path.
+   *
+   * For example, if the profileRoot in geckodriver is /usr/foo/firefox-stable/run-12345, the new webdriver session
+   * will take the base64 encoded profile contents we created in /usr/foo/firefox-stable/run-12345/* (via firefox-profile npm package) and
+   * copy it to a profile created in the profile root, which would look something like /usr/foo/firefox-stable/run-12345/rust_mozprofile<HASH>/*
+   * @see https://developer.mozilla.org/en-US/docs/Web/WebDriver/Capabilities/firefoxOptions
+   */
+  const base64EncodedProfile = await new Promise<string>((resolve, reject) => {
+    profile.encoded(function (err, encodedProfile) {
+      err ? reject(err) : resolve(encodedProfile)
+    })
+  })
 
-  const capabilitiesToSend = {
+  const newSessionCapabilities: RemoteConfig = {
+    logLevel: Debug.enabled(WEBDRIVER_DEBUG_NAMESPACE_VERBOSE) ? 'info' : 'silent',
     capabilities: {
       alwaysMatch: {
+        browserName: 'firefox',
         acceptInsecureCerts: true,
         // @see https://developer.mozilla.org/en-US/docs/Web/WebDriver/Capabilities/firefoxOptions
         'moz:firefoxOptions': {
+          profile: base64EncodedProfile,
           binary: browser.path,
           args: launchOptions.args,
           prefs: launchOptions.preferences,
         },
         // @see https://firefox-source-docs.mozilla.org/testing/geckodriver/Capabilities.html#moz-debuggeraddress
         // we specify the debugger address option for Webdriver, which will return us the CDP address when the capability is returned.
+        // @ts-expect-error
         'moz:debuggerAddress': true,
+        // @see https://webdriver.io/docs/capabilities/#wdiogeckodriveroptions
+        'wdio:geckodriverOptions': geckoDriverOptions,
       },
+      firstMatch: [],
     },
   }
+  // @ts-expect-error
+  let browserInstanceWrapper: BrowserInstance = new EventEmitter()
+
+  browserInstanceWrapper.kill = () => undefined
 
   try {
-    debugVerbose(`creating session with capabilities %s`, JSON.stringify(capabilitiesToSend.capabilities))
+    debugVerbose(`creating session with capabilities %s`, JSON.stringify(newSessionCapabilities.capabilities))
+
+    const WD = WebDriver.getWebDriverPackage()
 
     // this command starts the webdriver session and actually opens the browser
-    const { capabilities } = await wdcInstance.createSession(capabilitiesToSend)
+    // to debug geckodriver, set the DEBUG=cypress-verbose:server:browsers:geckodriver (debugs a third-party patched package geckodriver)
+    // @see ./WEB_DRIVER.md
+    const webdriverClient = await WD.newSession(newSessionCapabilities)
 
-    debugVerbose(`received capabilities %o`, capabilities)
+    debugVerbose(`received capabilities %o`, webdriverClient.capabilities)
 
-    const cdpPort = parseInt(new URL(`ws://${capabilities['moz:debuggerAddress']}`).port)
+    const cdpPort = parseInt(new URL(`ws://${webdriverClient.capabilities['moz:debuggerAddress']}`).port)
 
     debug(`CDP running on port ${cdpPort}`)
 
-    const browserPID = capabilities['moz:processID']
+    const browserPID: number = webdriverClient.capabilities['moz:processID']
 
     debug(`firefox running on pid: ${browserPID}`)
 
-    // makes it so get getRemoteDebuggingPort() is calculated correctly
-    process.env.CYPRESS_REMOTE_DEBUGGING_PORT = cdpPort.toString()
-
-    // maximize the window if running headful and no width or height args are provided.
-    // NOTE: We used to do this with xulstore.json, but this is no longer possible with geckodriver
-    // as firefox will create the profile under the profile root that we cannot control and we cannot consistently provide
-    // a base 64 encoded profile.
-    if (!browser.isHeadless && (!launchOptions.args.includes('-width') || !launchOptions.args.includes('-height'))) {
-      await wdcInstance.maximizeWindow()
-    }
-
-    // install the browser extensions
-    await Promise.all(_.map(launchOptions.extensions, (path) => {
-      debug(`installing extension at path: ${path}`)
-
-      return wdcInstance!.installAddOn({
-        path,
-        temporary: true,
-      })
-    }))
-
-    debug('setting up firefox utils')
-    browserCriClient = await firefoxUtil.setup({ automation, url, foxdriverPort, webDriverClassic: wdcInstance, remotePort: cdpPort, onError: options.onError })
+    const driverPID: number = webdriverClient.capabilities['wdio:driverPID'] as number
 
-    // monkey-patch the .kill method to that the CDP connection is closed
-    const originalGeckoDriverKill = geckoDriverInstance.kill
+    debug(`webdriver running on pid: ${driverPID}`)
 
-    geckoDriverInstance.kill = (...args) => {
+    // now that we have the driverPID and browser PID
+    browserInstanceWrapper.kill = (...args) => {
       // Do nothing on failure here since we're shutting down anyway
       clearInstanceState({ gracefulShutdown: true })
 
       debug('closing firefox')
 
-      process.kill(browserPID)
+      const browserReturnStatus = process.kill(browserPID)
 
-      debug('closing geckodriver')
+      debug('closing geckodriver and webdriver')
+      const driverReturnStatus = process.kill(driverPID)
 
-      return originalGeckoDriverKill.apply(geckoDriverInstance, args)
+      return browserReturnStatus || driverReturnStatus
     }
 
+    // makes it so get getRemoteDebuggingPort() is calculated correctly
+    process.env.CYPRESS_REMOTE_DEBUGGING_PORT = cdpPort.toString()
+
+    // install the browser extensions
+    await Promise.all(_.map(launchOptions.extensions, async (path) => {
+      debug(`installing extension at path: ${path}`)
+      const id = await webdriverClient.installAddOn(path, true)
+
+      debug(`extension with id ${id} installed!`)
+
+      return
+    }))
+
+    debug('setting up firefox utils')
+    browserCriClient = await firefoxUtil.setup({ automation, url, foxdriverPort, webdriverClient, remotePort: cdpPort, onError: options.onError })
+
     await utils.executeAfterBrowserLaunch(browser, {
       webSocketDebuggerUrl: browserCriClient.getWebSocketDebuggerUrl(),
     })
   } catch (err) {
     errors.throwErr('FIREFOX_COULD_NOT_CONNECT', err)
   }
 
-  return geckoDriverInstance
+  return browserInstanceWrapper
 }
 
 export async function closeExtraTargets () {