Merge A11Y-13878-wdio-mobile-followups

.changeset/pr-352.md

@@ -0,0 +1,7 @@
+---
+"@wdio/browserstack-service": patch
+---
+
+- Fixed accessibility scans returning no results on Safari on iPhone with Jasmine or multiremote.
+- Accessibility is no longer started for a build where every platform is unsupported, so no empty report is created; a warning explains why.
+- Added warnings when advanced rules are requested on mobile browsers and when accessibility could not be started for a build.

.changeset/pr-353.md

@@ -0,0 +1,5 @@
+---
+"@wdio/browserstack-service": minor
+---
+
+- A11 y 13878 wdio mobile followups

packages/browserstack-service/src/cli/index.ts

@@ -23,7 +23,7 @@ import WebdriverIOModule from './modules/webdriverIOModule.js'
 import AccessibilityModule from './modules/accessibilityModule.js'
 import CustomTagsModule from './modules/customTagsModule.js'
 import UploadAttachmentModule from './modules/uploadAttachmentModule.js'
-import { isObject, isTurboScale, processAccessibilityResponse, shouldAddServiceVersion } from '../util.js'
+import { ACCESSIBILITY_BUILD_START_FAILED_WARNING, isObject, isTrue, isTurboScale, processAccessibilityResponse, shouldAddServiceVersion } from '../util.js'
 import ObservabilityModule from './modules/observabilityModule.js'
 import type { BrowserstackConfig, BrowserstackOptions, LaunchResponse } from '../types.js'
 import CrashReporter from '../crash-reporter.js'
@@ -123,6 +123,9 @@ export class BrowserstackCLI {
         CrashReporter.recursivelyRedactKeysFromObject(redactedStartResponse, ['user', 'username', 'key', 'accesskey', 'password'])
         BStackLogger.debug(`start: startBinSession response=${JSON.stringify(redactedStartResponse)}`)
         this.loadModules(response)
+        if (isTrue((this.options as BrowserstackConfig & BrowserstackOptions).accessibility) && !response.accessibility?.success) {
+            BStackLogger.warn(ACCESSIBILITY_BUILD_START_FAILED_WARNING)
+        }
         this.isMainConnected = true
     }
 

packages/browserstack-service/src/launcher.ts

@@ -38,6 +38,7 @@ import {
     isUndefined,
     isAccessibilityAutomationSession,
     isTrue,
+    hasAppCap,
     getBrowserStackUser,
     getBrowserStackKey,
     getCredentialMismatchWarning,
@@ -50,7 +51,10 @@ import {
     isValidEnabledValue,
     isMultiRemoteCaps,
     coerceStringBooleans,
-    validateSkipAppOverride
+    validateSkipAppOverride,
+    getRequestedMobileA11yPlatform,
+    MOBILE_PLATFORM_UNSUPPORTED_WARNING,
+    MOBILE_ADVANCED_RULES_WARNING
 } from './util.js'
 import CrashReporter from './crash-reporter.js'
 import { initWdioConfigPath, isAutoCaptureLogsDisabled, publishAutoCaptureDisabled } from './configCapture.js'
@@ -235,6 +239,7 @@ export default class BrowserstackLauncherService implements Services.ServiceInst
             this._accessibilityAutomation ||= isTrue(this._options.accessibility)
         }
         this._options.accessibility = this._accessibilityAutomation
+        this._checkMobileA11yCapabilities(capabilities)
 
         // Default is true unless explicitly set to false
         this._options.testObservability = this._options.testObservability !== false
@@ -1070,6 +1075,44 @@ export default class BrowserstackLauncherService implements Services.ServiceInst
         }
     }
 
+    /* Sessions turn accessibility off on unsupported mobile platforms; when every capability is one, the build doesn't start it either, so no empty report is created. */
+    _checkMobileA11yCapabilities(capabilities: Capabilities.TestrunnerCapabilities | WebdriverIO.Capabilities) {
+        if (!this._accessibilityAutomation || !capabilities) {
+            return
+        }
+        // Same shapes as the constructor: plain or alwaysMatch caps, parallel multiremote entries, or a multiremote object.
+        const toCaps = (c: any): Record<string, any>[] => {
+            if ('alwaysMatch' in c) {
+                return [c.alwaysMatch]
+            }
+            const values = Object.values(c)
+            return values.length && values.every((v: any) => typeof v === 'object' && v?.capabilities) ? values.map((v: any) => v.capabilities) : [c]
+        }
+        const caps = Array.isArray(capabilities) ? capabilities.flatMap(toCaps) : toCaps(capabilities)
+        // App Accessibility has its own platform rules.
+        if (!caps.length || this._options.app || isTrue(this._options.skipAppOverride) || caps.some((cap) => hasAppCap(cap) || cap.app || cap['bstack:options']?.app)) {
+            return
+        }
+        const platforms = caps.map(getRequestedMobileA11yPlatform)
+        if (platforms.every((platform) => platform && !platform.supported)) {
+            BStackLogger.warn(MOBILE_PLATFORM_UNSUPPORTED_WARNING)
+            this._accessibilityAutomation = false
+            this._options.accessibility = false
+            caps.forEach((cap) => {
+                if (cap['bstack:options']) {
+                    cap['bstack:options'].accessibility = false
+                }
+                if (!isUndefined(cap['browserstack.accessibility'])) {
+                    cap['browserstack.accessibility'] = false
+                }
+            })
+            return
+        }
+        if (isTrue((this._options.accessibilityOptions?.includeIssueType as Record<string, unknown> | undefined)?.advanced) && platforms.some((platform) => platform?.supported)) {
+            BStackLogger.warn(MOBILE_ADVANCED_RULES_WARNING)
+        }
+    }
+
     /**
      * The runner schedules workers after onPrepare from each capability's wdio:maxInstances and
      * config.maxInstances, so a server parallelsPerPlatform is mapped onto them here, and server

packages/browserstack-service/src/service.ts

@@ -8,6 +8,7 @@ import {
     isBrowserstackSession,
     patchConsoleLogs,
     isTrue,
+    addNoopAccessibilityCommands,
     isFalse,
     getUniqueIdentifier,
     getHookType,
@@ -279,6 +280,9 @@ export default class BrowserstackService implements Services.ServiceInstance {
                     } else {
                         await this._accessibilityHandler.before(sessionId)
                     }
+                    if (this._isCliAccessibilityFlow()) {
+                        addNoopAccessibilityCommands(this._browser as WebdriverIO.Browser)
+                    }
 
                     Listener.setAccessibilityOptions(this._options.accessibilityOptions)
                 } catch (err) {

packages/browserstack-service/src/util.ts

@@ -492,6 +492,40 @@ export const MOBILE_PLATFORM_UNSUPPORTED_WARNING = 'Accessibility Automation wil
 
 export const MOBILE_SCANNING_DISABLED_WARNING = 'Mobile Accessibility Automation is not enabled for your account. Accessibility scans will be skipped on mobile sessions. Contact BrowserStack to enable mobile accessibility scanning.'
 
+export const MOBILE_ADVANCED_RULES_WARNING = 'Advanced accessibility rules are not supported on mobile browsers. Mobile sessions run the standard rules only.'
+
+export const ACCESSIBILITY_BUILD_START_FAILED_WARNING = 'Accessibility Automation could not be started for this build. Accessibility scans will be skipped.'
+
+/* Platform requested in a capability, or undefined for a desktop one. A browser that isn't named defaults to the device's own. */
+export const getRequestedMobileA11yPlatform = (capability: Record<string, any>) => {
+    const bstackOptions = capability?.['bstack:options'] || {}
+    const deviceName = bstackOptions.deviceName ?? bstackOptions.device ?? capability?.['appium:deviceName'] ?? capability?.deviceName ?? capability?.device
+    if (!deviceName) {
+        return undefined
+    }
+    const platformName = String(capability.platformName || bstackOptions.os || '').toLowerCase()
+    const platform = /iphone|ipad/i.test(String(deviceName)) || platformName === 'ios' ? 'ios' : 'android'
+    const browserName = capability.browserName || bstackOptions.browserName || (platform === 'ios' ? 'safari' : 'chrome')
+    const osVersion = bstackOptions.osVersion ?? capability['appium:platformVersion'] ?? capability.platformVersion ?? capability.os_version
+    return { platform, browserName, deviceName, osVersion, supported: isMobileA11yPlatformPair(platform, browserName, deviceName, osVersion) }
+}
+
+/* The CLI flow adds the accessibility commands only when the build started accessibility; without them, a test calling one would throw. The classic flow always adds them. */
+export const addNoopAccessibilityCommands = (browser: WebdriverIO.Browser) => {
+    const noops: Record<string, () => Promise<unknown>> = {
+        getAccessibilityResultsSummary: async () => ({}),
+        getAccessibilityResults: async () => [],
+        performScan: async () => undefined,
+        startA11yScanning: async () => undefined,
+        stopA11yScanning: async () => undefined
+    }
+    for (const [name, noop] of Object.entries(noops)) {
+        if (typeof (browser as unknown as Record<string, unknown>)[name] !== 'function') {
+            (browser as unknown as Record<string, unknown>)[name] = noop
+        }
+    }
+}
+
 export const isMobileScanningEnabledForGroup = () => process.env.BROWSERSTACK_A11Y_MOBILE_SCANNING_ENABLED_FOR_GROUP === 'true'
 
 // The scanner fails inside Safari before iOS 18, so older iPhones are rejected when the version is known.
@@ -2202,6 +2236,10 @@ export async function executeAccessibilityScript<ReturnType>(
     fnBody: string,
     arg?: unknown
 ): Promise<ReturnType> {
+    // iOS Safari doesn't await a Promise returned from execute/sync; mobile sessions already have a script timeout set.
+    if (mobileA11ySession) {
+        return arg === undefined ? browser.executeAsync(fnBody) : browser.executeAsync(fnBody, arg)
+    }
     return browser.execute(
         `return (function (...bstackSdkArgs) {
             return new Promise((resolve, reject) => {

packages/browserstack-service/tests/cli/frameworks/index.test.ts

@@ -229,6 +229,20 @@ describe('BrowserstackCLI', () => {
             expect(browserstackCLI.isMainConnected).toBe(true)
         })
 
+        it('warns when accessibility was requested but the build did not start it', async () => {
+            vi.spyOn(browserstackCLI, 'start').mockResolvedValue()
+            const warn = vi.spyOn(cliLogger.BStackLogger, 'warn').mockImplementation(() => {})
+            browserstackCLI.options = { accessibility: true }
+
+            await browserstackCLI.startMain()
+            expect(warn).toHaveBeenCalledWith('Accessibility Automation could not be started for this build. Accessibility scans will be skipped.')
+
+            warn.mockClear()
+            browserstackCLI.options = { accessibility: false }
+            await browserstackCLI.startMain()
+            expect(warn).not.toHaveBeenCalledWith('Accessibility Automation could not be started for this build. Accessibility scans will be skipped.')
+        })
+
         it('handles errors during start', async () => {
             const startError = new Error('Start failed')
             vi.spyOn(browserstackCLI, 'start').mockRejectedValue(startError)

packages/browserstack-service/tests/launcher.test.ts

@@ -954,6 +954,81 @@ describe('constructor', () => {
         ])
     })
 
+    it('turns accessibility off for the build when every capability is an unsupported mobile platform', async () => {
+        const warn = vi.spyOn(bstackLogger.BStackLogger, 'warn')
+        const serviceOptions: any = { accessibility: true }
+        const caps: any = [{ browserName: 'safari', 'bstack:options': { deviceName: 'iPhone 15', osVersion: '17' } }]
+        new BrowserstackLauncher(serviceOptions, caps, config)
+
+        expect(serviceOptions.accessibility).toEqual(false)
+        expect(caps[0]['bstack:options'].accessibility).toEqual(false)
+        expect(warn).toHaveBeenCalledWith('Accessibility Automation will run only on Desktop browsers, Chrome on Android or Safari on iPhone with iOS 18 or later.')
+    })
+
+    it('keeps accessibility on when any capability can be scanned, and warns about advanced rules on mobile', async () => {
+        const warn = vi.spyOn(bstackLogger.BStackLogger, 'warn')
+        const serviceOptions: any = { accessibility: true, accessibilityOptions: { includeIssueType: { advanced: true } } }
+        const caps: any = [
+            { browserName: 'safari', 'bstack:options': { deviceName: 'iPhone 15', osVersion: '17' } },
+            { browserName: 'safari', 'bstack:options': { deviceName: 'iPhone 17', osVersion: '26' } },
+            { browserName: 'chrome', 'bstack:options': { os: 'Windows' } }
+        ]
+        new BrowserstackLauncher(serviceOptions, caps, config)
+
+        expect(serviceOptions.accessibility).toEqual(true)
+        expect(caps.map((c: any) => c['bstack:options'].accessibility)).toEqual([true, true, true])
+        expect(warn).toHaveBeenCalledWith('Advanced accessibility rules are not supported on mobile browsers. Mobile sessions run the standard rules only.')
+    })
+
+    it('checks every browser of a multiremote run', async () => {
+        const serviceOptions: any = { accessibility: true }
+        const caps: any = {
+            a: { capabilities: { browserName: 'safari', 'bstack:options': { deviceName: 'iPhone 15', osVersion: '17' } } },
+            b: { capabilities: { browserName: 'safari', 'bstack:options': { deviceName: 'iPad Pro 13 2024', osVersion: '18' } } }
+        }
+        new BrowserstackLauncher(serviceOptions, caps, config)
+        expect(serviceOptions.accessibility).toEqual(false)
+        expect(caps.a.capabilities['bstack:options'].accessibility).toEqual(false)
+
+        const mixed: any = { accessibility: true }
+        new BrowserstackLauncher(mixed, {
+            a: { capabilities: { browserName: 'safari', 'bstack:options': { deviceName: 'iPhone 15', osVersion: '17' } } },
+            b: { capabilities: { browserName: 'chrome', 'bstack:options': { os: 'Windows' } } }
+        } as any, config)
+        expect(mixed.accessibility).toEqual(true)
+    })
+
+    it('leaves App Accessibility runs to their own platform checks', async () => {
+        const fromCaps: any = { accessibility: true }
+        new BrowserstackLauncher(fromCaps, [{ 'appium:app': 'bs://app', 'bstack:options': { deviceName: 'iPhone 15', osVersion: '17' } }] as any, config)
+        expect(fromCaps.accessibility).toEqual(true)
+
+        const fromOptions: any = { accessibility: true, app: 'bs://app' }
+        new BrowserstackLauncher(fromOptions, [{ 'bstack:options': { deviceName: 'iPad Pro 13 2024', osVersion: '17' } }] as any, config)
+        expect(fromOptions.accessibility).toEqual(true)
+
+        const fromAppiumOptions: any = { accessibility: true }
+        new BrowserstackLauncher(fromAppiumOptions, [{ 'appium:options': { app: 'bs://app' }, 'bstack:options': { deviceName: 'iPad Pro 13 2024', osVersion: '17' } }] as any, config)
+        expect(fromAppiumOptions.accessibility).toEqual(true)
+
+        const fromBundleId: any = { accessibility: true }
+        new BrowserstackLauncher(fromBundleId, [{ 'appium:bundleId': 'com.example.app', 'bstack:options': { deviceName: 'iPhone 15', osVersion: '17' } }] as any, config)
+        expect(fromBundleId.accessibility).toEqual(true)
+
+        const skipAppOverride: any = { accessibility: true, skipAppOverride: true }
+        new BrowserstackLauncher(skipAppOverride, [{ 'bstack:options': { deviceName: 'iPad Pro 13 2024', osVersion: '17' } }] as any, config)
+        expect(skipAppOverride.accessibility).toEqual(true)
+    })
+
+    it('does not warn about advanced rules for desktop-only runs', async () => {
+        const warn = vi.spyOn(bstackLogger.BStackLogger, 'warn')
+        const serviceOptions: any = { accessibility: true, accessibilityOptions: { includeIssueType: { advanced: true } } }
+        new BrowserstackLauncher(serviceOptions, [{ browserName: 'chrome', 'bstack:options': { os: 'Windows' } }] as any, config)
+
+        expect(serviceOptions.accessibility).toEqual(true)
+        expect(warn).not.toHaveBeenCalledWith('Advanced accessibility rules are not supported on mobile browsers. Mobile sessions run the standard rules only.')
+    })
+
     it('should add the "accessibility" property to an array of capabilities inside "bstack:options" if any extension cap present', async () => {
         const serviceOptions = { accessibility: true }
         const caps: any = [{ 'moz:firefoxOptions': {} }, { 'goog:chromeOptions': {} }]