diff --git a/.changeset/pr-354.md b/.changeset/pr-354.md new file mode 100644 index 00000000..b020a486 --- /dev/null +++ b/.changeset/pr-354.md @@ -0,0 +1,7 @@ +--- +"@wdio/browserstack-service": patch +--- + +- Fixed accessibility scans returning no results on Safari on iPhone with Jasmine or multiremote. +- Accessibility is no longer started for a build where every platform is unsupported, so no empty report is created; a warning explains why. +- Added warnings when advanced rules are requested on mobile browsers and when accessibility could not be started for a build. diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index 5d7c2000..8ecfee39 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -23,7 +23,7 @@ import WebdriverIOModule from './modules/webdriverIOModule.js' import AccessibilityModule from './modules/accessibilityModule.js' import CustomTagsModule from './modules/customTagsModule.js' import UploadAttachmentModule from './modules/uploadAttachmentModule.js' -import { isObject, isTurboScale, processAccessibilityResponse, shouldAddServiceVersion } from '../util.js' +import { ACCESSIBILITY_BUILD_START_FAILED_WARNING, isObject, isTrue, isTurboScale, processAccessibilityResponse, shouldAddServiceVersion } from '../util.js' import ObservabilityModule from './modules/observabilityModule.js' import type { BrowserstackConfig, BrowserstackOptions, LaunchResponse } from '../types.js' import CrashReporter from '../crash-reporter.js' @@ -123,6 +123,9 @@ export class BrowserstackCLI { CrashReporter.recursivelyRedactKeysFromObject(redactedStartResponse, ['user', 'username', 'key', 'accesskey', 'password']) BStackLogger.debug(`start: startBinSession response=${JSON.stringify(redactedStartResponse)}`) this.loadModules(response) + if (isTrue((this.options as BrowserstackConfig & BrowserstackOptions).accessibility) && !response.accessibility?.success) { + BStackLogger.warn(ACCESSIBILITY_BUILD_START_FAILED_WARNING) + } this.isMainConnected = true } diff --git a/packages/browserstack-service/src/launcher.ts b/packages/browserstack-service/src/launcher.ts index c2695b82..8aaa1e72 100644 --- a/packages/browserstack-service/src/launcher.ts +++ b/packages/browserstack-service/src/launcher.ts @@ -38,6 +38,7 @@ import { isUndefined, isAccessibilityAutomationSession, isTrue, + hasAppCap, getBrowserStackUser, getBrowserStackKey, getCredentialMismatchWarning, @@ -50,7 +51,10 @@ import { isValidEnabledValue, isMultiRemoteCaps, coerceStringBooleans, - validateSkipAppOverride + validateSkipAppOverride, + getRequestedMobileA11yPlatform, + MOBILE_PLATFORM_UNSUPPORTED_WARNING, + MOBILE_ADVANCED_RULES_WARNING } from './util.js' import CrashReporter from './crash-reporter.js' import { initWdioConfigPath, isAutoCaptureLogsDisabled, publishAutoCaptureDisabled } from './configCapture.js' @@ -235,6 +239,7 @@ export default class BrowserstackLauncherService implements Services.ServiceInst this._accessibilityAutomation ||= isTrue(this._options.accessibility) } this._options.accessibility = this._accessibilityAutomation + this._checkMobileA11yCapabilities(capabilities) // Default is true unless explicitly set to false this._options.testObservability = this._options.testObservability !== false @@ -1070,6 +1075,44 @@ export default class BrowserstackLauncherService implements Services.ServiceInst } } + /* Sessions turn accessibility off on unsupported mobile platforms; when every capability is one, the build doesn't start it either, so no empty report is created. */ + _checkMobileA11yCapabilities(capabilities: Capabilities.TestrunnerCapabilities | WebdriverIO.Capabilities) { + if (!this._accessibilityAutomation || !capabilities) { + return + } + // Same shapes as the constructor: plain or alwaysMatch caps, parallel multiremote entries, or a multiremote object. + const toCaps = (c: any): Record[] => { + if ('alwaysMatch' in c) { + return [c.alwaysMatch] + } + const values = Object.values(c) + return values.length && values.every((v: any) => typeof v === 'object' && v?.capabilities) ? values.map((v: any) => v.capabilities) : [c] + } + const caps = Array.isArray(capabilities) ? capabilities.flatMap(toCaps) : toCaps(capabilities) + // App Accessibility has its own platform rules. + if (!caps.length || this._options.app || isTrue(this._options.skipAppOverride) || caps.some((cap) => hasAppCap(cap) || cap.app || cap['bstack:options']?.app)) { + return + } + const platforms = caps.map(getRequestedMobileA11yPlatform) + if (platforms.every((platform) => platform && !platform.supported)) { + BStackLogger.warn(MOBILE_PLATFORM_UNSUPPORTED_WARNING) + this._accessibilityAutomation = false + this._options.accessibility = false + caps.forEach((cap) => { + if (cap['bstack:options']) { + cap['bstack:options'].accessibility = false + } + if (!isUndefined(cap['browserstack.accessibility'])) { + cap['browserstack.accessibility'] = false + } + }) + return + } + if (isTrue((this._options.accessibilityOptions?.includeIssueType as Record | undefined)?.advanced) && platforms.some((platform) => platform?.supported)) { + BStackLogger.warn(MOBILE_ADVANCED_RULES_WARNING) + } + } + /** * The runner schedules workers after onPrepare from each capability's wdio:maxInstances and * config.maxInstances, so a server parallelsPerPlatform is mapped onto them here, and server diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 29165c97..f00e434a 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -8,6 +8,7 @@ import { isBrowserstackSession, patchConsoleLogs, isTrue, + addNoopAccessibilityCommands, isFalse, getUniqueIdentifier, getHookType, @@ -279,6 +280,9 @@ export default class BrowserstackService implements Services.ServiceInstance { } else { await this._accessibilityHandler.before(sessionId) } + if (this._isCliAccessibilityFlow()) { + addNoopAccessibilityCommands(this._browser as WebdriverIO.Browser) + } Listener.setAccessibilityOptions(this._options.accessibilityOptions) } catch (err) { diff --git a/packages/browserstack-service/src/util.ts b/packages/browserstack-service/src/util.ts index c2c2e644..70b7682a 100644 --- a/packages/browserstack-service/src/util.ts +++ b/packages/browserstack-service/src/util.ts @@ -492,6 +492,40 @@ export const MOBILE_PLATFORM_UNSUPPORTED_WARNING = 'Accessibility Automation wil export const MOBILE_SCANNING_DISABLED_WARNING = 'Mobile Accessibility Automation is not enabled for your account. Accessibility scans will be skipped on mobile sessions. Contact BrowserStack to enable mobile accessibility scanning.' +export const MOBILE_ADVANCED_RULES_WARNING = 'Advanced accessibility rules are not supported on mobile browsers. Mobile sessions run the standard rules only.' + +export const ACCESSIBILITY_BUILD_START_FAILED_WARNING = 'Accessibility Automation could not be started for this build. Accessibility scans will be skipped.' + +/* Platform requested in a capability, or undefined for a desktop one. A browser that isn't named defaults to the device's own. */ +export const getRequestedMobileA11yPlatform = (capability: Record) => { + const bstackOptions = capability?.['bstack:options'] || {} + const deviceName = bstackOptions.deviceName ?? bstackOptions.device ?? capability?.['appium:deviceName'] ?? capability?.deviceName ?? capability?.device + if (!deviceName) { + return undefined + } + const platformName = String(capability.platformName || bstackOptions.os || '').toLowerCase() + const platform = /iphone|ipad/i.test(String(deviceName)) || platformName === 'ios' ? 'ios' : 'android' + const browserName = capability.browserName || bstackOptions.browserName || (platform === 'ios' ? 'safari' : 'chrome') + const osVersion = bstackOptions.osVersion ?? capability['appium:platformVersion'] ?? capability.platformVersion ?? capability.os_version + return { platform, browserName, deviceName, osVersion, supported: isMobileA11yPlatformPair(platform, browserName, deviceName, osVersion) } +} + +/* The CLI flow adds the accessibility commands only when the build started accessibility; without them, a test calling one would throw. The classic flow always adds them. */ +export const addNoopAccessibilityCommands = (browser: WebdriverIO.Browser) => { + const noops: Record Promise> = { + getAccessibilityResultsSummary: async () => ({}), + getAccessibilityResults: async () => [], + performScan: async () => undefined, + startA11yScanning: async () => undefined, + stopA11yScanning: async () => undefined + } + for (const [name, noop] of Object.entries(noops)) { + if (typeof (browser as unknown as Record)[name] !== 'function') { + (browser as unknown as Record)[name] = noop + } + } +} + export const isMobileScanningEnabledForGroup = () => process.env.BROWSERSTACK_A11Y_MOBILE_SCANNING_ENABLED_FOR_GROUP === 'true' // The scanner fails inside Safari before iOS 18, so older iPhones are rejected when the version is known. @@ -2202,6 +2236,10 @@ export async function executeAccessibilityScript( fnBody: string, arg?: unknown ): Promise { + // iOS Safari doesn't await a Promise returned from execute/sync; mobile sessions already have a script timeout set. + if (mobileA11ySession) { + return arg === undefined ? browser.executeAsync(fnBody) : browser.executeAsync(fnBody, arg) + } return browser.execute( `return (function (...bstackSdkArgs) { return new Promise((resolve, reject) => { diff --git a/packages/browserstack-service/tests/cli/frameworks/index.test.ts b/packages/browserstack-service/tests/cli/frameworks/index.test.ts index 6d223ad1..22a4162b 100644 --- a/packages/browserstack-service/tests/cli/frameworks/index.test.ts +++ b/packages/browserstack-service/tests/cli/frameworks/index.test.ts @@ -229,6 +229,20 @@ describe('BrowserstackCLI', () => { expect(browserstackCLI.isMainConnected).toBe(true) }) + it('warns when accessibility was requested but the build did not start it', async () => { + vi.spyOn(browserstackCLI, 'start').mockResolvedValue() + const warn = vi.spyOn(cliLogger.BStackLogger, 'warn').mockImplementation(() => {}) + browserstackCLI.options = { accessibility: true } + + await browserstackCLI.startMain() + expect(warn).toHaveBeenCalledWith('Accessibility Automation could not be started for this build. Accessibility scans will be skipped.') + + warn.mockClear() + browserstackCLI.options = { accessibility: false } + await browserstackCLI.startMain() + expect(warn).not.toHaveBeenCalledWith('Accessibility Automation could not be started for this build. Accessibility scans will be skipped.') + }) + it('handles errors during start', async () => { const startError = new Error('Start failed') vi.spyOn(browserstackCLI, 'start').mockRejectedValue(startError) diff --git a/packages/browserstack-service/tests/launcher.test.ts b/packages/browserstack-service/tests/launcher.test.ts index 7ff9ec01..a36c5655 100644 --- a/packages/browserstack-service/tests/launcher.test.ts +++ b/packages/browserstack-service/tests/launcher.test.ts @@ -954,6 +954,81 @@ describe('constructor', () => { ]) }) + it('turns accessibility off for the build when every capability is an unsupported mobile platform', async () => { + const warn = vi.spyOn(bstackLogger.BStackLogger, 'warn') + const serviceOptions: any = { accessibility: true } + const caps: any = [{ browserName: 'safari', 'bstack:options': { deviceName: 'iPhone 15', osVersion: '17' } }] + new BrowserstackLauncher(serviceOptions, caps, config) + + expect(serviceOptions.accessibility).toEqual(false) + expect(caps[0]['bstack:options'].accessibility).toEqual(false) + expect(warn).toHaveBeenCalledWith('Accessibility Automation will run only on Desktop browsers, Chrome on Android or Safari on iPhone with iOS 18 or later.') + }) + + it('keeps accessibility on when any capability can be scanned, and warns about advanced rules on mobile', async () => { + const warn = vi.spyOn(bstackLogger.BStackLogger, 'warn') + const serviceOptions: any = { accessibility: true, accessibilityOptions: { includeIssueType: { advanced: true } } } + const caps: any = [ + { browserName: 'safari', 'bstack:options': { deviceName: 'iPhone 15', osVersion: '17' } }, + { browserName: 'safari', 'bstack:options': { deviceName: 'iPhone 17', osVersion: '26' } }, + { browserName: 'chrome', 'bstack:options': { os: 'Windows' } } + ] + new BrowserstackLauncher(serviceOptions, caps, config) + + expect(serviceOptions.accessibility).toEqual(true) + expect(caps.map((c: any) => c['bstack:options'].accessibility)).toEqual([true, true, true]) + expect(warn).toHaveBeenCalledWith('Advanced accessibility rules are not supported on mobile browsers. Mobile sessions run the standard rules only.') + }) + + it('checks every browser of a multiremote run', async () => { + const serviceOptions: any = { accessibility: true } + const caps: any = { + a: { capabilities: { browserName: 'safari', 'bstack:options': { deviceName: 'iPhone 15', osVersion: '17' } } }, + b: { capabilities: { browserName: 'safari', 'bstack:options': { deviceName: 'iPad Pro 13 2024', osVersion: '18' } } } + } + new BrowserstackLauncher(serviceOptions, caps, config) + expect(serviceOptions.accessibility).toEqual(false) + expect(caps.a.capabilities['bstack:options'].accessibility).toEqual(false) + + const mixed: any = { accessibility: true } + new BrowserstackLauncher(mixed, { + a: { capabilities: { browserName: 'safari', 'bstack:options': { deviceName: 'iPhone 15', osVersion: '17' } } }, + b: { capabilities: { browserName: 'chrome', 'bstack:options': { os: 'Windows' } } } + } as any, config) + expect(mixed.accessibility).toEqual(true) + }) + + it('leaves App Accessibility runs to their own platform checks', async () => { + const fromCaps: any = { accessibility: true } + new BrowserstackLauncher(fromCaps, [{ 'appium:app': 'bs://app', 'bstack:options': { deviceName: 'iPhone 15', osVersion: '17' } }] as any, config) + expect(fromCaps.accessibility).toEqual(true) + + const fromOptions: any = { accessibility: true, app: 'bs://app' } + new BrowserstackLauncher(fromOptions, [{ 'bstack:options': { deviceName: 'iPad Pro 13 2024', osVersion: '17' } }] as any, config) + expect(fromOptions.accessibility).toEqual(true) + + const fromAppiumOptions: any = { accessibility: true } + new BrowserstackLauncher(fromAppiumOptions, [{ 'appium:options': { app: 'bs://app' }, 'bstack:options': { deviceName: 'iPad Pro 13 2024', osVersion: '17' } }] as any, config) + expect(fromAppiumOptions.accessibility).toEqual(true) + + const fromBundleId: any = { accessibility: true } + new BrowserstackLauncher(fromBundleId, [{ 'appium:bundleId': 'com.example.app', 'bstack:options': { deviceName: 'iPhone 15', osVersion: '17' } }] as any, config) + expect(fromBundleId.accessibility).toEqual(true) + + const skipAppOverride: any = { accessibility: true, skipAppOverride: true } + new BrowserstackLauncher(skipAppOverride, [{ 'bstack:options': { deviceName: 'iPad Pro 13 2024', osVersion: '17' } }] as any, config) + expect(skipAppOverride.accessibility).toEqual(true) + }) + + it('does not warn about advanced rules for desktop-only runs', async () => { + const warn = vi.spyOn(bstackLogger.BStackLogger, 'warn') + const serviceOptions: any = { accessibility: true, accessibilityOptions: { includeIssueType: { advanced: true } } } + new BrowserstackLauncher(serviceOptions, [{ browserName: 'chrome', 'bstack:options': { os: 'Windows' } }] as any, config) + + expect(serviceOptions.accessibility).toEqual(true) + expect(warn).not.toHaveBeenCalledWith('Advanced accessibility rules are not supported on mobile browsers. Mobile sessions run the standard rules only.') + }) + it('should add the "accessibility" property to an array of capabilities inside "bstack:options" if any extension cap present', async () => { const serviceOptions = { accessibility: true } const caps: any = [{ 'moz:firefoxOptions': {} }, { 'goog:chromeOptions': {} }] diff --git a/packages/browserstack-service/tests/util.test.ts b/packages/browserstack-service/tests/util.test.ts index 640b7568..ccc60d31 100644 --- a/packages/browserstack-service/tests/util.test.ts +++ b/packages/browserstack-service/tests/util.test.ts @@ -1504,6 +1504,41 @@ describe('validateCapsWithA11y', () => { expect(validateCapsWithA11y('Google Pixel 7', { browser_name: 'chrome', platform_name: 'android', platform_version: '13.0' })).toEqual(true) }) + it('runs a11y scripts with executeAsync on mobile and execute on desktop', async () => { + process.env.BROWSERSTACK_A11Y_MOBILE_SCANNING_ENABLED_FOR_GROUP = 'true' + const browser = { execute: vi.fn().mockResolvedValue('sync'), executeAsync: vi.fn().mockResolvedValue('async') } + validateCapsWithA11y('iPhone 17', { browser_name: 'safari', platform_name: 'ios', platform_version: '26' }) + expect(await utils.executeAccessibilityScript(browser, 'scan()', { method: 'url' })).toEqual('async') + expect(browser.executeAsync).toHaveBeenCalledWith('scan()', { method: 'url' }) + await utils.executeAccessibilityScript(browser, 'summary()') + expect(browser.executeAsync).toHaveBeenLastCalledWith('summary()') + + validateCapsWithA11y(undefined, { browser_name: 'chrome' }) + expect(await utils.executeAccessibilityScript(browser, 'scan()', { method: 'url' })).toEqual('sync') + expect(browser.executeAsync).toHaveBeenCalledTimes(2) + }) + + it('adds no-op accessibility commands without replacing existing ones', async () => { + const summary = vi.fn() + const browser: any = { getAccessibilityResultsSummary: summary } + utils.addNoopAccessibilityCommands(browser) + expect(browser.getAccessibilityResultsSummary).toBe(summary) + expect(await browser.getAccessibilityResults()).toEqual([]) + expect(await browser.performScan()).toBeUndefined() + await browser.startA11yScanning() + await browser.stopA11yScanning() + }) + + it('reads the requested mobile platform from a capability', async () => { + expect(utils.getRequestedMobileA11yPlatform({ browserName: 'chrome', 'bstack:options': { os: 'Windows' } })).toBeUndefined() + expect(utils.getRequestedMobileA11yPlatform({ browserName: 'safari', 'bstack:options': { deviceName: 'iPhone 15', osVersion: '17' } })?.supported).toEqual(false) + expect(utils.getRequestedMobileA11yPlatform({ 'bstack:options': { deviceName: 'iPhone 17', osVersion: '26' } })?.supported).toEqual(true) + expect(utils.getRequestedMobileA11yPlatform({ browserName: 'safari', 'bstack:options': { deviceName: 'iPad Pro 13 2024' } })?.supported).toEqual(false) + expect(utils.getRequestedMobileA11yPlatform({ platformName: 'android', 'appium:deviceName': 'Google Pixel 8', 'appium:platformVersion': '14.0' })?.supported).toEqual(true) + expect(utils.getRequestedMobileA11yPlatform({ browserName: 'samsung', 'bstack:options': { deviceName: 'Samsung Galaxy S23' } })?.supported).toEqual(false) + expect(utils.getRequestedMobileA11yPlatform({ browserName: 'safari', 'bstack:options': { device: 'iPhone 15', osVersion: '17' } })?.supported).toEqual(false) + }) + it('reads the iOS major from any version string and allows 18 and later', async () => { expect(utils.isMobileA11yPlatformPair('ios', 'safari', 'iPhone 15', 'iOS 17')).toEqual(false) expect(utils.isMobileA11yPlatformPair('ios', 'safari', 'iPhone 15', 17)).toEqual(false) @@ -1542,17 +1577,17 @@ describe('validateCapsWithA11y', () => { ] try { const browser = { execute: vi.fn().mockResolvedValue({}), executeAsync: vi.fn().mockResolvedValue({}) } as unknown as WebdriverIO.Browser - const mobileArgs = JSON.stringify({ is_mobile_session: true, thTestRunUuid: 'test-run-uuid' }) + const mobileArgs = { is_mobile_session: true, thTestRunUuid: 'test-run-uuid' } validateCapsWithA11y('iPhone 17', { browser_name: 'safari', platform_name: 'ios' }) await utils.performA11yScan(false, browser, true, true, 'click') await utils.getA11yResults(false, browser, true, true) await utils.getA11yResultsSummary(false, browser, true, true) - const scripts = vi.mocked(browser.execute).mock.calls.map((call) => String(call[0])) - expect(scripts.find((s) => s.includes('scanScript'))).toContain(JSON.stringify({ method: 'click', is_mobile_session: true, thTestRunUuid: 'test-run-uuid' })) - expect(scripts.find((s) => s.includes('resultsScript'))).toContain(mobileArgs) - expect(scripts.find((s) => s.includes('summaryScript'))).toContain(mobileArgs) + expect(browser.execute).not.toHaveBeenCalled() + expect(browser.executeAsync).toHaveBeenCalledWith('scanScript', { method: 'click', ...mobileArgs }) + expect(browser.executeAsync).toHaveBeenCalledWith('resultsScript', mobileArgs) + expect(browser.executeAsync).toHaveBeenCalledWith('summaryScript', mobileArgs) } finally { spies.forEach((spy) => spy.mockRestore()) delete process.env.BSTACK_A11Y_JWT