[browserstack-service] Add Funnel Data instrumentation [v8] (#12392)

* add TestObservability usage stats

* eslint fixes

* eslint fixes

* send funnel data from cleanup

* add test errors and refactor workers data

* remove test errors code

* review comments fixes

* review comment fixes

* removed events from log

* add and fix tests

* fix tests

* refactor

* refactor code and fix tests

* refactor code and add comments

* store env in constants.ts

* fix sending funnel data from cleanup

* Update packages/wdio-browserstack-service/src/config.ts

Co-authored-by: Christian Bromann <git@bromann.dev>

* Update packages/wdio-browserstack-service/src/request-handler.ts

Co-authored-by: Christian Bromann <git@bromann.dev>

* empty commit to trigger tests

* fix empty request and send funnel data from onPrepare hook

* Update packages/wdio-browserstack-service/tests/request-handler.test.ts

---------

Co-authored-by: Christian Bromann <git@bromann.dev>

packages/wdio-browserstack-service/src/bstackLogger.ts

@@ -11,15 +11,13 @@ const log = logger('@wdio/browserstack-service')
 
 export class BStackLogger {
     public static logFilePath = path.join(process.cwd(), LOGS_FILE)
-    private static logFolderPath = path.join(process.cwd(), 'logs')
+    public static logFolderPath = path.join(process.cwd(), 'logs')
     private static logFileStream: fs.WriteStream | null
 
     static logToFile(logMessage: string, logLevel: string) {
         try {
             if (!this.logFileStream) {
-                if (!fs.existsSync(this.logFolderPath)){
-                    fs.mkdirSync(this.logFolderPath)
-                }
+                this.ensureLogsFolder()
                 this.logFileStream = fs.createWriteStream(this.logFilePath, { flags: 'a' })
             }
             if (this.logFileStream && this.logFileStream.writable) {
@@ -75,4 +73,10 @@ export class BStackLogger {
             fs.truncateSync(this.logFilePath)
         }
     }
+
+    public static ensureLogsFolder() {
+        if (!fs.existsSync(this.logFolderPath)){
+            fs.mkdirSync(this.logFolderPath)
+        }
+    }
 }

packages/wdio-browserstack-service/src/cleanup.ts

@@ -1,24 +1,96 @@
-import { stopBuildUpstream } from './util.js'
+import { getErrorString, stopBuildUpstream } from './util.js'
 import { BStackLogger } from './bstackLogger.js'
+import fs from 'node:fs'
+import { fireFunnelRequest } from './instrumentation/funnelInstrumentation.js'
+import { TESTOPS_BUILD_ID_ENV, TESTOPS_JWT_ENV } from './constants.js'
 
 export default class BStackCleanup {
     static async startCleanup() {
         try {
-            await this.executeObservabilityCleanup()
+            // Get funnel data object from saved file
+            const funnelDataCleanup = process.argv.includes('--funnelData')
+            let funnelData = null
+            if (funnelDataCleanup) {
+                const index = process.argv.indexOf('--funnelData')
+                const filePath = process.argv[index + 1]
+                funnelData = this.getFunnelDataFromFile(filePath)
+            }
+
+            if (process.argv.includes('--observability')) {
+                await this.executeObservabilityCleanup(funnelData)
+            }
+
+            if (funnelDataCleanup && funnelData) {
+                await this.sendFunnelData(funnelData)
+            }
         } catch (err) {
             const error = err as string
             BStackLogger.error(error)
         }
     }
-    static async executeObservabilityCleanup() {
-        if (!process.env.BS_TESTOPS_JWT) {
+    static async executeObservabilityCleanup(funnelData: any) {
+        if (!process.env[TESTOPS_JWT_ENV]) {
             return
         }
         BStackLogger.debug('Executing observability cleanup')
-        await stopBuildUpstream()
-        if (process.env.BS_TESTOPS_BUILD_HASHED_ID) {
-            BStackLogger.info(`\nVisit https://observability.browserstack.com/builds/${process.env.BS_TESTOPS_BUILD_HASHED_ID} to view build report, insights, and many more debugging information all at one place!\n`)
+        try {
+            const result = await stopBuildUpstream()
+            if (process.env[TESTOPS_BUILD_ID_ENV]) {
+                BStackLogger.info(`\nVisit https://observability.browserstack.com/builds/${process.env[TESTOPS_BUILD_ID_ENV]} to view build report, insights, and many more debugging information all at one place!\n`)
+            }
+            const status = (result && result.status) || 'failed'
+            const message = (result && result.message)
+            this.updateO11yStopData(funnelData, status, status === 'failed' ? message : undefined)
+        } catch (e: unknown) {
+            BStackLogger.error('Error in stopping Observability build: ' + e)
+            this.updateO11yStopData(funnelData, 'failed', e)
+        }
+    }
+
+    static updateO11yStopData(funnelData: any, status: string, error: unknown = undefined) {
+        const toData = funnelData?.event_properties?.productUsage?.testObservability
+        // Return if no O11y data in funnel data
+        if (!toData) {
+            return
+        }
+        let existingStopData = toData.events.buildEvents.finished
+        existingStopData = existingStopData || {}
+
+        existingStopData = {
+            ...existingStopData,
+            status,
+            error: getErrorString(error),
+            stoppedFrom: 'exitHook'
+        }
+        toData.events.buildEvents.finished = existingStopData
+    }
+
+    static async sendFunnelData(funnelData: any) {
+        try {
+            await fireFunnelRequest(funnelData)
+            BStackLogger.debug('Funnel data sent successfully from cleanup')
+        } catch (e: unknown) {
+            BStackLogger.error('Error in sending funnel data: ' + e)
+        }
+    }
+
+    static getFunnelDataFromFile(filePath: string) {
+        if (!filePath) {
+            return null
+        }
+
+        const content = fs.readFileSync(filePath, 'utf8')
+
+        const data = JSON.parse(content)
+        this.removeFunnelDataFile(filePath)
+        return data
+    }
+
+    static removeFunnelDataFile(filePath?: string) {
+        if (!filePath) {
+            return
         }
+        fs.rmSync(filePath, { force: true })
     }
 }
 

packages/wdio-browserstack-service/src/config.ts

@@ -0,0 +1,47 @@
+import type { AppConfig, BrowserstackConfig } from './types.js'
+import type { Options } from '@wdio/types'
+import TestOpsConfig from './testOps/testOpsConfig.js'
+import { isUndefined } from './util.js'
+
+class BrowserStackConfig {
+    static getInstance(options?: BrowserstackConfig & Options.Testrunner, config?: Options.Testrunner): BrowserStackConfig {
+        if (!this._instance && options && config) {
+            this._instance = new BrowserStackConfig(options, config)
+        }
+        return this._instance
+    }
+
+    public userName?: string
+    public accessKey?: string
+    public framework?: string
+    public buildName?: string
+    public buildIdentifier?: string
+    public testObservability: TestOpsConfig
+    public percy: boolean
+    public accessibility: boolean
+    public app?: string|AppConfig
+    private static _instance: BrowserStackConfig
+    public appAutomate: boolean
+    public automate: boolean
+    public funnelDataSent: boolean = false
+
+    private constructor(options: BrowserstackConfig & Options.Testrunner, config: Options.Testrunner) {
+        this.framework = config.framework
+        this.userName = config.user
+        this.accessKey = config.key
+        this.testObservability = new TestOpsConfig(options.testObservability !== false, !isUndefined(options.testObservability))
+        this.percy = options.percy || false
+        this.accessibility = options.accessibility || false
+        this.app = options.app
+        this.appAutomate = !isUndefined(options.app)
+        this.automate = !this.appAutomate
+        this.buildIdentifier = options.buildIdentifier
+    }
+
+    sentFunnelData() {
+        this.funnelDataSent = true
+    }
+
+}
+
+export default BrowserStackConfig

packages/wdio-browserstack-service/src/constants.ts

@@ -60,3 +60,35 @@ export const PERCY_DOM_CHANGING_COMMANDS_ENDPOINTS = [
 ]
 
 export const CAPTURE_MODES = ['click', 'auto', 'screenshot', 'manual', 'testcase']
+
+export const LOG_KIND_USAGE_MAP = {
+    'TEST_LOG': 'log',
+    'TEST_SCREENSHOT': 'screenshot',
+    'TEST_STEP': 'step',
+    'HTTP': 'http'
+}
+
+export const FUNNEL_INSTRUMENTATION_URL = 'https://api.browserstack.com/sdk/v1/event'
+
+// Env variables - Define all the env variable constants over here
+
+// To store the JWT token returned the session launch
+export const TESTOPS_JWT_ENV = 'BS_TESTOPS_JWT'
+
+// To store the setting of whether to send screenshots or not
+export const TESTOPS_SCREENSHOT_ENV = 'BS_TESTOPS_ALLOW_SCREENSHOTS'
+
+// To store build hashed id
+export const TESTOPS_BUILD_ID_ENV = 'BS_TESTOPS_BUILD_HASHED_ID'
+
+// Whether to collect performance instrumentation or not
+export const PERF_MEASUREMENT_ENV = 'BROWSERSTACK_O11Y_PERF_MEASUREMENT'
+
+// Whether the current run is rerun or not
+export const RERUN_TESTS_ENV = 'BROWSERSTACK_RERUN_TESTS'
+
+// The tests that needs to be rerun
+export const RERUN_ENV = 'BROWSERSTACK_RERUN'
+
+// To store whether the build launch has completed or not
+export const TESTOPS_BUILD_COMPLETED_ENV = 'BS_TESTOPS_BUILD_COMPLETED'

packages/wdio-browserstack-service/src/crash-reporter.ts

@@ -1,7 +1,7 @@
 import type { Capabilities, Options } from '@wdio/types'
 import got from 'got'
 
-import { BSTACK_SERVICE_VERSION, DATA_ENDPOINT } from './constants.js'
+import { BSTACK_SERVICE_VERSION, DATA_ENDPOINT, TESTOPS_BUILD_ID_ENV } from './constants.js'
 import type { BrowserstackConfig, CredentialsForCrashReportUpload, UserConfigforReporting } from './types.js'
 import { DEFAULT_REQUEST_CONFIG, getObservabilityKey, getObservabilityUser } from './util.js'
 import { BStackLogger } from './bstackLogger.js'
@@ -61,7 +61,7 @@ export default class CrashReporter {
         }
 
         const data = {
-            hashed_id: process.env.BS_TESTOPS_BUILD_HASHED_ID,
+            hashed_id: process.env[TESTOPS_BUILD_ID_ENV],
             observability_version: {
                 frameworkName: 'WebdriverIO-' + (this.userConfigForReporting.framework || 'null'),
                 sdkVersion: BSTACK_SERVICE_VERSION

packages/wdio-browserstack-service/src/data-store.ts

@@ -0,0 +1,51 @@
+import path from 'node:path'
+import fs from 'node:fs'
+
+import { BStackLogger } from './bstackLogger.js'
+
+const workersDataDirPath = path.join(process.cwd(), 'logs', 'worker_data')
+
+export function getDataFromWorkers(){
+    const workersData: Record<string, object>[] = []
+
+    if (!fs.existsSync(workersDataDirPath)) {
+        return workersData
+    }
+
+    const files = fs.readdirSync(workersDataDirPath)
+    files.forEach((file) => {
+        BStackLogger.debug('Reading worker file ' + file)
+        const filePath = path.join(workersDataDirPath, file)
+        const fileContent = fs.readFileSync(filePath, 'utf8')
+        const workerData = JSON.parse(fileContent)
+        workersData.push(workerData)
+    })
+
+    // Remove worker data after all reading
+    removeWorkersDataDir()
+
+    return workersData
+}
+
+export function saveWorkerData(data: Record<string, any>) {
+    const filePath = path.join(workersDataDirPath, 'worker-data-' + process.pid + '.json')
+
+    try {
+        createWorkersDataDir()
+        fs.writeFileSync(filePath, JSON.stringify(data))
+    } catch (e) {
+        BStackLogger.debug('Exception in saving worker data: ' + e)
+    }
+}
+
+export function removeWorkersDataDir(): boolean {
+    fs.rmSync(workersDataDirPath, { recursive: true, force: true })
+    return true
+}
+
+function createWorkersDataDir() {
+    if (!fs.existsSync(workersDataDirPath)) {
+        fs.mkdirSync(workersDataDirPath, { recursive: true })
+    }
+    return true
+}

packages/wdio-browserstack-service/src/exitHandler.ts

@@ -0,0 +1,37 @@
+import { spawn } from 'node:child_process'
+import path from 'node:path'
+import BrowserStackConfig from './config.js'
+import { saveFunnelData } from './instrumentation/funnelInstrumentation.js'
+import { fileURLToPath } from 'node:url'
+import { TESTOPS_JWT_ENV } from './constants.js'
+import { BStackLogger } from './bstackLogger.js'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+export function setupExitHandlers() {
+    process.on('exit', (code) => {
+        BStackLogger.debug('Exit hook called')
+        const args = shouldCallCleanup(BrowserStackConfig.getInstance())
+        if (Array.isArray(args) && args.length) {
+            BStackLogger.debug('Spawning cleanup with args ' + args.toString())
+            const childProcess = spawn('node', [`${path.join(__dirname, 'cleanup.js')}`, ...args], { detached: true, stdio: 'inherit', env: { ...process.env } })
+            childProcess.unref()
+            process.exit(code)
+        }
+    })
+}
+
+export function shouldCallCleanup(config: BrowserStackConfig): string[] {
+    const args: string[] = []
+    if (!!process.env[TESTOPS_JWT_ENV] && !config.testObservability.buildStopped) {
+        args.push('--observability')
+    }
+
+    if (config.userName && config.accessKey && !config.funnelDataSent) {
+        const savedFilePath = saveFunnelData('SDKTestSuccessful', config)
+        args.push('--funnelData', savedFilePath)
+    }
+
+    return args
+}