👷 add api-performance-benchmark scheduled job + rename test-performance to bundle-size

.gitlab-ci.yml

@@ -188,7 +188,7 @@ build-and-lint:
     - yarn lint
     - node scripts/check-packages.ts
 
-test-performance:
+bundle-size:
   extends:
     - .base-configuration
     - .test-allowed-branches
@@ -609,6 +609,17 @@ performance-benchmark:
     - yarn
     - yarn test:performance
 
+api-performance-benchmark:
+  stage: task
+  extends:
+    - .base-configuration
+  only:
+    variables:
+      - $TARGET_TASK_NAME == "performance-benchmark-scheduled"
+  script:
+    - yarn
+    - node ./scripts/api-performance/index.ts
+
 ########################################################################################################################
 # Check expired telemetry
 ########################################################################################################################

scripts/api-performance/index.ts

@@ -0,0 +1,11 @@
+import { printLog, runMain } from '../lib/executionUtils.ts'
+import { runCpuPerformanceTest } from './lib/cpuPerformance.ts'
+import { runMemoryPerformanceTest } from './lib/memoryPerformance.ts'
+
+runMain(async () => {
+  printLog('CPU performance...')
+  await runCpuPerformanceTest()
+
+  printLog('Memory performance...')
+  await runMemoryPerformanceTest()
+})

scripts/api-performance/lib/cpuPerformance.ts

@@ -0,0 +1,105 @@
+import { fetchHandlingError, timeout } from '../../lib/executionUtils.ts'
+import { getOrg2ApiKey, getOrg2AppKey } from '../../lib/secrets.ts'
+import { TESTS } from './constants.ts'
+
+// The synthetic test itself reports per-API CPU metrics to Datadog (one metric per
+// button on the playground page). This script triggers the run, waits for completion,
+// then queries the just-reported metrics back so we can surface them in the CI log.
+const API_KEY = getOrg2ApiKey()
+const APP_KEY = getOrg2AppKey()
+const TIMEOUT_IN_MS = 15000
+const TEST_PUBLIC_ID = 'vcg-7rk-5av'
+const RETRIES_NUMBER = 6
+const ONE_DAY_IN_SECOND = 24 * 60 * 60
+
+interface SyntheticsTestResult {
+  results: Array<{ result_id: string }>
+}
+
+interface SyntheticsTestStatus {
+  length: number
+  status?: number
+}
+
+interface DatadogResponse {
+  series?: Array<{ pointlist?: Array<[number, number]> }>
+}
+
+export async function runCpuPerformanceTest(): Promise<void> {
+  const commitSha = process.env.CI_COMMIT_SHORT_SHA || ''
+  const resultId = await triggerSyntheticsTest(commitSha)
+  await waitForSyntheticsTestToFinish(resultId, RETRIES_NUMBER)
+
+  const rows = await Promise.all(
+    TESTS.map(async (test) => {
+      const value = await fetchCpuMetric(test.property, commitSha)
+      return { 'Action Name': test.name, 'CPU Time (ms)': value ?? 'N/A' }
+    })
+  )
+  console.log('CPU Performance:')
+  console.table(rows)
+}
+
+async function fetchCpuMetric(name: string, commitId: string): Promise<number | undefined> {
+  const now = Math.floor(Date.now() / 1000)
+  const from = now - 30 * ONE_DAY_IN_SECOND
+  const query = `avg:cpu.sdk.${name}.performance.average{commitid:${commitId}}&from=${from}&to=${now}`
+  const response = await fetchHandlingError(`https://api.datadoghq.com/api/v1/query?query=${query}`, {
+    method: 'GET',
+    headers: {
+      'Content-Type': 'application/json',
+      'DD-API-KEY': API_KEY,
+      'DD-APPLICATION-KEY': APP_KEY,
+    },
+  })
+  const data = (await response.json()) as DatadogResponse
+  const point = data.series?.[0]?.pointlist?.[0]
+  return point?.[1]
+}
+
+async function triggerSyntheticsTest(commitId: string): Promise<string> {
+  const body = {
+    tests: [
+      {
+        public_id: TEST_PUBLIC_ID,
+        startUrl: `https://datadoghq.dev/browser-sdk-test-playground/performance/cpu?commitId=${commitId}`,
+      },
+    ],
+  }
+  const response = await fetchHandlingError('https://api.datadoghq.com/api/v1/synthetics/tests/trigger/ci', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'DD-API-KEY': API_KEY,
+      'DD-APPLICATION-KEY': APP_KEY,
+    },
+    body: JSON.stringify(body),
+  })
+  const data = (await response.json()) as SyntheticsTestResult
+  return data.results[0].result_id
+}
+
+async function waitForSyntheticsTestToFinish(resultId: string, retriesNumber: number): Promise<void> {
+  const url = `https://api.datadoghq.com/api/v1/synthetics/tests/${TEST_PUBLIC_ID}/results/${resultId}`
+  for (let i = 0; i < retriesNumber; i++) {
+    const response = await fetch(url, {
+      method: 'GET',
+      headers: {
+        'Content-Type': 'application/json',
+        'DD-API-KEY': API_KEY,
+        'DD-APPLICATION-KEY': APP_KEY,
+      },
+    })
+    // do not use response.ok as we can have 404 responses
+    if (response.status >= 500) {
+      throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`)
+    }
+    const data = (await response.json()) as SyntheticsTestStatus
+    if (data.length !== 0 && data.status === 0) {
+      await timeout(TIMEOUT_IN_MS) // Wait for logs ingestion
+      return
+    }
+    await timeout(TIMEOUT_IN_MS)
+  }
+  throw new Error('Synthetics test did not finish within the specified number of retries')
+}

scripts/api-performance/lib/memoryPerformance.ts

@@ -0,0 +1,128 @@
+import type { Browser, CDPSession, Page, Protocol } from 'puppeteer'
+import { launch } from 'puppeteer'
+import { formatSize, printLog } from '../../lib/executionUtils.ts'
+import { reportToDatadog } from '../../lib/reportToDatadog.ts'
+import type { Test } from './constants.ts'
+import { TESTS } from './constants.ts'
+
+const NUMBER_OF_RUNS = 30 // Rule of thumb: this should be enough to get a good average
+const BATCH_SIZE = 2
+
+interface MemoryResult {
+  name: string
+  value: number
+}
+
+export async function runMemoryPerformanceTest(): Promise<void> {
+  const results = await computeMemoryPerformance()
+
+  console.log('Memory Performance:')
+  console.table(
+    results.map(({ name, value }) => ({
+      'Action Name': TESTS.find((test) => test.property === name)?.name ?? name,
+      'Memory Consumption': formatSize(value),
+    }))
+  )
+
+  await reportToDatadog({
+    message: 'Browser SDK memory consumption',
+    ...Object.fromEntries(results.map(({ name, value }) => [name, { memory_bytes: value }])),
+  })
+}
+
+async function computeMemoryPerformance(): Promise<MemoryResult[]> {
+  const results: MemoryResult[] = []
+  const benchmarkUrl = 'https://datadoghq.dev/browser-sdk-test-playground/performance/memory'
+
+  for (let i = 0; i < TESTS.length; i += BATCH_SIZE) {
+    await runTests(TESTS.slice(i, i + BATCH_SIZE), benchmarkUrl, (result) => results.push(result))
+  }
+
+  return results
+}
+
+async function runTests(tests: Test[], benchmarkUrl: string, cb: (result: MemoryResult) => void): Promise<void> {
+  await Promise.all(
+    tests.map(async (test) => {
+      const allBytesMeasurements: number[] = []
+      printLog(`Running test for: ${test.button}`)
+      for (let j = 0; j < NUMBER_OF_RUNS; j++) {
+        const bytes = await runTest(test.button, benchmarkUrl)
+        allBytesMeasurements.push(bytes)
+      }
+      const sdkMemoryBytes = average(allBytesMeasurements)
+      printLog(`Average memory used by SDK for ${test.name} over ${NUMBER_OF_RUNS} runs: ${sdkMemoryBytes} bytes`)
+      cb({ name: test.property, value: sdkMemoryBytes })
+    })
+  )
+}
+
+async function runTest(testButton: string, benchmarkUrl: string): Promise<number> {
+  const browser: Browser = await launch({
+    channel: 'chrome',
+    args: ['--no-sandbox', '--disable-setuid-sandbox'],
+  })
+  const page: Page = await browser.newPage()
+  await page.goto(benchmarkUrl)
+
+  // Start the Chrome DevTools Protocol session and enable the heap profiler
+  const client: CDPSession = await page.target().createCDPSession()
+  await client.send('HeapProfiler.enable')
+
+  // Select the button to trigger the test
+  await page.waitForSelector(testButton)
+  const button = await page.$(testButton)
+  if (!button) {
+    throw new Error(`Button ${testButton} not found`)
+  }
+
+  await client.send('HeapProfiler.collectGarbage')
+  await client.send('HeapProfiler.startSampling', { samplingInterval: 50 })
+
+  await button.click()
+  const { profile } = await client.send('HeapProfiler.stopSampling')
+
+  const measurementsBytes: number[] = []
+  const sizeForNodeId = new Map<number, number>()
+
+  for (const sample of profile.samples) {
+    sizeForNodeId.set(sample.nodeId, (sizeForNodeId.get(sample.nodeId) || 0) + sample.size)
+    let sdkConsumption = 0
+    for (const node of children(profile.head)) {
+      const consumption = sizeForNodeId.get(node.id) || 0
+      if (isSdkBundleUrl(node.callFrame.url)) {
+        sdkConsumption += consumption
+      }
+    }
+    measurementsBytes.push(sdkConsumption)
+  }
+
+  const medianBytes = median(measurementsBytes)
+  await browser.close()
+  return medianBytes
+}
+
+function* children(
+  node: Protocol.HeapProfiler.SamplingHeapProfileNode
+): Generator<Protocol.HeapProfiler.SamplingHeapProfileNode> {
+  yield node
+  for (const child of node.children || []) {
+    yield* children(child)
+  }
+}
+
+function isSdkBundleUrl(url: string): boolean {
+  return (
+    url.startsWith('https://www.datad0g-browser-agent.com/') ||
+    url.startsWith('https://www.datadoghq-browser-agent.com/')
+  )
+}
+
+function average(values: number[]): number {
+  return Number((values.reduce((a, b) => a + b, 0) / values.length).toFixed(2))
+}
+
+function median(values: number[]): number {
+  values.sort((a, b) => a - b)
+  return values[Math.floor(values.length / 2)]
+}

scripts/performance/lib/reportToDatadog.ts → scripts/lib/reportToDatadog.ts

@@ -1,6 +1,6 @@
-import { fetchHandlingError } from '../../lib/executionUtils.ts'
-import { getOrg2ApiKey } from '../../lib/secrets.ts'
-import { browserSdkVersion } from '../../lib/browserSdkVersion.ts'
+import { fetchHandlingError } from './executionUtils.ts'
+import { getOrg2ApiKey } from './secrets.ts'
+import { browserSdkVersion } from './browserSdkVersion.ts'
 
 export async function reportToDatadog(
   logData: Record<string, number | string | Record<string, number>>

scripts/performance/index.ts

@@ -1,9 +1,7 @@
 import { printLog, runMain } from '../lib/executionUtils.ts'
 import { fetchPR, LOCAL_BRANCH, getLastCommonCommit } from '../lib/gitUtils.ts'
 import { Pr } from './lib/reportAsAPrComment.ts'
-import { computeAndReportMemoryPerformance } from './lib/memoryPerformance.ts'
 import { computeAndReportBundleSizes } from './lib/bundleSizes.ts'
-import { computeAndReportCpuPerformance } from './lib/cpuPerformance.ts'
 
 runMain(async () => {
   const githubPr = await fetchPR(LOCAL_BRANCH!)
@@ -18,10 +16,4 @@ runMain(async () => {
 
   printLog('Bundle sizes...')
   await computeAndReportBundleSizes(pr)
-
-  printLog('Memory performance...')
-  await computeAndReportMemoryPerformance(pr)
-
-  printLog('CPU performance...')
-  await computeAndReportCpuPerformance(pr)
 })

scripts/performance/lib/bundleSizes.ts

@@ -1,9 +1,9 @@
 import { formatPercentage, formatSize } from '../../lib/executionUtils.ts'
 import { calculateBundleSizes } from '../../lib/computeBundleSize.ts'
+import { reportToDatadog } from '../../lib/reportToDatadog.ts'
 import type { PerformanceMetric } from './fetchPerformanceMetrics.ts'
 import { fetchPerformanceMetrics } from './fetchPerformanceMetrics.ts'
 import { markdownArray, type Pr } from './reportAsAPrComment.ts'
-import { reportToDatadog } from './reportToDatadog.ts'
 
 // The value is set to 5% as it's around 10 times the average value for small PRs.
 const SIZE_INCREASE_THRESHOLD = 5