Add AI agent detection to CLI analytics

packages/cli-kit/package.json

@@ -115,6 +115,7 @@
     "@opentelemetry/exporter-metrics-otlp-http": "0.57.0",
     "@opentelemetry/resources": "1.30.1",
     "@opentelemetry/sdk-metrics": "1.30.1",
+    "@vercel/detect-agent": "1.2.5",
     "ajv": "8.20.0",
     "ansi-escapes": "6.2.1",
     "archiver": "5.3.2",

packages/cli-kit/src/private/node/analytics.test.ts

@@ -0,0 +1,75 @@
+import {getSensitiveEnvironmentData} from './analytics.js'
+import {determineAgent, type KnownAgentNames} from '@vercel/detect-agent'
+import {afterEach, describe, expect, test, vi} from 'vitest'
+
+vi.mock('@vercel/detect-agent')
+vi.mock('../../public/node/output.js')
+
+function agentDetected(name: string) {
+  vi.mocked(determineAgent).mockResolvedValue({isAgent: true, agent: {name: name as KnownAgentNames}})
+}
+
+function noAgentDetected() {
+  vi.mocked(determineAgent).mockResolvedValue({isAgent: false, agent: undefined})
+}
+
+const declaredAgentVariableNames = ['SHOPIFY_CLI_AGENT_INFO', 'SHOPIFY_CLI_AGENT_IDS']
+
+function stubDeclaredAgentVariablesEmpty() {
+  declaredAgentVariableNames.forEach((variableName) => vi.stubEnv(variableName, undefined))
+}
+
+// Only `config.plugins.keys()` is read, so an empty plugin map is enough.
+const config = {plugins: new Map()} as any
+
+async function shopifyEnvironmentVariables() {
+  const {env_shopify_variables: shopifyVariables} = await getSensitiveEnvironmentData(config)
+  return JSON.parse(shopifyVariables)
+}
+
+describe('getSensitiveEnvironmentData', () => {
+  afterEach(() => {
+    vi.unstubAllEnvs()
+  })
+
+  test('adds detected agent variables when explicit attribution is missing', async () => {
+    agentDetected('claude')
+    stubDeclaredAgentVariablesEmpty()
+
+    const got = await shopifyEnvironmentVariables()
+
+    expect(got).toMatchObject({SHOPIFY_CLI_AGENT_INFO: 'n:claude-code', SHOPIFY_CLI_AGENT_DETECTION: 'detected'})
+  })
+
+  test('does not add a detected agent to the payload when only SHOPIFY_CLI_AGENT_INFO is declared', async () => {
+    agentDetected('claude')
+    vi.stubEnv('SHOPIFY_CLI_AGENT_INFO', 'n:cursor|v:2.1.0|p:openai|m:gpt-5')
+    vi.stubEnv('SHOPIFY_CLI_AGENT_IDS', undefined)
+
+    const got = await shopifyEnvironmentVariables()
+
+    expect(got).toMatchObject({
+      SHOPIFY_CLI_AGENT_INFO: 'n:cursor|v:2.1.0|p:openai|m:gpt-5',
+      SHOPIFY_CLI_AGENT_DETECTION: 'skipped',
+    })
+  })
+
+  test('does not fail telemetry if determineAgent throws an error, and reports the failure', async () => {
+    vi.mocked(determineAgent).mockRejectedValue(new Error('EACCES: permission denied'))
+    stubDeclaredAgentVariablesEmpty()
+
+    const got = await shopifyEnvironmentVariables()
+
+    expect(got).not.toHaveProperty('SHOPIFY_CLI_AGENT_INFO')
+    expect(got).toMatchObject({SHOPIFY_CLI_AGENT_DETECTION: 'failed'})
+  })
+
+  test('reports none when no agent is detected', async () => {
+    noAgentDetected()
+    stubDeclaredAgentVariablesEmpty()
+
+    const got = await shopifyEnvironmentVariables()
+
+    expect(got).toMatchObject({SHOPIFY_CLI_AGENT_DETECTION: 'none'})
+  })
+})

packages/cli-kit/src/private/node/analytics.ts

@@ -1,5 +1,6 @@
 import {getLastSeenAuthMethod} from './session.js'
 import {getAutoUpgradeEnabled} from './conf-store.js'
+import {detectedAgentEnvironmentVariables} from './context/agent.js'
 import {hashString} from '../../public/node/crypto.js'
 import {getPackageManager, packageManagerFromUserAgent} from '../../public/node/node-package-manager.js'
 import BaseCommand from '../../public/node/base-command.js'
@@ -115,14 +116,16 @@ export async function getEnvironmentData(config: Interfaces.Config): Promise<Env
 export async function getSensitiveEnvironmentData(config: Interfaces.Config) {
   return {
     env_plugin_installed_all: JSON.stringify(getPluginNames(config)),
-    env_shopify_variables: JSON.stringify(getShopifyEnvironmentVariables()),
+    env_shopify_variables: JSON.stringify(await getShopifyEnvironmentVariables()),
   }
 }
 
-function getShopifyEnvironmentVariables() {
-  return Object.fromEntries(
-    Object.entries(process.env).filter(([key]) => allowedShopifyEnvironmentVariableNames.has(key)),
+async function getShopifyEnvironmentVariables(env: NodeJS.ProcessEnv = process.env) {
+  const declaredVariables = Object.fromEntries(
+    Object.entries(env).filter(([key]) => allowedShopifyEnvironmentVariableNames.has(key)),
   )
+
+  return {...declaredVariables, ...(await detectedAgentEnvironmentVariables(env))}
 }
 
 function getPluginNames(config: Interfaces.Config) {

packages/cli-kit/src/private/node/context/agent.test.ts

@@ -0,0 +1,164 @@
+import {detectedAgentEnvironmentVariables} from './agent.js'
+import {determineAgent, type KnownAgentNames} from '@vercel/detect-agent'
+import {afterEach, describe, expect, test, vi} from 'vitest'
+
+vi.mock('@vercel/detect-agent')
+vi.mock('../../../public/node/output.js')
+
+function agentDetected(name: string) {
+  vi.mocked(determineAgent).mockResolvedValue({isAgent: true, agent: {name: name as KnownAgentNames}})
+}
+
+function noAgentDetected() {
+  vi.mocked(determineAgent).mockResolvedValue({isAgent: false, agent: undefined})
+}
+
+const declaredAgentVariableNames = ['SHOPIFY_CLI_AGENT_INFO', 'SHOPIFY_CLI_AGENT_IDS']
+
+function stubDeclaredAgentVariablesEmpty() {
+  declaredAgentVariableNames.forEach((variableName) => vi.stubEnv(variableName, undefined))
+}
+
+describe('detectedAgentEnvironmentVariables', () => {
+  afterEach(() => {
+    vi.unstubAllEnvs()
+  })
+
+  test('reports skipped when SHOPIFY_CLI_AGENT_INFO was declared, so a guess cannot clobber it', async () => {
+    agentDetected('claude')
+
+    const got = await detectedAgentEnvironmentVariables({
+      SHOPIFY_CLI_AGENT_INFO: 'n:cursor|v:2.1.0|p:openai|m:gpt-5',
+    })
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_DETECTION: 'skipped'})
+  })
+
+  test('reports skipped when SHOPIFY_CLI_AGENT_IDS was declared', async () => {
+    agentDetected('claude')
+
+    const got = await detectedAgentEnvironmentVariables({SHOPIFY_CLI_AGENT_IDS: 's:session-id|r:run-id'})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_DETECTION: 'skipped'})
+  })
+
+  test('leaves a declared SHOPIFY_CLI_AGENT_INFO value untouched', async () => {
+    agentDetected('claude')
+    const env = {SHOPIFY_CLI_AGENT_INFO: 'n:cursor|v:2.1.0'}
+
+    await detectedAgentEnvironmentVariables(env)
+
+    expect(env).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:cursor|v:2.1.0'})
+  })
+
+  test('a legacy declaration does not suppress detection', async () => {
+    agentDetected('devin')
+
+    const got = await detectedAgentEnvironmentVariables({SHOPIFY_CLI_AGENT: 'declared-by-the-producer'})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:devin', SHOPIFY_CLI_AGENT_DETECTION: 'detected'})
+  })
+
+  test.each([
+    ['undefined', undefined],
+    ['empty', ''],
+    ['whitespace-only', '   '],
+  ])('treats a %s declaration as absent and reports the detected agent', async (_description, declaredValue) => {
+    agentDetected('devin')
+
+    const got = await detectedAgentEnvironmentVariables({
+      SHOPIFY_CLI_AGENT_INFO: declaredValue,
+      SHOPIFY_CLI_AGENT_IDS: declaredValue,
+    })
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:devin', SHOPIFY_CLI_AGENT_DETECTION: 'detected'})
+  })
+
+  test('reports the detected agent when nothing was declared', async () => {
+    agentDetected('devin')
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:devin', SHOPIFY_CLI_AGENT_DETECTION: 'detected'})
+  })
+
+  test('reports none when no agent is detected', async () => {
+    noAgentDetected()
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_DETECTION: 'none'})
+  })
+
+  test.each([
+    ['claude', 'claude-code'],
+    ['gemini', 'gemini-cli'],
+  ])('reports the detected name %s using the toolkit name %s', async (detectedName, expectedName) => {
+    agentDetected(detectedName)
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: `n:${expectedName}`, SHOPIFY_CLI_AGENT_DETECTION: 'detected'})
+  })
+
+  test.each([
+    ['a known name needing no translation', 'cursor'],
+    ['an arbitrary AI_AGENT pass-through value', 'claude-code_2-1-267_agent'],
+  ])('reports %s verbatim', async (_description, detectedName) => {
+    agentDetected(detectedName)
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: `n:${detectedName}`, SHOPIFY_CLI_AGENT_DETECTION: 'detected'})
+  })
+
+  test.each([['constructor'], ['toString'], ['__proto__']])(
+    'reports the inherited object property name %s verbatim',
+    async (detectedName) => {
+      agentDetected(detectedName)
+
+      const got = await detectedAgentEnvironmentVariables({})
+
+      expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: `n:${detectedName}`, SHOPIFY_CLI_AGENT_DETECTION: 'detected'})
+    },
+  )
+
+  test('replaces the tag separator in the detected name so it cannot inject tags', async () => {
+    agentDetected('devin|v:9.9.9')
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:devin_v:9.9.9', SHOPIFY_CLI_AGENT_DETECTION: 'detected'})
+  })
+
+  test.each([
+    ['whitespace-only', '   '],
+    ['nothing but the tag separator', '|'],
+    ['nothing but tag separators', '||'],
+  ])('reports unusable_name when the detected name is %s', async (_description, detectedName) => {
+    agentDetected(detectedName)
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_DETECTION: 'unusable_name'})
+  })
+
+  test('reports failed when detection fails', async () => {
+    vi.mocked(determineAgent).mockRejectedValue(new Error('EACCES: permission denied, stat /opt/.devin'))
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_DETECTION: 'failed'})
+  })
+
+  test('does not mutate process.env when reporting a detected agent', async () => {
+    agentDetected('claude')
+    stubDeclaredAgentVariablesEmpty()
+    const environmentBefore = {...process.env}
+
+    const got = await detectedAgentEnvironmentVariables()
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:claude-code', SHOPIFY_CLI_AGENT_DETECTION: 'detected'})
+    expect({...process.env}).toEqual(environmentBefore)
+  })
+})

packages/cli-kit/src/private/node/context/agent.ts

@@ -0,0 +1,53 @@
+import {outputDebug} from '../../../public/node/output.js'
+import {determineAgent, KNOWN_AGENTS} from '@vercel/detect-agent'
+
+const toolkitAgentNamesByDetectedName = new Map<string, string>([
+  [KNOWN_AGENTS.CLAUDE, 'claude-code'],
+  [KNOWN_AGENTS.GEMINI, 'gemini-cli'],
+])
+
+const explicitAgentVariableNames = ['SHOPIFY_CLI_AGENT_INFO', 'SHOPIFY_CLI_AGENT_IDS']
+
+function hasExplicitAgentAttribution(env: NodeJS.ProcessEnv): boolean {
+  return explicitAgentVariableNames.some((variableName) => (env[variableName] ?? '').trim() !== '')
+}
+
+type AgentDetectionState = 'skipped' | 'detected' | 'none' | 'unusable_name' | 'failed'
+
+function agentDetectionResult(state: AgentDetectionState, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv {
+  return {...extra, SHOPIFY_CLI_AGENT_DETECTION: state}
+}
+
+export async function detectedAgentEnvironmentVariables(
+  env: NodeJS.ProcessEnv = process.env,
+): Promise<NodeJS.ProcessEnv> {
+  // Emitting while a declaration exists would clobber the producer's whole packed value, not just the name.
+  if (hasExplicitAgentAttribution(env)) return agentDetectionResult('skipped')
+
+  try {
+    const detection = await determineAgent()
+    if (!detection.isAgent) return agentDetectionResult('none')
+
+    const rawDetectedName = detection.agent.name
+    const nameHasContentBeyondSeparators = /[^|\s]/.test(rawDetectedName)
+    // The name detection produced was real but unusable (nothing but separators/whitespace), which
+    // is different from no agent being detected at all, so it gets its own state.
+    if (!nameHasContentBeyondSeparators) return agentDetectionResult('unusable_name')
+
+    // `|` separates tags, so a name containing one could otherwise fabricate tags we never detected.
+    const detectedName = rawDetectedName.replaceAll('|', '_').trim()
+
+    return agentDetectionResult('detected', {
+      SHOPIFY_CLI_AGENT_INFO: `n:${toolkitAgentNamesByDetectedName.get(detectedName) ?? detectedName}`,
+    })
+
+    // eslint-disable-next-line no-catch-all/no-catch-all
+  } catch (error) {
+    let message = 'Unable to detect which AI agent is running the CLI'
+    if (error instanceof Error) {
+      message = message.concat(`: ${error.message}`)
+    }
+    outputDebug(message)
+    return agentDetectionResult('failed')
+  }
+}