try to detect external sources without using host (#57033)

Author: Ebonsignori

src/search/components/helpers/execute-search-actions.ts

@@ -49,6 +49,7 @@ export async function executeAISearch(version: string, query: string, debug = fa
   const body = {
     query,
     version,
+    client_name: 'docs.github.com-client',
     ...(debug && { debug: '1' }),
   }
 
@@ -80,6 +81,9 @@ export async function executeCombinedSearch(
     params.set('debug', '1')
   }
 
+  // Add client_name to identify requests from our frontend
+  params.set('client_name', 'docs.github.com-client')
+
   // Always fetch 4 results for autocomplete
   params.set('size', '4')
 

src/search/lib/helpers/external-search-analytics.ts

@@ -12,27 +12,40 @@ export async function handleExternalSearchAnalytics(
   const host = req.headers['x-host'] || req.headers.host
   const normalizedHost = stripPort(host as string)
 
-  // Skip analytics entirely for production and internal staging environments
-  if (
-    normalizedHost === 'docs.github.com' ||
-    normalizedHost.endsWith('.github.net') ||
-    normalizedHost.endsWith('.githubapp.com')
-  ) {
-    return null
-  }
+  // Check if this is likely an external API call rather than a browser request
+  const isLikelyExternalAPI = isExternalAPIRequest(req)
 
-  // For localhost, send analytics but auto-set client_name if not provided
+  // Get client_name from query or body
   let client_name = req.query.client_name || req.body?.client_name
-  if (normalizedHost === 'localhost' && !client_name) {
-    client_name = 'localhost'
+
+  // Rule 1: Skip analytics for browser requests from our own frontend
+  if (!isLikelyExternalAPI && client_name === 'docs.github.com-client') {
+    return null
   }
 
-  // For all other external requests, require explicit client_name
-  if (!client_name) {
-    return {
-      status: 400,
-      error: "Missing required parameter 'client_name' for external requests",
+  // Rule 2: Send analytics for any request with a client_name that's not 'docs.github.com-client'
+  // (This includes partner APIs and other external clients)
+  if (client_name && client_name !== 'docs.github.com-client') {
+    // Analytics will be sent at the end of this function
+  }
+  // Rule 3: For requests without client_name, require it for external API requests
+  else if (!client_name) {
+    if (isLikelyExternalAPI) {
+      return {
+        status: 400,
+        error: "Missing required parameter 'client_name' for external requests",
+      }
     }
+    // For browser requests without client_name to internal environments, skip analytics
+    else if (normalizedHost.endsWith('.github.net') || normalizedHost.endsWith('.githubapp.com')) {
+      return null
+    }
+    // For localhost development without client_name, we'll still send analytics below
+  }
+
+  // For localhost, ensure we have a client_name for analytics
+  if (normalizedHost === 'localhost' && !client_name) {
+    client_name = 'localhost'
   }
 
   // Send search event with client identifier
@@ -71,19 +84,16 @@ export async function handleExternalSearchAnalytics(
 
 /**
  * Determines if a host should bypass client_name requirement for analytics
- * Returns true if the host is docs.github.com or ends with github.net or githubapp.com
- * (for production and internal staging environments)
+ * Returns true if the host ends with github.net or githubapp.com
+ * (for internal staging environments)
+ * Note: docs.github.com is removed since normalizedHost will always be docs.github.com in production
  * Note: localhost is NOT included here as it should send analytics with auto-set client_name
  */
 export function shouldBypassClientNameRequirement(host: string | undefined): boolean {
   if (!host) return false
 
   const normalizedHost = stripPort(host)
-  return (
-    normalizedHost === 'docs.github.com' ||
-    normalizedHost.endsWith('.github.net') ||
-    normalizedHost.endsWith('.githubapp.com')
-  )
+  return normalizedHost.endsWith('.github.net') || normalizedHost.endsWith('.githubapp.com')
 }
 
 /**
@@ -93,3 +103,42 @@ function stripPort(host: string): string {
   const [hostname] = host.split(':')
   return hostname
 }
+
+interface ExternalAPIRequestLike {
+  headers: Record<string, string | undefined>
+}
+
+/**
+ * Determines if a request is likely from an external API client rather than a browser
+ * Uses multiple heuristics to detect programmatic vs browser requests
+ */
+const userAgentRegex = /^(curl|wget|python-requests|axios|node-fetch|Go-http-client|okhttp)/i
+function isExternalAPIRequest(req: ExternalAPIRequestLike): boolean {
+  const headers = req.headers
+
+  // Browser security headers that APIs typically don't send
+  const hasSecFetchHeaders = headers['sec-fetch-site'] || headers['sec-fetch-mode']
+  const hasClientHints = headers['sec-ch-ua'] || headers['sec-ch-ua-mobile']
+
+  // Browsers typically request HTML, APIs typically request JSON
+  const acceptHeader = headers.accept || ''
+  const prefersJson =
+    acceptHeader.includes('application/json') && !acceptHeader.includes('text/html')
+
+  // Common API user agents (not exhaustive, but catches common cases)
+  const userAgent = headers['user-agent'] || ''
+  const hasAPIUserAgent = userAgentRegex.test(userAgent)
+
+  // If it has browser-specific headers, it's likely a browser
+  if (hasSecFetchHeaders || hasClientHints) {
+    return false
+  }
+
+  // If it prefers JSON or has a common API user agent, it's likely an API
+  if (prefersJson || hasAPIUserAgent) {
+    return true
+  }
+
+  // Default to treating it as a browser request to be conservative
+  return false
+}