Backport cluster-wide prom-client metrics aggregation (plus undefined-handler-label fix) to 6.x

CHANGELOG.md

@@ -7,6 +7,31 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [6.52.0]
+### Added
+- Cluster-wide Prometheus `/metrics` aggregation. Each cluster worker keeps its own
+  prom-client registry, so serving `/metrics` from a single round-robin selected
+  worker exposed only that worker's counters; Prometheus read the per-scrape braid
+  of independent monotonic counters as counter resets and inflated `rate()` /
+  `increase()` on `runtime_http_*` counters by orders of magnitude. In multi-worker
+  mode the worker answering a scrape now asks the master for a merged, monotonic
+  view built from every worker's registry over the existing cluster IPC (via
+  prom-client's `AggregatorRegistry`), with a bounded timeout and a local-registry
+  fallback. Single-worker mode (`workers === 1`, includes `LINKED`) is unchanged.
+  Backport of #667 to the 6.x line.
+### Fixed
+- `runtime_http_*` metrics no longer emit samples without a `handler` label. Requests
+  that never reach a named handler (unmatched paths, replica-level rate limit
+  rejections, errors before the route pipeline) were counted with
+  `handler: undefined`; Node's cluster IPC serializes worker registries as JSON, which
+  drops `undefined` values, so the aggregated `/metrics` exposed a second, unnamed
+  series that Prometheus reads as `handler=""`. Those requests are now labelled
+  `handler="undefined"` — the same value prom-client rendered locally before cluster
+  aggregation — keeping dashboards and alerts that filter on it working. Backport of
+  #673 to the 6.x line.
+- `/_status` requests are now reported as `handler="builtin:status-track"`, matching
+  the other builtin handlers, instead of falling into the unnamed bucket.
+
 ## [6.51.0] - 2026-06-23
 ### Added
 - Base `IOClients` getter `janusCatalogSystem` (Janus Catalog) and

jest.config.js

@@ -1,4 +1,17 @@
 module.exports = {
+  moduleNameMapper: {
+    // jest@25's resolver predates the package "exports" field, so it cannot load the
+    // modern @vtex/diagnostics-nodejs + OpenTelemetry logger chain that src/service/logger
+    // pulls in at module-eval time (reached transitively by nearly every service module).
+    // Stub the package so any suite importing the logger chain can load under the 6.x
+    // toolchain; the real telemetry/log-client paths are lazy and error-guarded, so this
+    // has no behavioural effect on the code under test. See jest/stubs/diagnosticsNodejs.js.
+    '^@vtex/diagnostics-nodejs$': '<rootDir>/jest/stubs/diagnosticsNodejs.js',
+    // Belt-and-braces for the same chain if it is reached directly rather than through
+    // the stub above.
+    '^@opentelemetry/otlp-exporter-base/node-http$':
+      '<rootDir>/node_modules/@opentelemetry/otlp-exporter-base/build/src/index-node-http.js',
+  },
   roots: ['<rootDir>/src'],
   transform: {
     '^.+\\.tsx?$': 'ts-jest',

jest/stubs/diagnosticsNodejs.js

@@ -0,0 +1,24 @@
+// Test stub for `@vtex/diagnostics-nodejs`.
+//
+// The 6.x toolchain pins `jest@25`, whose module resolver predates the package
+// `exports` field. `@vtex/diagnostics-nodejs` pulls in modern OpenTelemetry
+// exporter packages that expose their entry points only through `exports`
+// subpaths (e.g. `@opentelemetry/otlp-exporter-base/node-http`), which jest@25
+// cannot resolve. Because `src/service/logger` (imported transitively by nearly
+// every service module) loads that chain at module-evaluation time, any test
+// touching a service module fails to even load under jest@25.
+//
+// The real telemetry / log-client paths are lazy and error-guarded (see
+// `src/service/logger/logger.ts` and `src/service/telemetry/client.ts`), so
+// tests only need the named exports to exist for module evaluation. This stub
+// provides just enough surface for that, with no behavioural effect on the code
+// under test.
+module.exports = {
+  Exporters: {
+    CreateExporter: () => ({ initialize: async () => undefined }),
+    CreateLogsExporterConfig: () => ({}),
+  },
+  NewTelemetryClient: async () => ({
+    newLogsClient: async () => ({}),
+  }),
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vtex/api",
-  "version": "6.51.0",
+  "version": "6.52.0",
   "description": "VTEX I/O API client",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",

src/service/__tests__/master.test.ts

@@ -0,0 +1,30 @@
+import { Worker } from 'cluster'
+
+import { onMessage } from '../master'
+import * as aggregator from '../metrics/clusterMetricsAggregator'
+import { AGG_METRICS_REQ } from '../metrics/clusterMetricsAggregator'
+
+describe('master onMessage', () => {
+  const worker = { process: { pid: 123 }, send: jest.fn() } as unknown as Worker
+
+  afterEach(() => {
+    jest.restoreAllMocks()
+  })
+
+  it('routes aggregate metric requests from workers to the aggregation handler', () => {
+    const spy = jest.spyOn(aggregator, 'handleWorkerMetricsRequest').mockResolvedValue(undefined)
+    const message = { id: 1, type: AGG_METRICS_REQ }
+
+    onMessage(worker, message)
+
+    expect(spy).toHaveBeenCalledTimes(1)
+    expect(spy).toHaveBeenCalledWith(worker, message)
+  })
+
+  it('ignores prom-client cluster protocol messages without invoking the handler', () => {
+    const spy = jest.spyOn(aggregator, 'handleWorkerMetricsRequest')
+
+    expect(() => onMessage(worker, { type: 'prom-client:getMetricsReq' })).not.toThrow()
+    expect(spy).not.toHaveBeenCalled()
+  })
+})

src/service/master.ts

@@ -3,19 +3,29 @@ import { constants } from 'os'
 
 import { INSPECT_DEBUGGER_PORT, LINKED, UP_SIGNAL } from '../constants'
 import { isLog, logOnceToDevConsole } from './logger'
+import {
+  handleWorkerMetricsRequest,
+  initMasterAggregatorRegistry,
+  isAggMetricsRequest,
+  isPromClientMessage,
+} from './metrics/clusterMetricsAggregator'
 import { logger } from './worker/listeners'
 import { broadcastStatusTrack, isStatusTrackBroadcast, trackStatus } from './worker/runtime/statusTrack'
 import { ServiceJSON } from './worker/runtime/typings'
 
 let handledSignal: NodeJS.Signals | undefined
 
-const onMessage = (worker: Worker, message: any) => {
+export const onMessage = (worker: Worker, message: any) => {
   if (isLog(message)) {
     logOnceToDevConsole(message.message, message.level)
   } else if (isStatusTrackBroadcast(message)) {
     trackStatus()
     broadcastStatusTrack()
-  } else {
+  } else if (isAggMetricsRequest(message)) {
+    handleWorkerMetricsRequest(worker, message)
+  } else if (!isPromClientMessage(message)) {
+    // prom-client's own cluster messages are handled by its cluster listener;
+    // anything else that reaches here is genuinely unexpected.
     logger.warn({
       content: message,
       message: 'Worker sent message',
@@ -77,6 +87,12 @@ export const startMaster = (service: ServiceJSON) => {
     process.env.DETERMINISTIC_VARY = 'true'
   }
 
+  // Set up the master-side Prometheus aggregator so workers can request a
+  // merged, monotonic /metrics view across the whole cluster over IPC.
+  if (numWorkers > 1) {
+    initMasterAggregatorRegistry()
+  }
+
   // Setup dubugger
   if (LINKED) {
     cluster.setupMaster({ inspectPort: INSPECT_DEBUGGER_PORT })