Skip to content

Commit 36ea230

Browse files
EbonsignoriEbonsignori
authored
Add Datadog log forwarding with pod identity (#60347)
1 parent 8834d21 commit 36ea230

7 files changed

Lines changed: 140 additions & 1 deletion

File tree

‎config/kubernetes/default/deployments/webapp.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ spec:
1616
# See https://thehub.github.com/epd/engineering/dev-practicals/observability/logging/ for more details
1717
fluentbit.io/parser: logfmt_sloppy
1818
observability.github.com/splunk_index: docs-internal
19+
ad.datadoghq.com/webapp.logs: '[{"source":"nodejs","service":"docs-internal","tags":["env:staging"]}]'
20+
ad.datadoghq.com/tolerate-unready: 'true'
1921
spec:
2022
dnsPolicy: Default
2123
containers:
@@ -47,6 +49,15 @@ spec:
4749
# configuration set in config/moda/configuration/*/env.yaml
4850
- configMapRef:
4951
name: application-config
52+
env:
53+
- name: POD_NAME
54+
valueFrom:
55+
fieldRef:
56+
fieldPath: metadata.name
57+
- name: POD_NAMESPACE
58+
valueFrom:
59+
fieldRef:
60+
fieldPath: metadata.namespace
5061
# Zero-downtime deploys
5162
# https://thehub.github.com/engineering/products-and-services/internal/moda/feature-documentation/pod-lifecycle/#required-prestop-hook
5263
# https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks

‎config/kubernetes/production/deployments/webapp.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ spec:
1818
# See https://thehub.github.com/epd/engineering/dev-practicals/observability/logging/ for more details
1919
fluentbit.io/parser: logfmt_sloppy
2020
observability.github.com/splunk_index: docs-internal
21+
ad.datadoghq.com/webapp.logs: '[{"source":"nodejs","service":"docs-internal","tags":["env:production"]}]'
22+
ad.datadoghq.com/tolerate-unready: 'true'
2123
spec:
2224
dnsPolicy: Default
2325
containers:
@@ -50,6 +52,15 @@ spec:
5052
# configuration set in config/moda/configuration/*/env.yaml
5153
- configMapRef:
5254
name: application-config
55+
env:
56+
- name: POD_NAME
57+
valueFrom:
58+
fieldRef:
59+
fieldPath: metadata.name
60+
- name: POD_NAMESPACE
61+
valueFrom:
62+
fieldRef:
63+
fieldPath: metadata.namespace
5364
# Zero-downtime deploys
5465
# https://thehub.github.com/engineering/products-and-services/internal/moda/feature-documentation/pod-lifecycle/#required-prestop-hook
5566
# https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks

‎src/observability/logger/index.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
useProductionLogging,
77
} from '@/observability/logger/lib/log-levels'
88
import { toLogfmt } from '@/observability/logger/lib/to-logfmt'
9+
import { POD_IDENTITY } from '@/observability/logger/lib/pod-identity'
910

1011
type IncludeContext = { [key: string]: unknown }
1112

@@ -131,7 +132,8 @@ export function createLogger(filePath: string) {
131132
if (useProductionLogging()) {
132133
// Logfmt logging in production
133134
const logObject: IncludeContext = {
134-
...loggerContext,
135+
...POD_IDENTITY, // pod_name, pod_namespace, node_hostname (static; {} in local dev)
136+
...loggerContext, // requestUuid, path, method, headers, etc. (per-request)
135137
timestamp,
136138
level,
137139
file: path.relative(process.cwd(), new URL(filePath).pathname),

‎src/observability/logger/lib/pod-identity.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
// Static pod-identity fields — read once at module load, never change.
2+
// Only populated when running in Kubernetes (env vars set via Downward API + kube-cluster-metadata).
3+
export const POD_IDENTITY: Record<string, string> = {}
4+
if (process.env.POD_NAME) POD_IDENTITY.podName = process.env.POD_NAME
5+
if (process.env.POD_NAMESPACE) POD_IDENTITY.podNamespace = process.env.POD_NAMESPACE
6+
if (process.env.KUBE_NODE_HOSTNAME) POD_IDENTITY.nodeHostname = process.env.KUBE_NODE_HOSTNAME

‎src/observability/logger/middleware/get-automatic-request-logger.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { getLoggerContext } from '@/observability/logger/lib/logger-context'
33
import type { NextFunction, Request, Response } from 'express'
44
import { getLogLevelNumber, useProductionLogging } from '@/observability/logger/lib/log-levels'
55
import { toLogfmt } from '@/observability/logger/lib/to-logfmt'
6+
import { POD_IDENTITY } from '@/observability/logger/lib/pod-identity'
67

78
/**
89
* Check if automatic development logging is enabled.
@@ -45,6 +46,7 @@ export function getAutomaticRequestLogger() {
4546
const loggerContext = getLoggerContext()
4647
console.log(
4748
toLogfmt({
49+
...POD_IDENTITY,
4850
...loggerContext,
4951
status,
5052
responseTime: `${responseTime} ms`,

‎src/observability/tests/get-automatic-request-logger.ts‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,4 +394,69 @@ describe('getAutomaticRequestLogger', () => {
394394
}
395395
})
396396
})
397+
398+
describe('pod identity fields in production logs', () => {
399+
// Helper to build a minimal mock res/req and trigger res.end
400+
async function runMiddlewareAndCapture(
401+
middleware: ReturnType<
402+
typeof import('@/observability/logger/middleware/get-automatic-request-logger').getAutomaticRequestLogger
403+
>,
404+
): Promise<string[]> {
405+
const logs: string[] = []
406+
const savedLog = console.log
407+
console.log = vi.fn((msg: string) => logs.push(msg))
408+
409+
const req = { method: 'GET', url: '/test', originalUrl: '/test' }
410+
const originalEnd = vi.fn()
411+
const res = {
412+
statusCode: 200,
413+
getHeader: vi.fn(() => '0'),
414+
end: originalEnd,
415+
}
416+
const next = vi.fn()
417+
418+
middleware(req as Request, res as unknown as Response, next)
419+
;(res as unknown as { end: () => void }).end()
420+
await new Promise((resolve) => setTimeout(resolve, 20))
421+
422+
console.log = savedLog
423+
return logs
424+
}
425+
426+
it('should include podName, podNamespace, nodeHostname in logfmt output when env vars are set', async () => {
427+
vi.stubEnv('POD_NAME', 'webapp-abc123')
428+
vi.stubEnv('POD_NAMESPACE', 'docs-internal-staging-cedar')
429+
vi.stubEnv('KUBE_NODE_HOSTNAME', 'ghe-k8s-node-42')
430+
vi.stubEnv('LOG_LIKE_PRODUCTION', 'true')
431+
vi.stubEnv('NODE_ENV', 'production')
432+
433+
vi.resetModules()
434+
const { getAutomaticRequestLogger: freshGetMiddleware } =
435+
await import('@/observability/logger/middleware/get-automatic-request-logger')
436+
437+
const logs = await runMiddlewareAndCapture(freshGetMiddleware())
438+
expect(logs).toHaveLength(1)
439+
expect(logs[0]).toContain('podName=webapp-abc123')
440+
expect(logs[0]).toContain('podNamespace=docs-internal-staging-cedar')
441+
expect(logs[0]).toContain('nodeHostname=ghe-k8s-node-42')
442+
})
443+
444+
it('should not include pod identity fields when env vars are absent', async () => {
445+
vi.stubEnv('LOG_LIKE_PRODUCTION', 'true')
446+
vi.stubEnv('NODE_ENV', 'production')
447+
delete process.env.POD_NAME
448+
delete process.env.POD_NAMESPACE
449+
delete process.env.KUBE_NODE_HOSTNAME
450+
451+
vi.resetModules()
452+
const { getAutomaticRequestLogger: freshGetMiddleware } =
453+
await import('@/observability/logger/middleware/get-automatic-request-logger')
454+
455+
const logs = await runMiddlewareAndCapture(freshGetMiddleware())
456+
expect(logs).toHaveLength(1)
457+
expect(logs[0]).not.toContain('podName=')
458+
expect(logs[0]).not.toContain('podNamespace=')
459+
expect(logs[0]).not.toContain('nodeHostname=')
460+
})
461+
})
397462
})

‎src/observability/tests/logger.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,4 +389,46 @@ describe('createLogger', () => {
389389
expect(logOutput).toContain('included.key=value')
390390
})
391391
})
392+
393+
describe('pod identity fields in production logs', () => {
394+
it('should include podName, podNamespace, nodeHostname in logfmt output when env vars are set', async () => {
395+
vi.stubEnv('POD_NAME', 'webapp-abc123')
396+
vi.stubEnv('POD_NAMESPACE', 'docs-internal-staging-cedar')
397+
vi.stubEnv('KUBE_NODE_HOSTNAME', 'ghe-k8s-node-42')
398+
vi.stubEnv('LOG_LIKE_PRODUCTION', 'true')
399+
400+
// Reset modules so pod-identity is re-evaluated with the new env vars
401+
vi.resetModules()
402+
const { createLogger: freshCreateLogger } = await import('@/observability/logger')
403+
404+
const logger = freshCreateLogger('file:///path/to/test.js')
405+
logger.info('Pod identity test')
406+
407+
expect(consoleLogs).toHaveLength(1)
408+
const logOutput = consoleLogs[0]
409+
expect(logOutput).toContain('podName=webapp-abc123')
410+
expect(logOutput).toContain('podNamespace=docs-internal-staging-cedar')
411+
expect(logOutput).toContain('nodeHostname=ghe-k8s-node-42')
412+
})
413+
414+
it('should not include pod identity fields in logfmt output when env vars are absent', async () => {
415+
vi.stubEnv('LOG_LIKE_PRODUCTION', 'true')
416+
// Ensure pod env vars are absent
417+
delete process.env.POD_NAME
418+
delete process.env.POD_NAMESPACE
419+
delete process.env.KUBE_NODE_HOSTNAME
420+
421+
vi.resetModules()
422+
const { createLogger: freshCreateLogger } = await import('@/observability/logger')
423+
424+
const logger = freshCreateLogger('file:///path/to/test.js')
425+
logger.info('No pod identity test')
426+
427+
expect(consoleLogs).toHaveLength(1)
428+
const logOutput = consoleLogs[0]
429+
expect(logOutput).not.toContain('podName=')
430+
expect(logOutput).not.toContain('podNamespace=')
431+
expect(logOutput).not.toContain('nodeHostname=')
432+
})
433+
})
392434
})

0 commit comments

Comments
 (0)