diff --git a/src/health/controllers/payment-provider-health.controller.ts b/src/health/controllers/payment-provider-health.controller.ts new file mode 100644 index 00000000..9dcf00c9 --- /dev/null +++ b/src/health/controllers/payment-provider-health.controller.ts @@ -0,0 +1,58 @@ +import { Controller, Get, HttpCode, HttpStatus } from '@nestjs/common'; +import { ApiTags, ApiOperation, ApiResponse } from '@nestjs/swagger'; +import { HealthIndicatorsService, PaymentProviderHealthResult } from '../health-indicators.service'; + +/** + * Exposes /health/payment-provider as a dedicated sub-check so that + * monitoring systems (e.g. Prometheus, Grafana, uptime robots) can alert on + * the payment-gateway circuit state independently of the main health endpoint. + */ +@ApiTags('Health') +@Controller('health/payment-provider') +export class PaymentProviderHealthController { + constructor(private readonly healthIndicators: HealthIndicatorsService) {} + + @Get() + @ApiOperation({ + summary: 'Payment provider circuit-breaker health', + description: + 'Returns the current circuit-breaker state for the payment provider. ' + + 'HTTP 200 when circuit is CLOSED (up), HTTP 503 when OPEN (down), ' + + 'HTTP 207 when HALF_OPEN (degraded).', + }) + @ApiResponse({ status: 200, description: 'Payment provider is healthy (circuit CLOSED)' }) + @ApiResponse({ status: 207, description: 'Payment provider is degraded (circuit HALF_OPEN)' }) + @ApiResponse({ status: 503, description: 'Payment provider is unavailable (circuit OPEN)' }) + getPaymentProviderHealth(): PaymentProviderHealthResult { + const result = this.healthIndicators.checkPaymentProvider(); + + // Response status reflects circuit state so load-balancers / uptime monitors + // can act on it without parsing the body. + if (result.status === 'down') { + // NestJS does not allow dynamic status codes via decorators, so we throw + // instead — but we still want to return the body. Use HttpException directly. + const { HttpException } = require('@nestjs/common'); + throw new HttpException(result, HttpStatus.SERVICE_UNAVAILABLE); + } + + if (result.status === 'degraded') { + const { HttpException } = require('@nestjs/common'); + throw new HttpException(result, 207); + } + + return result; + } + + @Get('status') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'Payment provider circuit-breaker status (always 200)', + description: + 'Always returns HTTP 200 with circuit state in the body — useful for dashboards ' + + 'that poll metrics without relying on HTTP status semantics.', + }) + @ApiResponse({ status: 200, description: 'Circuit-breaker status payload' }) + getPaymentProviderStatus(): PaymentProviderHealthResult { + return this.healthIndicators.checkPaymentProvider(); + } +} diff --git a/src/health/health-indicators.service.ts b/src/health/health-indicators.service.ts index e0125394..95c5a28e 100644 --- a/src/health/health-indicators.service.ts +++ b/src/health/health-indicators.service.ts @@ -1,7 +1,21 @@ import { Injectable } from '@nestjs/common'; +import { PaymentProviderCircuitBreakerService } from '../payments/services/payment-provider-circuit-breaker.service'; + +export interface PaymentProviderHealthResult { + status: 'up' | 'degraded' | 'down'; + circuitState: string; + errorRate: number; + failures: number; + successes: number; + rejects: number; +} @Injectable() export class HealthIndicatorsService { + constructor( + private readonly paymentCircuitBreaker: PaymentProviderCircuitBreakerService, + ) {} + async checkPostgres(): Promise { return true; } @@ -18,12 +32,39 @@ export class HealthIndicatorsService { return true; } + checkPaymentProvider(): PaymentProviderHealthResult { + const stats = this.paymentCircuitBreaker.getStats(); + + let status: 'up' | 'degraded' | 'down'; + switch (stats.state) { + case 'OPEN': + status = 'down'; + break; + case 'HALF_OPEN': + status = 'degraded'; + break; + default: + status = 'up'; + } + + return { + status, + circuitState: stats.state, + errorRate: stats.errorRate, + failures: stats.failures, + successes: stats.successes, + rejects: stats.rejects, + }; + } + async readiness(): Promise> { + const paymentHealth = this.checkPaymentProvider(); const results = { postgres: (await this.checkPostgres()) ? 'up' : 'down', redis: (await this.checkRedis()) ? 'up' : 'down', elasticsearch: (await this.checkElasticsearch()) ? 'up' : 'down', queue: (await this.checkQueueDepth()) ? 'up' : 'down', + paymentProvider: paymentHealth.status, }; return results; } diff --git a/src/health/health.module.ts b/src/health/health.module.ts index 3a03f113..3f6f617c 100644 --- a/src/health/health.module.ts +++ b/src/health/health.module.ts @@ -1,5 +1,6 @@ import { Module } from '@nestjs/common'; import { ShutdownHealthController } from './controllers/shutdown-health.controller'; +import { PaymentProviderHealthController } from './controllers/payment-provider-health.controller'; import { GracefulShutdownService } from '../common/services/graceful-shutdown.service'; import { RequestTrackerService } from '../common/services/request-tracker.service'; import { DatabaseShutdownService } from '../database/services/database-shutdown.service'; @@ -7,9 +8,11 @@ import { WorkerShutdownService } from '../workers/services/worker-shutdown.servi import { ShutdownStateService } from '../common/services/shutdown-state.service'; import { PoolMonitorService } from '../database/pool/pool-monitor.service'; import { WorkerOrchestrationService } from '../workers/orchestration/worker-orchestration.service'; +import { HealthIndicatorsService } from './health-indicators.service'; +import { PaymentProviderCircuitBreakerService } from '../payments/services/payment-provider-circuit-breaker.service'; @Module({ - controllers: [ShutdownHealthController], + controllers: [ShutdownHealthController, PaymentProviderHealthController], providers: [ GracefulShutdownService, RequestTrackerService, @@ -18,12 +21,16 @@ import { WorkerOrchestrationService } from '../workers/orchestration/worker-orch ShutdownStateService, PoolMonitorService, WorkerOrchestrationService, + HealthIndicatorsService, + PaymentProviderCircuitBreakerService, ], exports: [ GracefulShutdownService, RequestTrackerService, DatabaseShutdownService, WorkerShutdownService, + HealthIndicatorsService, + PaymentProviderCircuitBreakerService, ], }) export class HealthModule {} diff --git a/src/payments/services/payment-provider-circuit-breaker.service.spec.ts b/src/payments/services/payment-provider-circuit-breaker.service.spec.ts new file mode 100644 index 00000000..59594ad4 --- /dev/null +++ b/src/payments/services/payment-provider-circuit-breaker.service.spec.ts @@ -0,0 +1,314 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ServiceUnavailableException } from '@nestjs/common'; +import { + PaymentProviderCircuitBreakerService, + CircuitState, +} from './payment-provider-circuit-breaker.service'; +import { IPaymentProvider } from '../providers/payment-provider.interface'; + +/** Creates a minimal mock IPaymentProvider */ +function makeMockProvider(overrides: Partial = {}): jest.Mocked { + return { + name: 'mock-provider', + createPaymentIntent: jest.fn(), + createSubscription: jest.fn(), + cancelSubscription: jest.fn(), + refundPayment: jest.fn(), + handleWebhook: jest.fn(), + verifyWebhookSignature: jest.fn(), + ...overrides, + } as jest.Mocked; +} + +describe('PaymentProviderCircuitBreakerService', () => { + let service: PaymentProviderCircuitBreakerService; + let provider: jest.Mocked; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [PaymentProviderCircuitBreakerService], + }).compile(); + + service = module.get( + PaymentProviderCircuitBreakerService, + ); + provider = makeMockProvider(); + }); + + afterEach(async () => { + await service.onModuleDestroy(); + }); + + // --------------------------------------------------------------------------- + // CLOSED state — happy path + // --------------------------------------------------------------------------- + describe('CLOSED state (healthy provider)', () => { + it('should start in CLOSED state', () => { + expect(service.getCircuitState()).toBe('CLOSED'); + }); + + it('should forward createPaymentIntent and return provider result', async () => { + provider.createPaymentIntent.mockResolvedValueOnce({ + clientSecret: 'secret_123', + paymentIntentId: 'pi_123', + requiresAction: false, + }); + + const result = await service.createPaymentIntent(provider, 5000, 'usd'); + + expect(provider.createPaymentIntent).toHaveBeenCalledWith(5000, 'usd', undefined); + expect(result.clientSecret).toBe('secret_123'); + expect(result.paymentIntentId).toBe('pi_123'); + }); + + it('should forward createSubscription and return provider result', async () => { + const now = new Date(); + provider.createSubscription.mockResolvedValueOnce({ + subscriptionId: 'sub_123', + status: 'active', + currentPeriodEnd: now, + }); + + const result = await service.createSubscription(provider, 'cus_123', 'price_123'); + + expect(provider.createSubscription).toHaveBeenCalledWith('cus_123', 'price_123', undefined); + expect(result.subscriptionId).toBe('sub_123'); + }); + + it('should forward cancelSubscription', async () => { + provider.cancelSubscription.mockResolvedValueOnce(true); + const result = await service.cancelSubscription(provider, 'sub_456'); + expect(result).toBe(true); + }); + + it('should forward refundPayment', async () => { + provider.refundPayment.mockResolvedValueOnce({ refundId: 're_123', status: 'succeeded' }); + const result = await service.refundPayment(provider, 'pi_789', 1000); + expect(result.refundId).toBe('re_123'); + }); + + it('should forward handleWebhook', async () => { + provider.handleWebhook.mockResolvedValueOnce({ type: 'payment.succeeded', data: {} }); + const result = await service.handleWebhook(provider, { raw: 'body' }, 'sig_abc'); + expect(result.type).toBe('payment.succeeded'); + }); + + it('should forward verifyWebhookSignature', async () => { + provider.verifyWebhookSignature.mockResolvedValueOnce(true); + const result = await service.verifyWebhookSignature(provider, { raw: 'body' }, 'sig_abc'); + expect(result).toBe(true); + }); + + it('should report 0 errorRate when calls succeed', async () => { + provider.createPaymentIntent.mockResolvedValue({ + clientSecret: 's', + paymentIntentId: 'p', + }); + await service.createPaymentIntent(provider, 100, 'usd'); + const stats = service.getStats(); + expect(stats.errorRate).toBe(0); + expect(stats.successes).toBeGreaterThanOrEqual(1); + }); + + it('should propagate provider errors without opening the circuit below threshold', async () => { + provider.createPaymentIntent.mockRejectedValueOnce(new Error('card declined')); + await expect(service.createPaymentIntent(provider, 100, 'usd')).rejects.toThrow( + 'card declined', + ); + // Below volumeThreshold (5) — circuit stays closed + expect(service.getCircuitState()).toBe('CLOSED'); + }); + }); + + // --------------------------------------------------------------------------- + // OPEN state — after sufficient failures + // --------------------------------------------------------------------------- + describe('OPEN state (provider failing)', () => { + /** + * Drive enough failures to trip the circuit. + * volumeThreshold = 5, errorThresholdPercentage = 50, so we need at least + * 5 failures (100 % error rate) in the rolling window. + */ + async function tripCircuit() { + provider.createPaymentIntent.mockRejectedValue(new Error('provider down')); + for (let i = 0; i < 5; i++) { + try { + await service.createPaymentIntent(provider, 100, 'usd'); + } catch { + // expected + } + } + // Wait a tick for opossum to update its internal state + await new Promise((r) => setImmediate(r)); + } + + it('should open the circuit after 5 consecutive failures', async () => { + await tripCircuit(); + expect(service.getCircuitState()).toBe('OPEN'); + }); + + it('should throw ServiceUnavailableException when circuit is OPEN', async () => { + await tripCircuit(); + // Circuit is now open — all subsequent calls should fast-fail + await expect(service.createPaymentIntent(provider, 100, 'usd')).rejects.toThrow( + ServiceUnavailableException, + ); + }); + + it('should report state = "down" in getStats() when circuit is OPEN', async () => { + await tripCircuit(); + const stats = service.getStats(); + expect(stats.state).toBe('OPEN'); + }); + + it('should reject calls without calling the provider when circuit is OPEN', async () => { + await tripCircuit(); + provider.createPaymentIntent.mockClear(); + + try { + await service.createPaymentIntent(provider, 100, 'usd'); + } catch { + // expected + } + + // Provider should NOT have been called (fast-fail) + expect(provider.createPaymentIntent).not.toHaveBeenCalled(); + }); + }); + + // --------------------------------------------------------------------------- + // HALF_OPEN state — recovery probe + // + // opossum transitions OPEN → HALF_OPEN automatically after `resetTimeout`. + // For tests we create a local breaker with resetTimeout=50ms and use + // real timers (jest.useFakeTimers is avoided to keep the test readable). + // --------------------------------------------------------------------------- + describe('HALF_OPEN state (probe after resetTimeout)', () => { + it('should expose a circuitBreaker property for observing state in tests', () => { + expect(service.circuitBreaker).toBeDefined(); + }); + + it('should emit "halfOpen" event after resetTimeout elapses', async () => { + // Use a fresh local breaker with a short resetTimeout so the test is fast. + const CircuitBreaker = require('opossum'); + const localBreaker: InstanceType = new CircuitBreaker( + (fn: () => Promise) => fn(), + { timeout: 1000, errorThresholdPercentage: 50, resetTimeout: 50, volumeThreshold: 5 }, + ); + + let halfOpenEmitted = false; + localBreaker.on('halfOpen', () => { halfOpenEmitted = true; }); + + // Trip the circuit with 5 failures. + for (let i = 0; i < 5; i++) { + try { await localBreaker.fire(() => Promise.reject(new Error('fail'))); } catch { /* expected */ } + } + expect(localBreaker.opened).toBe(true); + + // Wait for resetTimeout to fire the half-open transition. + await new Promise((r) => setTimeout(r, 100)); + + expect(halfOpenEmitted).toBe(true); + expect(localBreaker.halfOpen).toBe(true); + + await localBreaker.shutdown(); + }, 10_000); + + it('should return to CLOSED after a successful probe in half-open state', async () => { + const CircuitBreaker = require('opossum'); + const localBreaker: InstanceType = new CircuitBreaker( + (fn: () => Promise) => fn(), + { timeout: 1000, errorThresholdPercentage: 50, resetTimeout: 50, volumeThreshold: 5 }, + ); + + // Trip the circuit. + for (let i = 0; i < 5; i++) { + try { await localBreaker.fire(() => Promise.reject(new Error('fail'))); } catch { /* expected */ } + } + expect(localBreaker.opened).toBe(true); + + // Wait for half-open transition. + await new Promise((r) => setTimeout(r, 100)); + expect(localBreaker.halfOpen).toBe(true); + + // Successful probe — circuit should close. + await localBreaker.fire(() => Promise.resolve('ok')); + await new Promise((r) => setImmediate(r)); + + expect(localBreaker.closed).toBe(true); + + await localBreaker.shutdown(); + }, 10_000); + + it('should stay OPEN after a failed probe in half-open state', async () => { + const CircuitBreaker = require('opossum'); + const localBreaker: InstanceType = new CircuitBreaker( + (fn: () => Promise) => fn(), + { timeout: 1000, errorThresholdPercentage: 50, resetTimeout: 50, volumeThreshold: 5 }, + ); + + // Trip the circuit. + for (let i = 0; i < 5; i++) { + try { await localBreaker.fire(() => Promise.reject(new Error('fail'))); } catch { /* expected */ } + } + expect(localBreaker.opened).toBe(true); + + // Wait for half-open transition. + await new Promise((r) => setTimeout(r, 100)); + expect(localBreaker.halfOpen).toBe(true); + + // Failed probe — circuit should reopen. + try { await localBreaker.fire(() => Promise.reject(new Error('still down'))); } catch { /* expected */ } + await new Promise((r) => setImmediate(r)); + + expect(localBreaker.opened).toBe(true); + + await localBreaker.shutdown(); + }, 10_000); + }); + + // --------------------------------------------------------------------------- + // getStats() + // --------------------------------------------------------------------------- + describe('getStats()', () => { + it('should return correct stats structure', () => { + const stats = service.getStats(); + expect(stats).toMatchObject({ + state: expect.stringMatching(/^(CLOSED|OPEN|HALF_OPEN)$/), + failures: expect.any(Number), + successes: expect.any(Number), + fallbacks: expect.any(Number), + rejects: expect.any(Number), + timeouts: expect.any(Number), + errorRate: expect.any(Number), + }); + }); + + it('should increment failures on error', async () => { + provider.createPaymentIntent.mockRejectedValueOnce(new Error('fail')); + try { + await service.createPaymentIntent(provider, 100, 'usd'); + } catch { + // expected + } + const stats = service.getStats(); + expect(stats.failures).toBeGreaterThanOrEqual(1); + }); + + it('should increment successes on success', async () => { + provider.cancelSubscription.mockResolvedValueOnce(true); + await service.cancelSubscription(provider, 'sub_ok'); + const stats = service.getStats(); + expect(stats.successes).toBeGreaterThanOrEqual(1); + }); + }); + + // --------------------------------------------------------------------------- + // Lifecycle + // --------------------------------------------------------------------------- + describe('onModuleDestroy()', () => { + it('should shut down the underlying circuit breaker without throwing', async () => { + await expect(service.onModuleDestroy()).resolves.not.toThrow(); + }); + }); +}); diff --git a/src/payments/services/payment-provider-circuit-breaker.service.ts b/src/payments/services/payment-provider-circuit-breaker.service.ts new file mode 100644 index 00000000..c5f92de7 --- /dev/null +++ b/src/payments/services/payment-provider-circuit-breaker.service.ts @@ -0,0 +1,199 @@ +import { Injectable, Logger, OnModuleDestroy, ServiceUnavailableException } from '@nestjs/common'; +import CircuitBreaker from 'opossum'; +import { IPaymentProvider } from '../providers/payment-provider.interface'; + +export type CircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN'; + +export interface PaymentCircuitStats { + state: CircuitState; + failures: number; + successes: number; + fallbacks: number; + rejects: number; + timeouts: number; + errorRate: number; +} + +/** + * PaymentProviderCircuitBreakerService + * + * Wraps every IPaymentProvider method with a shared opossum circuit breaker. + * Configuration (per spec): + * - errorThresholdPercentage: 50 (open after ≥50% failures in the rolling window) + * - resetTimeout: 30 000 ms (half-open probe after 30 s) + * - timeout: 10 000 ms (individual call timeout) + * - volumeThreshold: 5 (need ≥5 calls before the % check matters) + * + * Acceptance criteria mapping: + * - After 5 consecutive failures the circuit opens and new requests reject with 503. + * - Circuit auto-recovers after a successful half-open probe (resetTimeout elapses). + * - `getCircuitState()` / `getStats()` expose the state for the health endpoint. + */ +@Injectable() +export class PaymentProviderCircuitBreakerService implements OnModuleDestroy { + private readonly logger = new Logger(PaymentProviderCircuitBreakerService.name); + + /** Key used to identify this breaker in logs and metrics */ + static readonly BREAKER_NAME = 'payment-provider'; + + private readonly breaker: CircuitBreaker; + + constructor() { + // `action` is a generic placeholder; each public method supplies its own + // async thunk via breaker.fire(fn), so we register a no-op here. + this.breaker = new CircuitBreaker( + (fn: () => Promise) => fn(), + { + name: PaymentProviderCircuitBreakerService.BREAKER_NAME, + timeout: 10_000, // 10 s per call + errorThresholdPercentage: 50, // open when ≥50 % of calls in window fail + resetTimeout: 30_000, // wait 30 s before probing in half-open + volumeThreshold: 5, // require at least 5 calls before tripping + rollingCountTimeout: 60_000, // 60 s rolling stats window + rollingCountBuckets: 10, + }, + ); + + this.breaker.on('open', () => + this.logger.warn('[payment-provider] Circuit OPENED — fast-failing all payment calls'), + ); + this.breaker.on('close', () => + this.logger.log('[payment-provider] Circuit CLOSED — payment provider recovered'), + ); + this.breaker.on('halfOpen', () => + this.logger.log('[payment-provider] Circuit HALF_OPEN — sending probe request'), + ); + this.breaker.on('fallback', () => + this.logger.warn('[payment-provider] Circuit FALLBACK triggered'), + ); + this.breaker.on('reject', () => + this.logger.warn('[payment-provider] Circuit REJECTED request (circuit is open)'), + ); + this.breaker.on('timeout', () => + this.logger.warn('[payment-provider] Circuit TIMEOUT — provider call exceeded 10 s'), + ); + this.breaker.on('failure', (err: Error) => + this.logger.error(`[payment-provider] Circuit FAILURE: ${err?.message}`), + ); + this.breaker.on('success', () => + this.logger.debug('[payment-provider] Circuit SUCCESS'), + ); + } + + async onModuleDestroy(): Promise { + await this.breaker.shutdown(); + this.logger.log('[payment-provider] Circuit breaker shut down'); + } + + // --------------------------------------------------------------------------- + // Internal helper + // --------------------------------------------------------------------------- + + /** + * Fires the circuit breaker with the provided thunk. + * If the circuit is open, opossum throws an error that we convert to 503. + */ + private async fire(thunk: () => Promise): Promise { + try { + return (await this.breaker.fire(thunk)) as T; + } catch (err: unknown) { + const isCircuitOpen = + err instanceof Error && + (err.message.includes('Breaker is open') || + err.message.includes('open') || + (err as any).code === 'EOPENBREAKER'); + + if (isCircuitOpen) { + throw new ServiceUnavailableException( + 'Payment provider is temporarily unavailable. Please try again later.', + ); + } + + throw err; + } + } + + // --------------------------------------------------------------------------- + // IPaymentProvider proxy methods + // --------------------------------------------------------------------------- + + async createPaymentIntent( + provider: IPaymentProvider, + amount: number, + currency: string, + metadata?: Record, + ): Promise<{ clientSecret: string; paymentIntentId: string; requiresAction?: boolean }> { + return this.fire(() => provider.createPaymentIntent(amount, currency, metadata)); + } + + async createSubscription( + provider: IPaymentProvider, + customerId: string, + priceId: string, + metadata?: Record, + ): Promise<{ subscriptionId: string; status: string; currentPeriodEnd: Date }> { + return this.fire(() => provider.createSubscription(customerId, priceId, metadata)); + } + + async cancelSubscription( + provider: IPaymentProvider, + subscriptionId: string, + ): Promise { + return this.fire(() => provider.cancelSubscription(subscriptionId)); + } + + async refundPayment( + provider: IPaymentProvider, + paymentId: string, + amount?: number, + ): Promise<{ refundId: string; status: string }> { + return this.fire(() => provider.refundPayment(paymentId, amount)); + } + + async handleWebhook( + provider: IPaymentProvider, + payload: unknown, + signature: string, + ): Promise<{ type: string; data: unknown }> { + return this.fire(() => provider.handleWebhook(payload, signature)); + } + + async verifyWebhookSignature( + provider: IPaymentProvider, + payload: unknown, + signature: string, + ): Promise { + return this.fire(() => provider.verifyWebhookSignature(payload, signature)); + } + + // --------------------------------------------------------------------------- + // Observability helpers (used by health endpoint) + // --------------------------------------------------------------------------- + + getCircuitState(): CircuitState { + if (this.breaker.opened) return 'OPEN'; + if (this.breaker.halfOpen) return 'HALF_OPEN'; + return 'CLOSED'; + } + + getStats(): PaymentCircuitStats { + const raw = this.breaker.status.stats; + const total = raw.successes + raw.failures + raw.rejects; + const errorRate = total > 0 ? Math.round((raw.failures / total) * 100) : 0; + + return { + state: this.getCircuitState(), + failures: raw.failures, + successes: raw.successes, + fallbacks: raw.fallbacks, + rejects: raw.rejects, + timeouts: raw.timeouts, + errorRate, + }; + } + + /** Expose the raw breaker for testing */ + get circuitBreaker(): CircuitBreaker { + return this.breaker; + } +}