From 2c8fc82e7bd6dfec5e73f5225b26acf2d394d2d5 Mon Sep 17 00:00:00 2001 From: Nathan Lie Date: Thu, 22 Aug 2024 09:50:16 +0200 Subject: [PATCH 1/3] feat(backend): add trace to outgoing payment lifecycle --- localenv/cloud-nine-wallet/docker-compose.yml | 1 + localenv/happy-life-bank/docker-compose.yml | 1 + .../open_payments/payment/outgoing/worker.ts | 27 ++++++++++++------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/localenv/cloud-nine-wallet/docker-compose.yml b/localenv/cloud-nine-wallet/docker-compose.yml index 9d2921bd89..3bfdbcab72 100644 --- a/localenv/cloud-nine-wallet/docker-compose.yml +++ b/localenv/cloud-nine-wallet/docker-compose.yml @@ -74,6 +74,7 @@ services: WALLET_ADDRESS_URL: ${CLOUD_NINE_WALLET_ADDRESS_URL:-https://cloud-nine-wallet-backend/.well-known/pay} ILP_CONNECTOR_URL: ${CLOUD_NINE_CONNECTOR_URL:-http://cloud-nine-wallet-backend:3002} ENABLE_TELEMETRY: true + ENABLE_TELEMETRY_TRACES: true KEY_ID: 7097F83B-CB84-469E-96C6-2141C72E22C0 depends_on: - shared-database diff --git a/localenv/happy-life-bank/docker-compose.yml b/localenv/happy-life-bank/docker-compose.yml index 465bd030ff..d45ec80565 100644 --- a/localenv/happy-life-bank/docker-compose.yml +++ b/localenv/happy-life-bank/docker-compose.yml @@ -66,6 +66,7 @@ services: REDIS_URL: redis://shared-redis:6379/2 WALLET_ADDRESS_URL: ${HAPPY_LIFE_BANK_WALLET_ADDRESS_URL:-https://happy-life-bank-backend/.well-known/pay} ENABLE_TELEMETRY: true + ENABLE_TELEMETRY_TRACES: true KEY_ID: 53f2d913-e98a-40b9-b270-372d0547f23d depends_on: - cloud-nine-backend diff --git a/packages/backend/src/open_payments/payment/outgoing/worker.ts b/packages/backend/src/open_payments/payment/outgoing/worker.ts index 4c9e35cf60..675a663dda 100644 --- a/packages/backend/src/open_payments/payment/outgoing/worker.ts +++ b/packages/backend/src/open_payments/payment/outgoing/worker.ts @@ -5,6 +5,7 @@ import { OutgoingPayment, OutgoingPaymentState } from './model' import { LifecycleError, PaymentError } from './errors' import * as lifecycle from './lifecycle' import { PaymentMethodHandlerError } from '../../../payment-method/handler/errors' +import { trace, Span } from '@opentelemetry/api' // First retry waits 10 seconds, second retry waits 20 (more) seconds, etc. export const RETRY_BACKOFF_SECONDS = 10 @@ -63,16 +64,24 @@ async function handlePaymentLifecycle( deps: ServiceDependencies, payment: OutgoingPayment ): Promise { - if (payment.state !== OutgoingPaymentState.Sending) { - deps.logger.warn('unexpected payment in lifecycle') - return - } + const tracer = trace.getTracer('outgoing_payment_worker') + await tracer.startActiveSpan( + 'outgoingPaymentLifecycle', + async (span: Span) => { + if (payment.state !== OutgoingPaymentState.Sending) { + deps.logger.warn('unexpected payment in lifecycle') + span.end() + return + } - try { - await lifecycle.handleSending(deps, payment) - } catch (error) { - await onLifecycleError(deps, payment, error as Error | PaymentError) - } + try { + await lifecycle.handleSending(deps, payment) + } catch (error) { + await onLifecycleError(deps, payment, error as Error | PaymentError) + } + span.end() + } + ) } async function onLifecycleError( From d49321c05b97b83da41ad4b05adaa4f9979f7d80 Mon Sep 17 00:00:00 2001 From: Max Kurapov Date: Tue, 27 Aug 2024 11:16:27 +0300 Subject: [PATCH 2/3] feat(backend): encapsulate query lookup in trace --- .../open_payments/payment/outgoing/worker.ts | 72 ++++++++++--------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/packages/backend/src/open_payments/payment/outgoing/worker.ts b/packages/backend/src/open_payments/payment/outgoing/worker.ts index 675a663dda..0df698f960 100644 --- a/packages/backend/src/open_payments/payment/outgoing/worker.ts +++ b/packages/backend/src/open_payments/payment/outgoing/worker.ts @@ -16,23 +16,33 @@ const MAX_STATE_ATTEMPTS = 5 export async function processPendingPayment( deps_: ServiceDependencies ): Promise { - return deps_.knex.transaction(async (trx) => { - const payment = await getPendingPayment(trx) - if (!payment) return - - await handlePaymentLifecycle( - { - ...deps_, - knex: trx, - logger: deps_.logger.child({ - payment: payment.id, - from_state: payment.state - }) - }, - payment - ) - return payment.id - }) + const tracer = trace.getTracer('outgoing_payment_worker') + + return tracer.startActiveSpan( + 'outgoingPaymentLifecycle', + async (span: Span) => { + const paymentId = await deps_.knex.transaction(async (trx) => { + const payment = await getPendingPayment(trx) + if (!payment) return + + await handlePaymentLifecycle( + { + ...deps_, + knex: trx, + logger: deps_.logger.child({ + payment: payment.id, + from_state: payment.state + }) + }, + payment + ) + return payment.id + }) + + span.end() + return paymentId + } + ) } // Fetch (and lock) a payment for work. @@ -64,24 +74,16 @@ async function handlePaymentLifecycle( deps: ServiceDependencies, payment: OutgoingPayment ): Promise { - const tracer = trace.getTracer('outgoing_payment_worker') - await tracer.startActiveSpan( - 'outgoingPaymentLifecycle', - async (span: Span) => { - if (payment.state !== OutgoingPaymentState.Sending) { - deps.logger.warn('unexpected payment in lifecycle') - span.end() - return - } - - try { - await lifecycle.handleSending(deps, payment) - } catch (error) { - await onLifecycleError(deps, payment, error as Error | PaymentError) - } - span.end() - } - ) + if (payment.state !== OutgoingPaymentState.Sending) { + deps.logger.warn('unexpected payment in lifecycle') + return + } + + try { + await lifecycle.handleSending(deps, payment) + } catch (error) { + await onLifecycleError(deps, payment, error as Error | PaymentError) + } } async function onLifecycleError( From dfe6b1dc81e9093a29a4bfe02fc23de4295c1641 Mon Sep 17 00:00:00 2001 From: Max Kurapov Date: Tue, 27 Aug 2024 11:17:34 +0300 Subject: [PATCH 3/3] chore(localenv): remove traces from default localenv --- localenv/cloud-nine-wallet/docker-compose.yml | 1 - localenv/happy-life-bank/docker-compose.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/localenv/cloud-nine-wallet/docker-compose.yml b/localenv/cloud-nine-wallet/docker-compose.yml index 3bfdbcab72..9d2921bd89 100644 --- a/localenv/cloud-nine-wallet/docker-compose.yml +++ b/localenv/cloud-nine-wallet/docker-compose.yml @@ -74,7 +74,6 @@ services: WALLET_ADDRESS_URL: ${CLOUD_NINE_WALLET_ADDRESS_URL:-https://cloud-nine-wallet-backend/.well-known/pay} ILP_CONNECTOR_URL: ${CLOUD_NINE_CONNECTOR_URL:-http://cloud-nine-wallet-backend:3002} ENABLE_TELEMETRY: true - ENABLE_TELEMETRY_TRACES: true KEY_ID: 7097F83B-CB84-469E-96C6-2141C72E22C0 depends_on: - shared-database diff --git a/localenv/happy-life-bank/docker-compose.yml b/localenv/happy-life-bank/docker-compose.yml index d45ec80565..465bd030ff 100644 --- a/localenv/happy-life-bank/docker-compose.yml +++ b/localenv/happy-life-bank/docker-compose.yml @@ -66,7 +66,6 @@ services: REDIS_URL: redis://shared-redis:6379/2 WALLET_ADDRESS_URL: ${HAPPY_LIFE_BANK_WALLET_ADDRESS_URL:-https://happy-life-bank-backend/.well-known/pay} ENABLE_TELEMETRY: true - ENABLE_TELEMETRY_TRACES: true KEY_ID: 53f2d913-e98a-40b9-b270-372d0547f23d depends_on: - cloud-nine-backend