diff --git a/packages/backend/src/accounting/psql/service.ts b/packages/backend/src/accounting/psql/service.ts index b0b491a035..c2952d477c 100644 --- a/packages/backend/src/accounting/psql/service.ts +++ b/packages/backend/src/accounting/psql/service.ts @@ -200,8 +200,9 @@ export async function createTransfer( deps: ServiceDependencies, args: TransferOptions ): Promise { - return createAccountToAccountTransfer(deps, { + return createAccountToAccountTransfer({ transferArgs: args, + withdrawalThrottleDelay: deps.withdrawalThrottleDelay, voidTransfers: async (transferRefs) => voidTransfers(deps, transferRefs), postTransfers: async (transferRefs) => postTransfers(deps, transferRefs), getAccountReceived: async (accountRef) => diff --git a/packages/backend/src/accounting/service.ts b/packages/backend/src/accounting/service.ts index 214dd006a8..731c232c0b 100644 --- a/packages/backend/src/accounting/service.ts +++ b/packages/backend/src/accounting/service.ts @@ -1,5 +1,4 @@ import { TransactionOrKnex } from 'objection' -import { BaseService } from '../shared/baseService' import { TransferError, isTransferError } from './errors' export enum LiquidityAccountType { @@ -10,17 +9,13 @@ export enum LiquidityAccountType { WEB_MONETIZATION = 'WEB_MONETIZATION' } -export interface LiquidityAccountAsset { - id: string - code?: string - scale?: number - ledger: number - onDebit?: (options: OnDebitOptions) => Promise -} - export interface LiquidityAccount { id: string - asset: LiquidityAccountAsset + asset: { + id: string + ledger: number + onDebit?: (options: OnDebitOptions) => Promise + } onCredit?: (options: OnCreditOptions) => Promise onDebit?: (options: OnDebitOptions) => Promise } @@ -90,10 +85,6 @@ export interface TransferToCreate { ledger: number } -export interface BaseAccountingServiceDependencies extends BaseService { - withdrawalThrottleDelay?: number -} - interface CreateAccountToAccountTransferArgs { transferArgs: TransferOptions voidTransfers(transferIds: string[]): Promise @@ -103,10 +94,10 @@ interface CreateAccountToAccountTransferArgs { createPendingTransfers( transfers: TransferToCreate[] ): Promise + withdrawalThrottleDelay?: number } export async function createAccountToAccountTransfer( - deps: BaseAccountingServiceDependencies, args: CreateAccountToAccountTransferArgs ): Promise { const { @@ -115,11 +106,10 @@ export async function createAccountToAccountTransfer( createPendingTransfers, getAccountReceived, getAccountBalance, + withdrawalThrottleDelay, transferArgs } = args - const { withdrawalThrottleDelay } = deps - const { sourceAccount, destinationAccount, sourceAmount, destinationAmount } = transferArgs diff --git a/packages/backend/src/accounting/tigerbeetle/service.ts b/packages/backend/src/accounting/tigerbeetle/service.ts index 16e6ffa0e2..9d03024628 100644 --- a/packages/backend/src/accounting/tigerbeetle/service.ts +++ b/packages/backend/src/accounting/tigerbeetle/service.ts @@ -215,8 +215,9 @@ export async function createTransfer( deps: ServiceDependencies, args: TransferOptions ): Promise { - return createAccountToAccountTransfer(deps, { + return createAccountToAccountTransfer({ transferArgs: args, + withdrawalThrottleDelay: deps.withdrawalThrottleDelay, voidTransfers: async (transferIds) => { const error = await createTransfers( deps, diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index bf7c82be33..7789b887ad 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -287,6 +287,9 @@ export function initIocContainer( knex: await deps.use('knex'), accountingService: await deps.use('accountingService'), walletAddressService: await deps.use('walletAddressService'), + telemetry: config.enableTelemetry + ? await deps.use('telemetry') + : undefined, config: await deps.use('config') }) }) diff --git a/packages/backend/src/open_payments/payment/incoming/service.ts b/packages/backend/src/open_payments/payment/incoming/service.ts index afad128bbe..83a8520c50 100644 --- a/packages/backend/src/open_payments/payment/incoming/service.ts +++ b/packages/backend/src/open_payments/payment/incoming/service.ts @@ -17,6 +17,7 @@ import { import { Amount } from '../../amount' import { IncomingPaymentError } from './errors' import { IAppConfig } from '../../../config/app' +import { TelemetryService } from '../../../telemetry/service' export const POSITIVE_SLIPPAGE = BigInt(1) // First retry waits 10 seconds @@ -47,6 +48,7 @@ export interface ServiceDependencies extends BaseService { accountingService: AccountingService walletAddressService: WalletAddressService config: IAppConfig + telemetry?: TelemetryService } export async function createIncomingPaymentService( @@ -220,6 +222,11 @@ async function handleDeactivated( ) await incomingPayment.$query(deps.knex).patch({ processAt: null }) return + } else if (deps.telemetry) { + // TODO: tests + deps.telemetry.incrementCounter('transactions_count_incoming', 1, { + description: 'Count of funded incoming transactions' + }) } const type = diff --git a/packages/backend/src/open_payments/payment/outgoing/lifecycle.ts b/packages/backend/src/open_payments/payment/outgoing/lifecycle.ts index a825bd6515..30585c1399 100644 --- a/packages/backend/src/open_payments/payment/outgoing/lifecycle.ts +++ b/packages/backend/src/open_payments/payment/outgoing/lifecycle.ts @@ -85,8 +85,8 @@ export async function handleSending( const payEndTime = Date.now() if (deps.telemetry) { - deps.telemetry.incrementCounter('transactions_total', 1, { - description: 'Count of funded transactions' + deps.telemetry.incrementCounter('transactions_count_outgoing', 1, { + description: 'Count of funded outgoing transactions' }) const payDuration = payEndTime - payStartTime diff --git a/packages/backend/src/payment-method/ilp/connector/core/middleware/ilp-packet.ts b/packages/backend/src/payment-method/ilp/connector/core/middleware/ilp-packet.ts index 3f7d903b47..dce6a8c628 100644 --- a/packages/backend/src/payment-method/ilp/connector/core/middleware/ilp-packet.ts +++ b/packages/backend/src/payment-method/ilp/connector/core/middleware/ilp-packet.ts @@ -17,6 +17,7 @@ import { import { Readable } from 'stream' import { HttpContext, HttpMiddleware, ILPMiddleware } from '../rafiki' import getRawBody from 'raw-body' +import { ValueType } from '@opentelemetry/api' export const CONTENT_TYPE = 'application/octet-stream' @@ -234,5 +235,34 @@ export function createIlpPacketMiddleware( ctx.assert(!ctx.body, 500, 'response body already set') ctx.assert(response.rawReply, 500, 'ilp reply not set') ctx.body = response.rawReply + + if (ctx.services.telemetry && Number(prepare.amount)) { + ctx.services.telemetry?.incrementCounter('packet_count_prepare', 1, { + description: 'Count of incoming prepare packets' + }) + if (response.fulfill) { + const { code, scale } = ctx.state.incomingAccount.asset // Need to type these + const value = BigInt(prepare.amount) + await ctx.services.telemetry.incrementCounterWithTransactionAmount( + 'transactions_amount', + { + value, + code, + scale + }, + { + description: 'Amount sent through the network', + valueType: ValueType.DOUBLE + } + ) + ctx.services.telemetry?.incrementCounter('packet_count_fulfill', 1, { + description: 'Count of outgoing fulfill packets' + }) + } else if (response.reject) { + ctx.services.telemetry?.incrementCounter('packet_count_reject', 1, { + description: 'Count of outgoing reject packets' + }) + } + } } } diff --git a/packages/backend/src/payment-method/ilp/connector/core/middleware/telemetry.ts b/packages/backend/src/payment-method/ilp/connector/core/middleware/telemetry.ts deleted file mode 100644 index 8b28cf5cf6..0000000000 --- a/packages/backend/src/payment-method/ilp/connector/core/middleware/telemetry.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { ValueType } from '@opentelemetry/api' -import { ILPContext, ILPMiddleware } from '../rafiki' - -export function createTelemetryMiddleware(): ILPMiddleware { - return async ( - { request, services, accounts, response }: ILPContext, - next: () => Promise - ): Promise => { - await next() - - const value = BigInt(request.prepare.amount) - - if (services.telemetry && value && response.fulfill) { - const { code: assetCode, scale: assetScale } = accounts.outgoing.asset - - await services.telemetry.incrementCounterWithTransactionAmount( - 'transactions_amount', - { - value, - assetCode, - assetScale - }, - { - description: 'Amount sent through the network', - valueType: ValueType.DOUBLE - } - ) - } - } -} diff --git a/packages/backend/src/payment-method/ilp/connector/core/test/middleware/ilp-packet-middleware.test.ts b/packages/backend/src/payment-method/ilp/connector/core/test/middleware/ilp-packet-middleware.test.ts index 3108134a19..3ee6647427 100644 --- a/packages/backend/src/payment-method/ilp/connector/core/test/middleware/ilp-packet-middleware.test.ts +++ b/packages/backend/src/payment-method/ilp/connector/core/test/middleware/ilp-packet-middleware.test.ts @@ -1,3 +1,4 @@ +// import assert from 'assert' import { Readable } from 'stream' import { serializeIlpPrepare, @@ -11,20 +12,46 @@ import { createIlpPacketMiddleware, IlpResponse, ZeroCopyIlpPrepare -} from '../../middleware/ilp-packet' + // OutgoingAccount +} from '../../' import { IlpPrepareFactory, IlpFulfillFactory, - IlpRejectFactory + IlpRejectFactory, + // IncomingAccountFactory, + RafikiServicesFactory } from '../../factories' import { HttpContext } from '../../rafiki' describe('ILP Packet Middleware', () => { test('sets up request and response', async () => { + // const incomingAccount = IncomingAccountFactory.build({ id: 'alice' }) + // assert.ok(incomingAccount.id) + const services = RafikiServicesFactory.build({}) + const options: MockIncomingMessageOptions = { headers: { 'content-type': 'application/octet-stream' } } - const ctx = createContext({ req: options }) + + const ctx = createContext({ + req: options, + services: { ...services, telemetry: undefined } + // accounts: { + // incoming: incomingAccount, + // outgoing: { asset: { code: 'USD', scale: 2 } } as OutgoingAccount + // }, + // state: { + // unfulfillable: false, + // incomingAccount: { + // quote: 'exists', + // asset: { + // code: 'USD', + // scale: 2 + // } + // } + // } + }) + const prepare = IlpPrepareFactory.build() const getRawBody = async (_req: Readable) => serializeIlpPrepare(prepare) const rawReply = serializeIlpFulfill(IlpFulfillFactory.build()) diff --git a/packages/backend/src/payment-method/ilp/connector/core/test/middleware/telemetry.test.ts b/packages/backend/src/payment-method/ilp/connector/core/test/middleware/telemetry.test.ts deleted file mode 100644 index e33e21afa9..0000000000 --- a/packages/backend/src/payment-method/ilp/connector/core/test/middleware/telemetry.test.ts +++ /dev/null @@ -1,113 +0,0 @@ -import assert from 'assert' -import { IlpResponse, OutgoingAccount, ZeroCopyIlpPrepare } from '../..' -import { IncomingAccountFactory, RafikiServicesFactory } from '../../factories' -import { createTelemetryMiddleware } from '../../middleware/telemetry' -import { createILPContext } from '../../utils' -import { IlpFulfill } from 'ilp-packet' - -const incomingAccount = IncomingAccountFactory.build({ id: 'alice' }) - -assert.ok(incomingAccount.id) -const services = RafikiServicesFactory.build({}) - -const ctx = createILPContext({ - services, - request: { - prepare: { - amount: 100n - } as unknown as ZeroCopyIlpPrepare, - rawPrepare: Buffer.from('') - }, - accounts: { - incoming: incomingAccount, - outgoing: { asset: { code: 'USD', scale: 2 } } as OutgoingAccount - }, - state: { - unfulfillable: false, - incomingAccount: { - quote: 'exists' - } - }, - response: { - fulfill: 'exists' as unknown as IlpFulfill - } as IlpResponse -}) - -const middleware = createTelemetryMiddleware() -const next = jest.fn().mockImplementation(() => Promise.resolve()) - -beforeEach(async () => { - incomingAccount.balance = 100n - incomingAccount.asset.scale = 2 - incomingAccount.asset.code = 'USD' -}) - -describe('Telemetry Middleware', function () { - it('should not gather telemetry if telemetry is not enabled (service is undefined)', async () => { - const incrementCounterWithTransactionAmountSpy = jest - .spyOn(services.telemetry, 'incrementCounterWithTransactionAmount') - .mockImplementation(() => Promise.resolve()) - - await middleware( - { ...ctx, services: { ...ctx.services, telemetry: undefined } }, - next - ) - - expect(incrementCounterWithTransactionAmountSpy).not.toHaveBeenCalled() - expect(next).toHaveBeenCalled() - }) - - it('should not gather telemetry if response.fulfill undefined', async () => { - const incrementCounterWithTransactionAmountSpy = jest - .spyOn(services.telemetry, 'incrementCounterWithTransactionAmount') - .mockImplementation(() => Promise.resolve()) - - await middleware( - { ...ctx, response: { fulfill: undefined } as IlpResponse }, - next - ) - - expect(incrementCounterWithTransactionAmountSpy).not.toHaveBeenCalled() - expect(next).toHaveBeenCalled() - }) - - it('should not gather telemetry if request.prepare.amount is 0', async () => { - const incrementCounterWithTransactionAmountSpy = jest - .spyOn(services.telemetry, 'incrementCounterWithTransactionAmount') - .mockImplementation(() => Promise.resolve()) - - await middleware( - { - ...ctx, - request: { - ...ctx.request, - prepare: { amount: '0' } as ZeroCopyIlpPrepare - } - }, - next - ) - - expect(incrementCounterWithTransactionAmountSpy).not.toHaveBeenCalled() - expect(next).toHaveBeenCalled() - }) - - it('should gather telemetry without blocking middleware chain', async () => { - let nextCalled = false - const next = jest.fn().mockImplementation(() => { - nextCalled = true - return Promise.resolve() - }) - - const incrementCounterWithTransactionAmountSpy = jest - .spyOn(services.telemetry, 'incrementCounterWithTransactionAmount') - .mockImplementation(() => { - expect(nextCalled).toBe(true) - return Promise.resolve() - }) - - await middleware(ctx, next) - - expect(incrementCounterWithTransactionAmountSpy).toHaveBeenCalled() - expect(next).toHaveBeenCalled() - }) -}) diff --git a/packages/backend/src/payment-method/ilp/connector/index.ts b/packages/backend/src/payment-method/ilp/connector/index.ts index c97f3349d2..9f3ecc3f95 100644 --- a/packages/backend/src/payment-method/ilp/connector/index.ts +++ b/packages/backend/src/payment-method/ilp/connector/index.ts @@ -27,20 +27,18 @@ import { createStreamAddressMiddleware, createStreamController } from './core' - import { TelemetryService } from '../../../telemetry/service' -import { createTelemetryMiddleware } from './core/middleware/telemetry' interface ServiceDependencies extends BaseService { redis: Redis ratesService: RatesService accountingService: AccountingService - telemetry?: TelemetryService walletAddressService: WalletAddressService incomingPaymentService: IncomingPaymentService peerService: PeerService streamServer: StreamServer ilpAddress: string + telemetry?: TelemetryService } export async function createConnectorService({ @@ -48,12 +46,12 @@ export async function createConnectorService({ redis, ratesService, accountingService, - telemetry, walletAddressService, incomingPaymentService, peerService, streamServer, - ilpAddress + ilpAddress, + telemetry }: ServiceDependencies): Promise { return createApp( { @@ -62,13 +60,13 @@ export async function createConnectorService({ service: 'ConnectorService' }), accounting: accountingService, - telemetry, walletAddresses: walletAddressService, incomingPayments: incomingPaymentService, peers: peerService, redis, rates: ratesService, - streamServer + streamServer, + telemetry }, compose([ // Incoming Rules @@ -82,7 +80,6 @@ export async function createConnectorService({ // Local pay createBalanceMiddleware(), - createTelemetryMiddleware(), // Outgoing Rules createStreamController(), diff --git a/packages/backend/src/telemetry/service.ts b/packages/backend/src/telemetry/service.ts index f7f18cf0ec..6a27f8c019 100644 --- a/packages/backend/src/telemetry/service.ts +++ b/packages/backend/src/telemetry/service.ts @@ -94,6 +94,7 @@ class TelemetryServiceImpl implements TelemetryService { }) } + // TODO: is this still including 'source' public async incrementCounterWithTransactionAmount( name: string, amount: { value: bigint; assetCode: string; assetScale: number }, diff --git a/packages/documentation/src/content/docs/telemetry/overview.md b/packages/documentation/src/content/docs/telemetry/overview.md index 8527f9c499..61cde0d447 100644 --- a/packages/documentation/src/content/docs/telemetry/overview.md +++ b/packages/documentation/src/content/docs/telemetry/overview.md @@ -7,8 +7,10 @@ title: Overview The objective of the telemetry feature is to gather metrics and establish an infrastructure for visualizing valuable network insights. The metrics we at the Interledger Foundation collect include: - The total amount of money transferred via packet data within a specified time frame (daily, weekly, monthly). -- The number of transactions from outgoing payments that have been at least partially successful. +- The number of transactions from incoming payments that have been at least partially successful. +- The number of ILP packets flowing through the network. - The average amount of money held within the network per transaction. +- The avergae time it takes for an outgoing payment to complete. We aim to track the growth of the network in terms of transaction sizes and the number of transactions processed. Our goal is to use these data for our own insights and to enable [Account Servicing Entities](/reference/glossary#account-servicing-entity) (ASEs) to gain their own insights. @@ -60,18 +62,31 @@ If an ASE does not provide the necessary exchange rate for a transaction, the te ## Instrumentation -Rafiki currently has three counter metrics. All data points (counter increases and histogram records) are exported to collection endpoints at a configurable interval (default recommended to 15s). +Rafiki currently has seven counter metrics. All data points (counter increases and histogram records) are exported to collection endpoints at a configurable interval (default recommended to 15s). -Currently collected metrics: +Metrics collected from the the sending side of a transaction: -- `transactions_total` - Counter metric - - Description: “Count of funded transactions” - - This counter metric increases by 1 for each successfully sent transaction. -- `transactions_amount` - Counter metric - - Description: “Amount sent through the network”. - - This amount metric increases by the amount sent in each ILP packet. +- `transactions_count_outgoing` - Counter metric + - Description: “Count of funded outgoing transactions” + - This counter metric increases by 1 for each successfully funded outgoing payment resource. - `ilp_pay_time_ms` - Histogram metric - - Description: “Time to complete an ILP payment” + - Description: “Time to complete an outgoing ILP payment” - This histogram metric records the time taken to make an ILP payment. -**Note**: The current implementation only collects metrics on the SENDING side of a transaction. Metrics for external open-payments transactions RECEIVED by a Rafiki instance in the network are not collected. +Metrics collected from the the receiving side of a transaction: + +- `packet_count_prepare` - Counter metric + - Description: “Count of incoming prepare packets” + - This counter metric increases by 1 for each prepare packet that is sent. +- `packet_count_fulfill` - Counter metric + - Description: “Count of outgoing fulfill packets” + - This counter metric increases by 1 for each fulfill packet that is sent. +- `packet_count_reject` - Counter metric + - Description: “Count of outgoing reject packets” + - This counter metric increases by 1 for each reject packet that is sent. +- `transactions_count_incoming` - Counter metric + - Description: “Count of funded incoming transactions” + - This counter metric increases by 1 for each successfully funded incoming payment resource. +- `packet_amount_fulfill` - Counter metric + - Description: “Amount sent through the network” + - This amount metric increases by the amount sent in each ILP packet.