feat(backend): performance monitor and caching (jason fix)

localenv/telemetry/grafana/provisioning/dashboards/example.json

@@ -63,9 +63,7 @@
         "orientation": "auto",
         "percentChangeColorMode": "standard",
         "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
+          "calcs": ["lastNotNull"],
           "fields": "",
           "values": false
         },
@@ -163,9 +161,7 @@
       "id": 10,
       "options": {
         "legend": {
-          "calcs": [
-            "p95"
-          ],
+          "calcs": ["p95"],
           "displayMode": "table",
           "placement": "right",
           "showLegend": true
@@ -342,9 +338,7 @@
         "namePlacement": "auto",
         "orientation": "horizontal",
         "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
+          "calcs": ["lastNotNull"],
           "fields": "",
           "values": false
         },
@@ -441,9 +435,7 @@
       "id": 9,
       "options": {
         "legend": {
-          "calcs": [
-            "p95"
-          ],
+          "calcs": ["p95"],
           "displayMode": "table",
           "placement": "right",
           "showLegend": true
@@ -607,9 +599,7 @@
       "id": 11,
       "options": {
         "legend": {
-          "calcs": [
-            "p95"
-          ],
+          "calcs": ["p95"],
           "displayMode": "table",
           "placement": "right",
           "showLegend": true
@@ -908,9 +898,7 @@
       "id": 13,
       "options": {
         "legend": {
-          "calcs": [
-            "p95"
-          ],
+          "calcs": ["p95"],
           "displayMode": "table",
           "placement": "right",
           "showLegend": true
@@ -984,9 +972,7 @@
         "footer": {
           "countRows": false,
           "fields": "",
-          "reducer": [
-            "sum"
-          ],
+          "reducer": ["sum"],
           "show": false
         },
         "showHeader": true,
@@ -1015,9 +1001,7 @@
               "operator": "=",
               "scope": "resource",
               "tag": "service.name",
-              "value": [
-                "RAFIKI_NETWORK"
-              ],
+              "value": ["RAFIKI_NETWORK"],
               "valueType": "string"
             },
             {
@@ -1144,7 +1128,7 @@
             "uid": "${PrometheusDS}"
           },
           "editorMode": "code",
-          "expr": "rate(fundPayment_sum[$__rate_interval])/rate(fundPayment_sum[$__rate_interval])",
+          "expr": "rate(fundPayment_sum[$__rate_interval])/rate(fundPayment_count[$__rate_interval])",
           "hide": false,
           "instant": false,
           "legendFormat": "{{callName}}",
@@ -1226,9 +1210,7 @@
       "id": 7,
       "options": {
         "legend": {
-          "calcs": [
-            "p95"
-          ],
+          "calcs": ["p95"],
           "displayMode": "table",
           "placement": "right",
           "showLegend": true
@@ -1391,9 +1373,7 @@
       "id": 12,
       "options": {
         "legend": {
-          "calcs": [
-            "p95"
-          ],
+          "calcs": ["p95"],
           "displayMode": "table",
           "placement": "right",
           "showLegend": true
@@ -1690,9 +1670,7 @@
         "orientation": "auto",
         "percentChangeColorMode": "standard",
         "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
+          "calcs": ["lastNotNull"],
           "fields": "",
           "values": false
         },
@@ -1788,9 +1766,7 @@
         "orientation": "auto",
         "percentChangeColorMode": "standard",
         "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
+          "calcs": ["lastNotNull"],
           "fields": "",
           "values": false
         },
@@ -1909,4 +1885,4 @@
   "uid": "fdr58stwkr6yof",
   "version": 1,
   "weekStart": ""
-}
+}

packages/backend/src/accounting/psql/balance.ts

@@ -14,7 +14,7 @@ export async function getAccountBalances(
   account: LedgerAccount,
   trx?: TransactionOrKnex
 ): Promise<AccountBalance> {
-  const stopTimer = deps.telemetry?.startTimer_('psql_getAccountBalances', {
+  const stopTimer = deps.telemetry?.startTimer('psql_getAccountBalances', {
     callName: 'psql_getAccountBalances'
   })
   try {

packages/backend/src/accounting/psql/service.ts

@@ -149,26 +149,19 @@ export async function getAccountTotalSent(
   deps: ServiceDependencies,
   accountRef: string
 ): Promise<bigint | undefined> {
-  deps.telemetry &&
-    deps.telemetry.startTimer('psql_getAccountTotalSent', {
-      callName: 'psql_getAccountTotalSent'
-    })
+  const stopTimer = deps.telemetry?.startTimer('psql_getAccountTotalSent', {
+    callName: 'psql_getAccountTotalSent'
+  })
   const account = await getLiquidityAccount(deps, accountRef)
 
   if (!account) {
-    deps.telemetry &&
-      deps.telemetry.startTimer('psql_getAccountTotalSent', {
-        callName: 'psql_getAccountTotalSent'
-      })
+    stopTimer && stopTimer()
     return
   }
 
   const totalsSent = (await getAccountBalances(deps, account)).debitsPosted
 
-  deps.telemetry &&
-    deps.telemetry.startTimer('psql_getAccountTotalSent', {
-      callName: 'psql_getAccountTotalSent'
-    })
+  stopTimer && stopTimer()
 
   return totalsSent
 }

packages/backend/src/accounting/tigerbeetle/service.ts

@@ -222,16 +222,15 @@ export async function getAccountTotalSent(
   deps: ServiceDependencies,
   id: string
 ): Promise<bigint | undefined> {
-  deps.telemetry &&
-    deps.telemetry.startTimer('tb_getAccountTotalSent', {
-      callName: 'tb_getAccountTotalSent'
-    })
+  const stopTimer = deps.telemetry?.startTimer('tb_getAccountTotalSent', {
+    callName: 'tb_getAccountTotalSent'
+  })
   const account = (await getAccounts(deps, [id]))[0]
   if (account) {
-    deps.telemetry && deps.telemetry.stopTimer('tb_getAccountTotalSent')
+    stopTimer && stopTimer()
     return account.debits_posted
   }
-  deps.telemetry && deps.telemetry.stopTimer('tb_getAccountTotalSent')
+  stopTimer && stopTimer()
 }
 
 export async function getAccountsTotalSent(

packages/backend/src/asset/service.ts

@@ -9,6 +9,7 @@ import { WalletAddress } from '../open_payments/wallet_address/model'
 import { Peer } from '../payment-method/ilp/peer/model'
 import { Quote } from '../open_payments/quote/model'
 import { IncomingPayment } from '../open_payments/payment/incoming/model'
+import { CacheDataStore } from '../cache'
 
 export interface AssetOptions {
   code: string
@@ -44,14 +45,14 @@ export interface AssetService {
 
 interface ServiceDependencies extends BaseService {
   accountingService: AccountingService
-  assetCache: Map<string, Asset>
+  cacheDataStore: CacheDataStore
 }
 
 export async function createAssetService({
   logger,
   knex,
   accountingService,
-  assetCache
+  cacheDataStore
 }: ServiceDependencies): Promise<AssetService> {
   const log = logger.child({
     service: 'AssetService'
@@ -61,7 +62,7 @@ export async function createAssetService({
     logger: log,
     knex,
     accountingService,
-    assetCache
+    cacheDataStore
   }
 
   return {
@@ -89,7 +90,7 @@ async function createAsset(
       .first()
 
     if (deletedAsset) {
-      deps.assetCache.delete(deletedAsset.id)
+      await deps.cacheDataStore.delete(deletedAsset.id)
 
       // if found, enable
       return await Asset.query(deps.knex)
@@ -115,7 +116,6 @@ async function createAsset(
         LiquidityAccountType.ASSET,
         trx
       )
-      deps.assetCache.set(asset.id, asset)
       return asset
     })
   } catch (err) {
@@ -138,7 +138,7 @@ async function updateAsset(
       .patchAndFetchById(id, { withdrawalThreshold, liquidityThreshold })
       .throwIfNotFound()
 
-    deps.assetCache.set(id, returnVal)
+    await deps.cacheDataStore.delete(id)
     return returnVal
   } catch (err) {
     if (err instanceof NotFoundError) {
@@ -156,6 +156,8 @@ async function deleteAsset(
   if (!deps.knex) {
     throw new Error('Knex undefined')
   }
+
+  await deps.cacheDataStore.delete(id)
   try {
     // return error in case there is a peer or wallet address using the asset
     const peer = await Peer.query(deps.knex).where('assetId', id).first()
@@ -169,7 +171,6 @@ async function deleteAsset(
     if (walletAddress) {
       return AssetError.CannotDeleteInUseAsset
     }
-    deps.assetCache.delete(id)
     return await Asset.query(deps.knex)
       .patchAndFetchById(id, { deletedAt: deletedAt.toISOString() })
       .throwIfNotFound()
@@ -185,11 +186,11 @@ async function getAsset(
   deps: ServiceDependencies,
   id: string
 ): Promise<void | Asset> {
-  const inMem = deps.assetCache.get(id)
+  const inMem = (await deps.cacheDataStore.get(id)) as Asset
   if (inMem) return inMem
 
   const asset = await Asset.query(deps.knex).whereNull('deletedAt').findById(id)
-  if (asset) deps.assetCache.set(asset.id, asset)
+  if (asset) await deps.cacheDataStore.set(asset.id, asset)
 
   return asset
 }
@@ -211,8 +212,9 @@ async function getAll(deps: ServiceDependencies): Promise<Asset[]> {
 async function setAssetOn(
   deps: ServiceDependencies,
   obj: ToSetOn
-): Promise<void> {
+): Promise<void | Asset> {
   if (!obj) return
   const asset = await getAsset(deps, obj.assetId)
   if (asset) obj.asset = asset
+  return asset
 }

packages/backend/src/cache/cache.ts

@@ -0,0 +1,46 @@
+import { CacheDataStore, CacheSupport } from './index'
+
+export interface Cached {
+  expiry: number
+  data: CacheSupport
+}
+
+export function createInMemoryDataStore(keyTtlMs: number): CacheDataStore {
+  const map = new Map<string, Cached>()
+
+  const getAndCheckExpiry = (key: string): Cached | undefined => {
+    const cached = map.get(key)
+    if (cached?.expiry && Date.now() >= cached.expiry) {
+      deleteKey(key)
+      return undefined
+    }
+    return cached
+  }
+
+  const deleteKey = (key: string) => map.delete(key)
+
+  return {
+    async get(key: string): Promise<CacheSupport> {
+      if (keyTtlMs < 1) return undefined
+      const cached = getAndCheckExpiry(key)
+      return cached?.data
+    },
+    async getKeyExpiry(key: string): Promise<Date | undefined> {
+      const cached = getAndCheckExpiry(key)
+
+      return cached ? new Date(cached.expiry) : undefined
+    },
+    async delete(key: string): Promise<void> {
+      deleteKey(key)
+    },
+    async set(key: string, value: CacheSupport): Promise<boolean> {
+      if (keyTtlMs < 1) return true
+      map.set(key, { expiry: Date.now() + keyTtlMs, data: value })
+      return true
+    },
+    async deleteAll(): Promise<void> {
+      map.clear()
+    }
+  }
+}
+export { CacheDataStore }

packages/backend/src/cache/index.ts

@@ -0,0 +1,19 @@
+import { Quote } from '../open_payments/quote/model'
+import { WalletAddress } from '../open_payments/wallet_address/model'
+import { OutgoingPayment } from '../open_payments/payment/outgoing/model'
+import { Asset } from '../asset/model'
+
+export type CacheSupport =
+  | Quote
+  | WalletAddress
+  | OutgoingPayment
+  | Asset
+  | undefined
+
+export interface CacheDataStore {
+  get(key: string): Promise<CacheSupport>
+  getKeyExpiry(key: string): Promise<Date | undefined>
+  set(key: string, value: CacheSupport): Promise<boolean>
+  delete(key: string): Promise<void>
+  deleteAll(): Promise<void>
+}

packages/backend/src/config/app.ts

@@ -187,7 +187,8 @@ export const Config = {
     'INCOMING_PAYMENT_EXPIRY_MAX_MS',
     2592000000
   ), // 30 days
-  enableSpspPaymentPointers: envBool('ENABLE_SPSP_PAYMENT_POINTERS', true)
+  enableSpspPaymentPointers: envBool('ENABLE_SPSP_PAYMENT_POINTERS', true),
+  localCacheDuration: envInt('LOCAL_CACHE_DURATION', 5000)
 }
 
 function parseRedisTlsConfig(