fix: do not log names of all queues, cardinality is too high

ceramicnetwork · Oct 10, 2024 · ee58239 · ee58239
1 parent eecc1dd
commit ee58239
Showing 1 changed file with 13 additions and 3 deletions.
diff --git a/packages/core/src/state-management/named-task-queue.ts b/packages/core/src/state-management/named-task-queue.ts
@@ -4,6 +4,8 @@ import { ServiceMetrics as Metrics } from '@ceramicnetwork/observability'
 const NAMED_TASK_QUEUE_SIZE = 'named_task_queue_size'
 const NAMED_TASK_QUEUE_RUN = 'named_task_queue_run'
 const NAMED_TASK_QUEUE_ADD = 'named_task_queue_add'
+const NAMED_TASK_QUEUE_LARGE_SIZE = 'named_task_queue_large_size'
+const LARGE_QUEUE_THRESHOLD = 50
 
 /**
  * Set of named PQueues.
@@ -54,8 +56,12 @@ export class NamedTaskQueue {
    */
   run<A>(name: string, task: () => Promise<A>): Promise<A> {
     const queue = this.queue(name)
-    Metrics.observe(NAMED_TASK_QUEUE_SIZE, queue.size, { name: name })
-    Metrics.count(NAMED_TASK_QUEUE_RUN, 1, { name: name })
+    Metrics.observe(NAMED_TASK_QUEUE_SIZE, queue.size, {'method': 'run'})
+    Metrics.count(NAMED_TASK_QUEUE_RUN, 1)
+    // only log the names of queues over a threshold size
+    if (queue.size > LARGE_QUEUE_THRESHOLD) {
+        Metrics.observe(NAMED_TASK_QUEUE_LARGE_SIZE, queue.size, {'name': name, 'method': 'run'})
+    }
     return queue.run(task).finally(() => {
       this.remove(name)
     })
@@ -69,8 +75,12 @@ export class NamedTaskQueue {
    */
   add(name: string, task: () => Promise<void>): void {
     const queue = this.queue(name)
-    Metrics.observe(NAMED_TASK_QUEUE_SIZE, queue.size, { name: name })
+    Metrics.observe(NAMED_TASK_QUEUE_SIZE, queue.size, {'method': 'add'})
     Metrics.count(NAMED_TASK_QUEUE_ADD, 1, { name: name })
+    if (queue.size > LARGE_QUEUE_THRESHOLD) {
+        Metrics.observe(NAMED_TASK_QUEUE_LARGE_SIZE, queue.size, {'name': name, 'method': 'add'})
+    }
+
     queue.add(
       () => task(),
       () => this.remove(name)