feat(relay): Add performance prometheus metrics

metacraft-labs · Oct 9, 2024 · 2f65ced · 2f65ced
1 parent b266124
commit 2f65ced
Show file tree

Hide file tree

Showing 4 changed files with 69 additions and 1 deletion.
diff --git a/beacon-light-client/solidity/tasks/start-publishing.ts b/beacon-light-client/solidity/tasks/start-publishing.ts
@@ -15,6 +15,7 @@ import {
   accountBalanceGauge,
   initPrometheusSetup,
   registerGaugesForStartPublishing,
+  startResourceMetricsUpdate,
 } from '@dendreth/utils/ts-utils/prometheus-utils';
 
 const logger = getGenericLogger();
@@ -78,6 +79,7 @@ task('start-publishing', 'Run relayer')
 
       initPrometheusSetup(args.prometheusPort, networkName);
       registerGaugesForStartPublishing();
+      startResourceMetricsUpdate(networkName);
     }
 
     const currentConfig = await getNetworkConfig(args.followNetwork);

diff --git a/libs/typescript/ts-utils/prometheus-utils.ts b/libs/typescript/ts-utils/prometheus-utils.ts
@@ -157,3 +157,63 @@ export const numberOfProofPublished = new client.Counter({
   help: 'The number of proofs published(since last restart)',
   labelNames: ['network'],
 });
+
+const gaugeCpuUser = new client.Gauge({
+  name: 'process_cpu_user_seconds_total',
+  help: 'Total user CPU time spent by the process in seconds.',
+  labelNames: ['label'], // Adding label
+});
+
+const gaugeCpuSystem = new client.Gauge({
+  name: 'process_cpu_system_seconds_total',
+  help: 'Total system CPU time spent by the process in seconds.',
+  labelNames: ['label'], // Adding label
+});
+
+const gaugeMemoryUsage = new client.Gauge({
+  name: 'process_memory_bytes',
+  help: 'Memory usage of the process in bytes.',
+  labelNames: ['label'], // Adding label
+});
+
+const gaugeHeapTotal = new client.Gauge({
+  name: 'process_heap_total_bytes',
+  help: 'Total heap memory allocated by the process in bytes.',
+  labelNames: ['label'], // Adding label
+});
+
+const gaugeHeapUsed = new client.Gauge({
+  name: 'process_heap_used_bytes',
+  help: 'Heap memory used by the process in bytes.',
+  labelNames: ['label'], // Adding label
+});
+
+const gaugeRss = new client.Gauge({
+  name: 'process_rss_bytes',
+  help: 'Resident set size (RSS) memory used by the process in bytes.',
+  labelNames: ['label'], // Adding label
+});
+
+export function startResourceMetricsUpdate(label: string) {
+  register.registerMetric(gaugeCpuUser);
+  register.registerMetric(gaugeCpuSystem);
+  register.registerMetric(gaugeMemoryUsage);
+  register.registerMetric(gaugeHeapTotal);
+  register.registerMetric(gaugeRss);
+
+  // Set an interval to regularly update the resource metrics
+  setInterval(() => {
+    const memoryUsage = process.memoryUsage();
+    const cpuUsage = process.cpuUsage();
+
+    // Update CPU metrics
+    gaugeCpuUser.labels(label).set(cpuUsage.user / 1e6); // Convert from microseconds to seconds
+    gaugeCpuSystem.labels(label).set(cpuUsage.system / 1e6); // Convert from microseconds to seconds
+
+    // Update memory metrics
+    gaugeMemoryUsage.labels(label).set(memoryUsage.rss); // Resident set size (RSS)
+    gaugeHeapTotal.labels(label).set(memoryUsage.heapTotal); // Total heap allocated
+    gaugeHeapUsed.labels(label).set(memoryUsage.heapUsed); // Heap memory in use
+    gaugeRss.labels(label).set(memoryUsage.rss); // RSS memory in bytes
+  }, 5000); // Update metrics every 5 seconds
+}
diff --git a/relay/workers/poll-updates/poll-updates-worker.ts b/relay/workers/poll-updates/poll-updates-worker.ts
@@ -9,10 +9,14 @@ import doUpdate from '@/workers/poll-updates/do_update';
 import { getBeaconApi } from '@/implementations/beacon-api';
 import { checkConfig } from '@dendreth/utils/ts-utils/common-utils';
 import { getGenericLogger } from '@dendreth/utils/ts-utils/logger';
-import { initPrometheusSetup } from '@dendreth/utils/ts-utils/prometheus-utils';
+import {
+  initPrometheusSetup,
+  startResourceMetricsUpdate,
+} from '@dendreth/utils/ts-utils/prometheus-utils';
 
 const logger = getGenericLogger();
 initPrometheusSetup();
+startResourceMetricsUpdate('poll-update-worker');
 
 (async () => {
   const updatePollingConfig = {

diff --git a/relay/workers/prover/prover-worker.ts b/relay/workers/prover/prover-worker.ts
@@ -10,6 +10,7 @@ import yargs from 'yargs';
 import {
   initPrometheusSetup,
   registerGaugesForProver,
+  startResourceMetricsUpdate,
 } from '@dendreth/utils/ts-utils/prometheus-utils';
 
 (async () => {
@@ -19,6 +20,7 @@ import {
   };
   initPrometheusSetup(3000);
   registerGaugesForProver();
+  startResourceMetricsUpdate('proofGenerationWorker');
 
   checkConfig(proverConfig);