immich/server/src/utils/instrumentation.ts

import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';
import { NodeSDK, contextBase, metrics, resources } from '@opentelemetry/sdk-node';
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
import { snakeCase, startCase } from 'lodash';
import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils';
import { performance } from 'node:perf_hooks';
import { excludePaths, serverVersion } from 'src/constants';
import { DecorateAll } from 'src/decorators';

let metricsEnabled = process.env.IMMICH_METRICS === 'true';
export const hostMetrics =
  process.env.IMMICH_HOST_METRICS == null ? metricsEnabled : process.env.IMMICH_HOST_METRICS === 'true';
export const apiMetrics =
  process.env.IMMICH_API_METRICS == null ? metricsEnabled : process.env.IMMICH_API_METRICS === 'true';
export const repoMetrics =
  process.env.IMMICH_IO_METRICS == null ? metricsEnabled : process.env.IMMICH_IO_METRICS === 'true';
export const jobMetrics =
  process.env.IMMICH_JOB_METRICS == null ? metricsEnabled : process.env.IMMICH_JOB_METRICS === 'true';

metricsEnabled ||= hostMetrics || apiMetrics || repoMetrics || jobMetrics;
if (!metricsEnabled && process.env.OTEL_SDK_DISABLED === undefined) {
  process.env.OTEL_SDK_DISABLED = 'true';
}

const aggregation = new metrics.ExplicitBucketHistogramAggregation(
  [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000],
  true,
);

let otelSingleton: NodeSDK | undefined;

export const otelStart = (port: number) => {
  if (otelSingleton) {
    throw new Error('OpenTelemetry SDK already started');
  }
  otelSingleton = new NodeSDK({
    resource: new resources.Resource({
      [SemanticResourceAttributes.SERVICE_NAME]: `immich`,
      [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(),
    }),
    metricReader: new PrometheusExporter({ port }),
    contextManager: new AsyncLocalStorageContextManager(),
    instrumentations: [
      new HttpInstrumentation(),
      new IORedisInstrumentation(),
      new NestInstrumentation(),
      new PgInstrumentation(),
    ],
    views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })],
  });
  otelSingleton.start();
};

export const otelShutdown = async () => {
  if (otelSingleton) {
    await otelSingleton.shutdown();
    otelSingleton = undefined;
  }
};

export const otelConfig: OpenTelemetryModuleOptions = {
  metrics: {
    hostMetrics,
    apiMetrics: {
      enable: apiMetrics,
      ignoreRoutes: excludePaths,
    },
  },
};

function ExecutionTimeHistogram({
  description,
  unit = 'ms',
  valueType = contextBase.ValueType.DOUBLE,
}: contextBase.MetricOptions = {}) {
  return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
    if (!repoMetrics || process.env.OTEL_SDK_DISABLED) {
      return;
    }

    const method = descriptor.value;
    const className = target.constructor.name as string;
    const propertyName = String(propertyKey);
    const metricName = `${snakeCase(className).replaceAll(/_(?=(repository)|(controller)|(provider)|(service)|(module))/g, '.')}.${snakeCase(propertyName)}.duration`;

    const metricDescription =
      description ??
      `The elapsed time in ${unit} for the ${startCase(className)} to ${startCase(propertyName).toLowerCase()}`;

    let histogram: contextBase.Histogram | undefined;

    descriptor.value = function (...args: any[]) {
      const start = performance.now();
      const result = method.apply(this, args);

      void Promise.resolve(result)
        .then(() => {
          const end = performance.now();
          if (!histogram) {
            histogram = contextBase.metrics
              .getMeter('immich')
              .createHistogram(metricName, { description: metricDescription, unit, valueType });
          }
          histogram.record(end - start, {});
        })
        .catch(() => {
          // noop
        });

      return result;
    };

    copyMetadataFromFunctionToFunction(method, descriptor.value);
  };
}

export const Instrumentation = () => DecorateAll(ExecutionTimeHistogram());