Spring Boot Actuator と Grafana Stack が連携できるように設定する

...
dependencies {
    implementation("org.springframework.boot:spring-boot-starter-actuator")
}
...

@Configuration
@EnableWebSecurity
class SecurityConfig {
    @Bean
    fun securityFilterChain(http: HttpSecurity): SecurityFilterChain {

        http.authorizeHttpRequests {
            it.requestMatchers("/csrf_token", "/actuator/**").permitAll()
            it.requestMatchers("/admin/**").hasAuthority(RoleType.ADMIN.toString())
            it.anyRequest().authenticated()
       }
       ....
}

management:
  server:
    port: 8081

management:
  endpoints:
    web:
      exposure:
        include:
          - health
          - prometheus

dependencies {
    runtimeOnly("io.micrometer:micrometer-registry-prometheus")
}

# HELP application_ready_time_seconds Time taken (ms) for the application to be ready to service requests
# TYPE application_ready_time_seconds gauge
application_ready_time_seconds{main_application_class="com.book.manager.BookManagerApplicationKt",} 2.416
# HELP jvm_gc_max_data_size_bytes Max size of long-lived heap memory pool
# TYPE jvm_gc_max_data_size_bytes gauge
jvm_gc_max_data_size_bytes 8.589934592E9
# HELP jvm_classes_unloaded_classes_total The total number of classes unloaded since the Java virtual machine has started execution
# TYPE jvm_classes_unloaded_classes_total counter
jvm_classes_unloaded_classes_total 0.0
# HELP jvm_memory_usage_after_gc_percent The percentage of long-lived heap pool used after the last GC event, in the range [0..1]
# TYPE jvm_memory_usage_after_gc_percent gauge
jvm_memory_usage_after_gc_percent{area="heap",pool="long-lived",} 0.005299091339111328
# HELP tomcat_sessions_created_sessions_total  
# TYPE tomcat_sessions_created_sessions_total counter
tomcat_sessions_created_sessions_total 0.0
# HELP jvm_buffer_total_capacity_bytes An estimate of the total capacity of the buffers in this pool
# TYPE jvm_buffer_total_capacity_bytes gauge
jvm_buffer_total_capacity_bytes{id="mapped - 'non-volatile memory'",} 0.0
jvm_buffer_total_capacity_bytes{id="mapped",} 0.0
jvm_buffer_total_capacity_bytes{id="direct",} 16384.0

global:
  # 15secごとにメトリクスを取得しにいく
  scrape_interval: 15s
scrape_configs:
  - job_name: "spring-actuator"
    metrics_path: "/actuator/prometheus"
    static_configs:
      - targets: [ 'host.docker.internal:8081' ]

services:
  prometheus:
    image: prom/prometheus
    ports:
      - "9090:9090"
    volumes:
      - "./config/prometheus.yml:/etc/prometheus/prometheus.yml"

services:
  prometheus:
    image: prom/prometheus
    container_name: prometheus
    ports:
      - "9090:9090"
    volumes:
      - ./config/prometheus.yml:/etc/prometheus/prometheus.yml
  grafana:
    image: grafana/grafana
    container_name: grafana
    ports:
      - "3000:3000"
    volumes:
      - grafana-storage:/var/lib/grafana
volumes:
  grafana-storage:

dependencies {
    runtimeOnly("io.micrometer:micrometer-tracing-bridge-otel")
    runtimeOnly("io.opentelemetry:opentelemetry-exporter-zipkin")
}

management:
  ...
  tracing:
    sampling:
      probability: 1.0

services:
  tempo:
    image: grafana/tempo
    container_name: tempo
    command: [ "-config.file=/etc/tempo.yml" ]
    volumes:
      - ./config/tempo.yml:/etc/tempo.yml
      - tempo-data:/tmp/tempo
    ports:
      - "3200:3200"   # tempo
      - "9411:9411"   # zipkin
volumes:
  tempo-data:

server:
  http_listen_port: 3200  # tempoサーバーの公開ポート

distributor:
  receivers:
    zipkin:   # トレースのレシーバーとしてZipkin互換のAPIを利用するための設定

storage:
  trace:
    backend: local  # tempoのデータストレージはサーバーローカルとする
    local:
      path: /tmp/tempo/blocks  # ローカルストレージの保存先

logging:
  level:
    root: info
  pattern:
    console: "%d{yyyy-MM-dd HH:mm:ss.SSS} %highlight{%-5level}[${spring.application.name:},%X{traceId},%X{spanId}] [%style{%t}{bright,blue}] %style{%C}{bright,yellow}: %m%n"

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <Appenders>
        <Console name="Console" target="SYSTEM_OUT">
            <PatternLayout pattern="${spring:logging.pattern.console}"/>-->
        </Console>
    </Appenders>
    <Loggers>
        <Root level="${spring:logging.level.root}">
            <AppenderRef ref="Console"/>
        </Root>
    </Loggers>
</configuration>

logging:
  level:
    root: info
  pattern:
    console: "%d{${logging.pattern.dateformat}} %highlight{${logging.pattern.level}} [%style{%t}{bright,blue}] %style{%C}{bright,yellow}: %m%n"
    file: "%d{${logging.pattern.dateformat}} ${logging.pattern.level} [%t] %C: %m%n"
    level: "%5p [${spring.application.name:},%X{traceId:-},%X{spanId:-}]"
    dateformat: yyyy-MM-dd'T'HH:mm:ss.SSSXXX
  file:
    path: ./logs
    name: app.log

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <Appenders>
        <Console name="Console" target="SYSTEM_OUT">
            <PatternLayout>
                <pattern>${sys:CONSOLE_LOG_PATTERN}</pattern>
            </PatternLayout>
        </Console>
        <RollingFile name="RollingFile"
                     fileName="${sys:LOG_PATH}/${sys:LOG_FILE}"
                     filePattern="${sys:LOG_PATH}/$${date:yyyy-MM}/${sys:LOG_FILE}-%d{yyyy-MM-dd}-%i.gz">
            <PatternLayout>
                <pattern>${sys:FILE_LOG_PATTERN}</pattern>
            </PatternLayout>
            <Policies>
                <SizeBasedTriggeringPolicy size="10 MB"/>
                <TimeBasedTriggeringPolicy/>
            </Policies>
        </RollingFile>
    </Appenders>
    <Loggers>
        <Root level="${spring:logging.level.root}">
            <AppenderRef ref="Console"/>
            <AppenderRef ref="RollingFile"/>
        </Root>
    </Loggers>
</configuration>

services:
  loki:
    image: grafana/loki
    container_name: loki
    command: [ "-config.file=/etc/config/loki-config.yml" ]
    environment:
      TZ: Asia/Tokyo
    volumes:
      - ./config/loki-config.yml:/etc/config/loki-config.yml
    networks:
      - datasource
    ports:
      - "3100:3100"
  grafana:
    image: grafana/grafana
    container_name: grafana
    environment:
      TZ: Asia/Tokyo
    volumes:
      - grafana-storage:/var/lib/grafana
    networks:
      - grafana
      - datasource
    ports:
      - "3000:3000"
volumes:
  grafana-storage:
  tempo-data:
networks:
  # Grafanaのネットワーク
  grafana:
    driver: bridge
    internal: false
    name: grafana_network
  # GrafanaのDataSources(Prometheus, Tempo, Loki)のネットワーク。SpringBootActuator と GrafanaAgentからアクセスするために公開
  datasource:
    driver: bridge
    internal: false
    name: datasources_network

auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096

common:
  instance_addr: 127.0.0.1
  path_prefix: /tmp/loki
  storage:
    filesystem:
      chunks_directory: /tmp/loki/chunks
      rules_directory: /tmp/loki/rules
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

query_range:
  results_cache:
    cache:
      embedded_cache:
        enabled: true
        max_size_mb: 100

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

ruler:
  alertmanager_url: http://localhost:9093

services:
  backend:
    build:
      context: ../../
      dockerfile: ./docker/backend/app/Dockerfile
    container_name: backend
    hostname: backend.app
    environment:
      TZ: Asia/Tokyo
    volumes:
      - app-logs:/app/logs
    networks:
      - app
      - db_external
      - redis_external
      - datasources_external  # otel-exporter-zipkin で Tempoにアクセスするため
    ports:
      - "8080:8080"
      - "8081:8081"
  promtail:
    image: grafana/promtail
    container_name: promtail
    command: [ "-config.file=/etc/promtail.yml" ]
    environment:
      TZ: Asia/Tokyo
    volumes:
      - ./config/promtail.yml:/etc/promtail.yml
      - app-logs:/var/logs
    networks:
      - agent
      - app
      - datasources_external
    ports:
      - "9080:9080"
volumes:
  app-logs:
    driver: local
networks:
  # アプリのネットワーク。ホストから http:localhost:8080 にアクセスするため公開
  app:
    driver: bridge
    internal: false
    name: app_network
  # DBとRedisのネットワーク
  db_external:
    external: true
    name: db_network
  redis_external:
    external: true
    name: redis_network
  # Grafana Agent のネットワーク
  agent:
    driver: bridge
    internal: true
    name: agent_network
  # GrafanaのDataSourceネットワークに接続定義
  datasources_external:
    external: true
    name: datasources_network

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
  - job_name: app
    pipeline_stages:
    static_configs:
      - targets:
          - localhost
        labels:
          job: app
          host: app
          __path__: /var/logs/*.log

scrape_configs:
  - job_name: app
    pipeline_stages:
      - regex:
          expression: "^(?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}\\+\\d{2}:\\d{2})\\s+(?P<level>\\w+)\\s+\\[(?P<app>[\\w-]+),(?P<trace_id>[\\w-]*),(?P<span_id>[\\w-]*)\\]\\s+\\[(?P<thread>[\\w-]+)\\]\\s+(?P<classname>[\\w.-]*):\\s+(?P<message>.*)$"
      - labels:
          level:
          app:
          trace_id:
          span_id:

import io.micrometer.observation.ObservationRegistry
import io.micrometer.observation.aop.ObservedAspect
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration

@Configuration
class ObservationConfiguration {

    @Bean
    fun observedAspect(registry: ObservationRegistry) = ObservedAspect(registry)
}

@Bean
    fun redisConnectionFactory(meterRegistry: MeterRegistry): LettuceConnectionFactory {
        val configuration = RedisStandaloneConfiguration(redisHostName, redisPort)
        val clientResources: ClientResources = ClientResources.builder()
            .commandLatencyRecorder(MicrometerCommandLatencyRecorder(meterRegistry, MicrometerOptions.create()))
            .build()
        val lettuceClientConfiguration: LettuceClientConfiguration = LettuceClientConfiguration.builder()
            .clientResources(clientResources)
            .build()
        return LettuceConnectionFactory(configuration, lettuceClientConfiguration)
    }

//    runtimeOnly("io.micrometer:micrometer-tracing-bridge-otel")
//    runtimeOnly("io.opentelemetry:opentelemetry-exporter-zipkin")
    runtimeOnly("io.micrometer:micrometer-tracing-bridge-brave")
    runtimeOnly("io.zipkin.reporter2:zipkin-reporter-brave")

services:
  prometheus:
    image: prom/prometheus
    container_name: prometheus
    command:
      - --enable-feature=exemplar-storage
      - --config.file=/etc/prometheus/prometheus.yml
    environment:
      TZ: Asia/Tokyo
    volumes:
      - ./config/prometheus.yml:/etc/prometheus/prometheus.yml:ro
    networks:
      - datasource
    ports:
      - "9090:9090"

apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://host.docker.internal:9090
    editable: false
    jsonData:
      httpMethod: POST
      manageAlerts: true
      prometheusType: Prometheus
      prometheusVersion: 2.43.0
      exemplarTraceIdDestinations:
        - datasourceUid: Tempo
          name: trace_id
  - name: Tempo
    type: tempo
    access: proxy
    url: http://host.docker.internal:3200
    editable: false
    jsonData:
      httpMethod: GET
      tracesToLogs:
        datasourceUid: 'Loki'
        tags: [ 'job', 'instance', 'pod', 'namespace' ]
        mappedTags: [ { key: 'traceId', value: 'trace_id' } ]
        mapTagNamesEnabled: true
        spanStartTimeShift: '-1h'
        spanEndTimeShift: '1h'
        filterByTraceID: true
        filterBYSpanID: false
      serviceMap:
        datasourceUid: 'Prometheus'
      search:
        hide: false
      nodeGraph:
        enabled: true
      lokiSearch:
        datasourceUid: 'Loki'
  - name: Loki
    type: loki
    access: proxy
    url: http://host.docker.internal:3100
    editable: false
    jsonData:
      maxLines: 1000
      derivedFields:
        - datasourceUid: Tempo
          matcherRegex: "\\[[\\w-]+,(\\w*),"
          name: TraceID
          url: "$${__value.raw}"

  grafana:
    image: grafana/grafana
    container_name: grafana
    environment:
      TZ: Asia/Tokyo
    volumes:
      - grafana-storage:/var/lib/grafana
      - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
    networks:
      - grafana
      - datasource
    ports:
      - "3000:3000"

services:
  app:
    build:
      context: ../../
      dockerfile: ./docker/backend/app/Dockerfile
    container_name: backend
    hostname: backend.app
    environment:
      TZ: Asia/Tokyo
    volumes:
      - app-logs:/app/logs
    networks:
      - app
      - db_external
      - redis_external
    ports:
      - "8080:8080"
      - "8081:8081"
  grafana-agent:
    image: grafana/agent
    container_name: grafana-agent
    command:
      - --config.file=/etc/agent/agent.yml
    environment:
      TZ: Asia/Tokyo
    volumes:
      - ./agent/agent.yml:/etc/agent/agent.yml:lo
      - agent-store:/etc/agent/data
      - app-logs:/var/logs
    networks:
      - agent
      - app
      - db_external
      - redis_external
      - datasources_external
    ports:
      - "12345:12345"
      - "9411"     # zipkin receiver for Backend
volumes:
  app-logs:
    driver: local
  agent-store:
networks:
  app:
    driver: bridge
    internal: false
    name: app_network
  # DBとRedisのネットワーク
  db_external:
    external: true
    name: db_network
  redis_external:
    external: true
    name: redis_network
  # Grafana Agent のネットワーク
  agent:
    driver: bridge
    internal: true
    name: agent_network
  # GrafanaのDataSourceネットワークに接続定義
  datasources_external:
    external: true
    name: datasources_network

server:
  log_level: info

metrics:
  wal_directory: /etc/agent/data
  global:
    scrape_interval: 2s
  configs:
    - name: default
      scrape_configs:
        - job_name: "spring-actuator"
          metrics_path: "/actuator/prometheus"
          static_configs:
            - targets: [ "app:8081" ]
      remote_write:
        - url: http://mimir:9009/api/v1/push
          send_exemplars: true

traces:
  configs:
    - name: default
      receivers:
        zipkin:
      remote_write:
        - endpoint: "tempo:4317"
          insecure: true

logs:
  configs:
    - name: default
      positions:
        filename: /tmp/positions.yml
      scrape_configs:
        - job_name: app
          pipeline_stages:
            - regex:
                expression: "^(?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}\\+\\d{2}:\\d{2})\\s+(?P<level>\\w+)\\s+\\[(?P<app>[\\w-]+),(?P<trace_id>[\\w-]*),(?P<span_id>[\\w-]*)\\]\\s+\\[(?P<thread>[\\w-]+)\\]\\s+(?P<classname>[\\w.-]*):\\s+(?P<message>.*)$"
            - labels:
                level:
                app:
                trace_id:
                span_id:
          static_configs:
            - targets:
                - localhost
              labels:
                job: app
                host: app
                __path__: /var/logs/*.log
      clients:
        - url: http://loki:3100/loki/api/v1/push

services:
  mimir:
    image: grafana/mimir
    container_name: mimir
    command: [ "-config.file=/etc/mimir.yml" ]
    environment:
      TZ: Asia/Tokyo
    volumes:
      - ./config/mimir.yml:/etc/mimir.yml:ro
      - mimir-storage:/tmp/mimir
    networks:
      - datasource
    ports:
      - "9009:9009"

multitenancy_enabled: false

limits:
  max_global_exemplars_per_user: 100000

blocks_storage:
  backend: filesystem
  bucket_store:
    sync_dir: /tmp/mimir/tsdb-sync
  filesystem:
    dir: /tmp/mimir/data/tsdb
  tsdb:
    dir: /tmp/mimir/tsdb

compactor:
  data_dir: /tmp/mimir/compactor
  sharding_ring:
    kvstore:
      store: memberlist

distributor:
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: memberlist

ingester:
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: memberlist
    replication_factor: 1

ruler_storage:
  backend: filesystem
  filesystem:
    dir: /tmp/mimir/rules

server:
  http_listen_port: 9009
  log_level: info

store_gateway:
  sharding_ring:
    replication_factor: 1

datasources:
  - name: Prometheus(Mimir)
    type: prometheus
    access: proxy
    url: http://host.docker.internal:9009/prometheus
    editable: false
    jsonData:
      httpMethod: POST
      manageAlerts: true
      prometheusType: Mimir
      prometheusVersion: 2.7.x
      exemplarTraceIdDestinations:
        - datasourceUid: Tempo
          name: trace_id

metrics:
  wal_directory: /etc/agent/data
  global:
    scrape_interval: 2s
  configs:
    - name: default
      scrape_configs:
        - job_name: "spring-actuator"
          metrics_path: "/actuator/prometheus"
          static_configs:
            - targets: [ "app:8081" ]
      remote_write:
        - url: http://mimir:9009/api/v1/push
          send_exemplars: true

server:
  http_listen_port: 3200

distributor:
  receivers:
    otlp:
      protocols:
        grpc:

storage:
  trace:
    backend: local
    local:
      path: /tmp/tempo/blocks

  tempo:
    image: grafana/tempo
    container_name: tempo
    command: [ "-config.file=/etc/tempo.yml" ]
    environment:
      TZ: Asia/Tokyo
    volumes:
      - ./config/tempo.yml:/etc/tempo.yml:ro
      - tempo-data:/tmp/tempo
    networks:
      - datasource
    ports:
      - "3200:3200"   # tempo
      - "4317:4317"   # for Grafana Agent exports via gRPC using OTLP

traces:
  configs:
    - name: default
      receivers:
        zipkin:
      remote_write:
        - endpoint: "tempo:4317"
          insecure: true

  grafana-agent:
    image: grafana/agent
    container_name: grafana-agent
    command:
      - --config.file=/etc/agent/agent.yml
    environment:
      TZ: Asia/Tokyo
    volumes:
      - ./agent/agent.yml:/etc/agent/agent.yml:lo
      - agent-store:/etc/agent/data
      - app-logs:/var/logs
    networks:
      - agent
      - app
      - db_external
      - redis_external
      - datasources_external
    ports:
      - "12345:12345"
      - "9411"     # zipkin receiver for Backend

logs:
  configs:
    - name: default
      positions:
        filename: /tmp/positions.yml
      scrape_configs:
        - job_name: app
          pipeline_stages:
            - regex:
                expression: "^(?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}\\+\\d{2}:\\d{2})\\s+(?P<level>\\w+)\\s+\\[(?P<app>[\\w-]+),(?P<trace_id>[\\w-]*),(?P<span_id>[\\w-]*)\\]\\s+\\[(?P<thread>[\\w-]+)\\]\\s+(?P<classname>[\\w.-]*):\\s+(?P<message>.*)$"
            - labels:
                level:
                app:
                trace_id:
                span_id:
          static_configs:
            - targets:
                - localhost
              labels:
                job: app
                host: app
                __path__: /var/logs/*.log
      clients:
        - url: http://loki:3100/loki/api/v1/push

integrations:
  prometheus_remote_write:
    - url: http://mimir:9009/api/v1/push

integrations:
  postgres_exporter:
    enabled: true
    data_source_names:
      - "postgresql://<usre>:<password>@<host>:<port>/<database_name>?sslmode=disable"

integrations:
  redis_exporter:
    enabled: true
    redis_addr: "<host>:<port>"