docker 모니터링 (prometheus,grafana,cadvisor)

오늘 할작업은 4대의 서버의 서버 상태 및 도커상태 모니터링이며
cadvisor 도커상태
node_exporter 서버상태
모니터링이다 다음엔 로그도 모니터링해볼예정

 

 

 

어쩌다 보니 서버가 총 4대가 되었다.

 

NAS서버(데이터 및 amd64 용도커)

라즈베리파이(내부망 + 외부망연결용 + arm용 집도커)

OCI1,2 (오라클 A1 서버 + ARM 외부도커)

이 4개를 모니터링하기위하여 연결하여보았다.

 

사전설정

>> OCI에서 우리집 ip를 방화벽허용해줬다. 다른곳에서는 접속안되게 설정한것이다.

 

방식은 이렇다

나스서버에서 모든 데이터를 수집하며

라즈베리+OCI1,2의 정보를 수집하여 가공 처리 한다.

 

 

 

prometheus.yml 설정파일을 만들어둔다

 

/volume1/SSD_DATA/prome/prometheus.yml

global:
  scrape_interval:     15s # By default, scrape targets every 15 seconds.

scrape_configs:
  - job_name: 'prometheus'
    scrape_interval: 5s
    static_configs:
      - targets: ['localhost:9090']
      
  - job_name: 'node_exporter_nas'
    static_configs:
      - targets: ['node_exporter_nas:9100']
      
  - job_name: 'node_exporter_raspberry'
    static_configs:
      - targets: ['node_exporter_raspberry:9100']

  - job_name: 'node_exporter_oci1'
    static_configs:
      - targets: ['oci1_ipaddr:9100']

  - job_name: 'node_exporter_oci2'
    static_configs:
      - targets: ['oci2_ipaddr:9100']

  - job_name: 'cadvisor_nas'
    static_configs:
      - targets: ['cadvisor_nas:8080']
      
  - job_name: 'cadvisor_raspberry'
    static_configs:
      - targets: ['cadvisor_raspberry:8080']

  - job_name: 'cadvisor_oci1'
    static_configs:
      - targets: ['oci1_ipaddr:8080']
      
  - job_name: 'cadvisor_oci2'
    static_configs:
      - targets: ['oci2_ipaddr:8080']

 

설치 라즈베리파이+NAS 이두개는 도커스웜으로 연결되어있어(내부망이라 연결해둠) 아래 소스로 배포하면 끝이다.

version: '3.2'

volumes:
  prometheus-data:
    driver: local
  grafana-data:
    driver: local

services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    ports:
      - "9090:9090"
    volumes:
      - /volume1/SSD_DATA/prome:/etc/prometheus
      - prometheus-data:/prometheus
    restart: unless-stopped
    command:
      - "--config.file=/etc/prometheus/prometheus.yml"
    networks:
      - moniter_network
    deploy:
      mode: replicated
      replicas: 1
      placement:
        constraints: [node.role == manager]

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    ports:
      - "3000:3000"
    volumes:
      - grafana-data:/var/lib/grafana
    restart: unless-stopped
    networks:
      - moniter_network
    deploy:
      mode: replicated
      replicas: 1
      placement:
        constraints: [node.role == manager]
        
  node_exporter_nas:
    hostname: '{{.Node.Hostname}}'
    image: quay.io/prometheus/node-exporter:latest
    command:
      - '--path.rootfs=/host'
    pid: host
    restart: unless-stopped
    volumes:
      - '/:/host:ro,rslave'
    networks:
      moniter_network:
        aliases:
          - node_exporter_nas
    deploy:
      mode: global
      placement:
        constraints: [node.labels.yos.kr == nas]

  node_exporter_raspberry:
    hostname: '{{.Node.Hostname}}'
    image: quay.io/prometheus/node-exporter:latest
    command:
      - '--path.rootfs=/host'
    pid: host
    restart: unless-stopped
    volumes:
      - '/:/host:ro,rslave'
    networks:
      moniter_network:
        aliases:
          - node_exporter_raspberry
    deploy:
      mode: global
      placement:
        constraints: [node.labels.yos.kr == raspberry]

  cadvisor_raspberry:
    hostname: '{{.Node.Hostname}}'
    image: zcube/cadvisor:latest
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /dev/disk/:/dev/disk:ro
      - /etc/machine-id:/etc/machine-id:ro
    devices:
      - /dev/kmsg
    #restart: unless-stopped
    networks:
      moniter_network:
        aliases:
          - cadvisor_raspberry
    deploy:
      mode: global
      placement:
        constraints: [node.labels.yos.kr == raspberry]

  cadvisor_nas:
    hostname: '{{.Node.Hostname}}'
    image: google/cadvisor:latest
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /volume1/@docker:/var/lib/docker:ro
    #  - /sys/power/disk/:/dev/disk:ro
    devices:
      - /dev/kmsg
    #restart: unless-stopped
    networks:
      moniter_network:
        aliases:
          - cadvisor_nas
    deploy:
      mode: global
      placement:
        constraints: [node.labels.yos.kr == nas]
        
        
networks:
  moniter_network:
    driver: overlay
    attachable: true

OCI 모니터링툴 설치

각각 오라클 서버에 아래와 같이 작성한다.

cat > moniter.yaml << EOF
version: '3.2'
services:
  node_exporter_oci1:
    hostname: '{{.Node.Hostname}}'
    image: quay.io/prometheus/node-exporter:latest
    ports:
      - "9100:9100"
    command:
      - '--path.rootfs=/host'
    pid: host
    restart: unless-stopped
    volumes:
      - '/:/host:ro,rslave'

  cadvisor_oci1:
    hostname: '{{.Node.Hostname}}'
    image: zcube/cadvisor:latest
    ports:
      - "8080:8080"
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /dev/disk/:/dev/disk:ro
      - /etc/machine-id:/etc/machine-id:ro
    devices:
      - /dev/kmsg
    restart: unless-stopped
EOF

docker-compose -f moniter.yaml up -d

 

 

그라나파나에 접속하여 대시보드를 만든다