Prometheus、Alertmanager、Grafana 监控 Linux 与 MySQL

//部署各个模块与应用

cd /usr/local/prometheus_compose
vim Docker-compose.yml
version: "3"

services:
  prom:
    image: quay.io/prometheus/prometheus:latest
    container_name: prometheus
    volumes:
     - ./prometheus:/etc/prometheus
    command: "--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus"
    ports:
     - 9090:9090 
    depends_on:
     - exporter
    environment: 
     - TZ=Asia/Shanghai

  exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
    hostname: cicd
    ports:
     - "9100:9100"
    environment:
     - TZ=Asia/Shanghai

  grafana:
    image: grafana/grafana
    container_name: grafana
    ports:
     - "3000:3000"
    environment:
    - “GF_SECURITY_ADMIN_PASSWord=123123”
    - “GF_INSTALL_PLUGINS=alexanderzobnin-zabbix-app”
    - TZ=Asia/Shanghai
    restart: "always"
    volumes:
    - ./grafana:/etc/grafana/”
    - ./grafana/conf/grafana.ini:/etc/grafana/grafana.ini
    - ./grafana/data:/var/lib/grafana:rw
    - ./grafana/plugins:/var/lib/grafana/plugins:rw
    - /etc/localtime:/etc/localtime
    depends_on:
      - prom

  alertmanager:
    image: prom/alertmanager:latest
    container_name: alertmanager
    hostname: alertmanager
    restart: always
    ports:
      - ‘9093:9093‘
    volumes:
      - ‘./alertmanager/config:/etc/alertmanager‘
      - ‘./alertmanager/data:/alertmanager/data‘
    command:
      - ‘--config.file=/etc/alertmanager/alertmanager.yml‘
    environment:
     - TZ=Asia/Shanghai

  cadvisor:
    image: Google/cadvisor
    container_name: cadvisors
    restart: always
    volumes:
       - /:/rootfs:ro
       - /var/run:/var/run:rw
       - /sys:/sys:ro
       - /var/lib/docker/:/var/lib/docker:ro
    ports: 
      - 8080:8080
    privileged: true
    environment:
    - TZ=Asia/Shanghai

  grafana-reporter:
     image: izakmarais/grafana-reporter
     container_name: grafana_reporter
     ports:
     - 8686:8686

     command: "-ip grafana.mitaiot.com"
     environment:
     - TZ=Asia/Shanghai

//编辑报警模块的配置文件

cd /usr/local/Prometheus_compose/alertmanager/config
cat alertmanager.yml
global:
  resolve_timeout: 5m
  smtp_from: ‘123456789@sina.com‘
  smtp_smarthost: ‘smtp.sina.com:587‘
  smtp_auth_username: ‘123456789@sina.com‘
  smtp_auth_password: ‘aabbccdd‘
  smtp_require_tls: false   
  smtp_hello: ‘sina.com‘

route:
  group_by: [‘alertname‘]
  group_wait: 5s
  group_interval: 5s
  repeat_interval: 5m
  receiver: ‘email‘

receivers:
- name: ‘email‘
  email_configs:
   - to: ‘{{ template "email.to" . }}‘
     html: ‘{{ template "email.to.html" . }}‘
     send_resolved: true

inhibit_rules:
  - source_match:
      severity: ‘critical‘
    target_match:
      severity: ‘warning‘
    equal: [‘alertname‘, ‘dev‘, ‘instance‘]

templates:
 - "/etc/alertmanager/alertmanager-tmpl/email.tmpl"

//编辑发送的邮件模板

cd /usr/local/Prometheus_compose/alertmanager/config/alertmanager-tmpl
cat email.tmpl 
{{ define "email.from" }}123456789@sina.com{{ end }}
{{ define "email.to" }}123456789@sina.com{{ end }}
{{ define "email.to.html" }}
{{ range .Alerts }}
=========start==========

告警程序: prometheus_alert 

告警级别: {{ .Labels.severity }} 级 

告警类型: {{ .Labels.alertname }} 

故障主机: {{ .Labels.instance }} 

告警主题: {{ .Annotations.summary }} 

告警详情: {{ .Annotations.description }} 

触发时间: {{ .StartsAt.FORMat "2019-08-04 16:58:15" }} 

=========end==========

{{ end }}
{{ end }}

cd /usr/local/Prometheus_compose/grafana/conf
vim grafana.ini    # 配置文件太长，标出修改部分
[auth.anonymous]
enabled = true
org_name = Main Org.
org_role = Viewer
[smtp]
enabled = true
host = smtp.sina.com:587
user = 123456789@sina.com
password =dc28ac6ec64af9c1
skip_verify = true
from_address = 123456789@sina.com
from_name = Grafana
ehlo_identity =
default_timezone = Asia/Shanghai  #添加时区

//修改 普罗米修斯 的配置文件

cd /usr/local/Prometheus_compose/prometheus

global:
  scrape_interval:     5s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 5s # Evaluate rules every 15 seconds. The default is every 1 minute.

scrape_configs:
    - job_name: ‘prometheus‘
      static_configs:
      - targets: [‘172.16.225.154:9090‘]   

    - job_name: ‘node‘
      file_sd_configs:
      - files: [‘/etc/prometheus/groups/nodegroups/*.json‘]
      static_configs:
      - targets:
          - ‘172.16.225.154:9100‘
          - ‘172.16.225.156:9100‘
          - ‘172.16.225.155:9100‘
          - ‘172.16.225.157:9100‘
          - ‘172.16.225.156:8085‘
          - ‘172.16.225.154:8080‘
          - ‘172.16.225.155:8085‘
          - ‘172.16.225.157:8085‘
          - ‘172.16.225.157:9104‘

alerting:
  alertmanagers:
  - static_configs:
    - targets:
       - ‘172.16.225.154:9093‘

rule_files:
 - "/etc/prometheus/rules/*.yml"

//配置报警规则
cd /usr/local/Prometheus_compose/prometheus/rules
groups:
- name: node-up
  rules:
  - alert: node-up
    expr: up{job="node"} == 0
    for: 15s
    labels:
      severity: 1
      team: node
    annotations:
      summary: "{{ $labels.instance }} 已停止运行!"
      description: "{{ $labels.instance }} 检测到异常停止！请重点关注！！！" 

- name: node-cpu
  rules:
  - alert: node-cpu
    expr: 100 - ((avg by (instance,job,env)(irate(node_cpu_seconds_total{mode="idle"}[30s]))) *100) > 90
    for: 1m
    labels:
      severity: 1
      team: node
      level: warning
    annotations:
      summary: "{{ $labels.instance }} CPU使用率超过 百分之90!"
      description: "{{ $labels.instance }} 检测CPU连续1分钟占用率超出90%！请重点关注！！！"

- name: node-mem
  rules:
  - alert: node-mem
    expr: ((node_memory_MemTotal_bytes -(node_memory_MemFree_bytes+node_memory_Buffers_bytes+node_memory_Cached_bytes) )/node_memory_MemTotal_bytes ) * 100 > 90 #设置内存使用率高于90时发送告警，计算方式为 总内存-空闲内存 - buffers - cached
    for: 5s
    labels:
      severity: 1
      team: node
      level: warning
    annotations:
      summary: "{{ $labels.instance }} MEM使用率超过 百分之90!"
      description: "{{ $labels.instance }} 检测CPU连续1分钟占用率超出90%！请重点关注！！！"

- name: node-disk_used
  rules:
  - alert: node-disk_used
    expr: 100 - (node_filesystem_free_bytes{fstype=~"ext3|ext4|xfs"} / node_filesystem_size_bytes{fstype=~"ext3|ext4|xfs"} * 100) > 90  #设置挂载分区使用率为95以上时告警
    for: 1m
    labels:
      severity: 1
      team: node
      level: warning
    annotations:
      summary: "{{ $labels.instance }} 挂在分区使用率超过 百分之90!"
      description: "{{ $labels.instance }} 挂在分区使用率超出90%！请重点关注！！！"

#如需监控 Mysql和容器和主机信息 需要在 主机上部署 prom/node-exporter 、cadvisor、prom/mysqld-exporter。

version: "3"

services:
  exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
    hostname: db01
    ports:
     - "9100:9100"

  cadvisor:
    image: google/cadvisor
    container_name: cadvisor
    restart: always
    volumes:
       - /:/rootfs:ro
       - /var/run:/var/run:rw
       - /sys:/sys:ro
       - /var/lib/docker/:/var/lib/docker:ro
    ports:
      - 8085:8080
    privileged: true

  mysqld-exporter:
    image: prom/mysqld-exporter
    ports:
    - 9104:9104
    restart: always
    container_name: mysql_exporter
    hostname: db01
    environment:
    - DATA_SOURCE_NAME=root:0GXwwchW4rP@(172.16.225.157:3306)/
    - TZ=Asia/Shanghai

导入模板，模板 ID 分别是：8919、7362
更多模块链接访问：点击获取监控模块，只需要导入对应ID即可

Prometheus、Alertmanager、Grafana 监控 linux 与 MySQL

原文地址：https://blog.51cto.com/13746824/2541350