（脚本部署）prometheus+grafana+alertmanager+dingtalk （linux监控系统）

1.编写脚本配置 pgawd.sh

#!/bin/bash
echo "注意看看 要删文件了"
rm -rf /data/PGAWD

echo "创建预警|规则目录"
mkdir -p /data/PGAWD/{alert,rules}
echo "创建普罗米休斯数据目录"
mkdir -p /data/PGAWD/prometheus/data
echo "写入普罗米休斯配置文件"
cat >> /data/PGAWD/prometheus/prometheus.yml << EOF

global:
  scrape_interval:     15s    # 多久 收集 一次数据
  evaluation_interval: 15s    # 多久 评估 一次规则
  scrape_timeout:      10s    # 每次 收集数据的 超时时间

# 收集数据 配置 列表
scrape_configs:
  - job_name: prometheus            # 必须配置, 自动附加的job labels, 必须唯一
    static_configs:
      - targets: ['127.0.0.1:9090']       # 指定prometheus ip端口
        labels:
          instance: prometheus                 #标签

  - job_name: ehospital-exploit-database
    static_configs:
      - targets: ['127.0.0.1:9100']
        labels:
          instance: eehospital-exploit-database


alerting:                         #Alertmanager相关的配置
  alertmanagers:
  - static_configs:
    - targets:
      - 127.0.0.1:9093         #指定告警模块

rule_files:                      #告警规则文件, 可以使用通配符
  - "/etc/prometheus/rules/*.yml"


EOF

echo "拷贝通用规则alert-rules配置文件"

cp ./alert-rules.yml /data/PGAWD/rules/alert-rules.yml


echo "写入通用规则record-rules配置文件"
cat >> /data/PGAWD/rules/record-rules.yml << EOF

groups:
  - name: prometheus-record
    rules:
    - expr: up{job!="prometheus"}
      record: prometheus:up
      labels:
        desc: "节点是否在线, 在线1,不在线0"
        unit: " "
        job: "prometheus"
    - expr: time() - node_boot_time_seconds{}
      record: prometheus:node_uptime
      labels:
        desc: "节点的运行时间"
        unit: "s"
        job: "prometheus"
##############################################################################################
#                              cpu                                                           #
    - expr: (1 - avg by (environment,instance) (irate(node_cpu_seconds_total{job!="prometheus",mode="idle"}[5m])))  * 100
      record: prometheus:cpu:total:percent
      labels:
        desc: "节点的cpu总消耗百分比"
        unit: "%"
        job: "prometheus"

    - expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job!="prometheus",mode="idle"}[5m])))  * 100
      record: prometheus:cpu:idle:percent
      labels:
        desc: "节点的cpu idle百分比"
        unit: "%"
        job: "prometheus"

    - expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job!="prometheus",mode="iowait"}[5m])))  * 100
      record: prometheus:cpu:iowait:percent
      labels:
        desc: "节点的cpu iowait百分比"
        unit: "%"
        job: "prometheus"



    - expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job!="prometheus",mode="system"}[5m])))  * 100
      record: prometheus:cpu:system:percent
      labels:
        desc: "节点的cpu system百分比"
        unit: "%"
        job: "prometheus"

    - expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job!="prometheus",mode="user"}[5m])))  * 100
      record: prometheus:cpu:user:percent
      labels:
        desc: "节点的cpu user百分比"
        unit: "%"
        job: "prometheus"

    - expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job!="prometheus",mode=~"softirq|nice|irq|steal"}[5m])))  * 100
      record: prometheus:cpu:other:percent
      labels:
        desc: "节点的cpu 其他的百分比"
        unit: "%"
        job: "prometheus"
##############################################################################################

##############################################################################################
#                                    memory                                                  #
    - expr: node_memory_MemTotal_bytes{job!="prometheus"}
      record: prometheus:memory:total
      labels:
        desc: "节点的内存总量"
        unit: byte
        job: "prometheus"

    - expr: node_memory_MemFree_bytes{job!="prometheus"}
      record: prometheus:memory:free
      labels:
        desc: "节点的剩余内存量"
        unit: byte
        job: "prometheus"

    - expr: node_memory_MemTotal_bytes{job!="prometheus"} - node_memory_MemFree_bytes{job!="prometheus"}
      record: prometheus:memory:used
      labels:
        desc: "节点的已使用内存量"
        unit: byte
        job: "prometheus"

    - expr: node_memory_MemTotal_bytes{job!="prometheus"} - node_memory_MemAvailable_bytes{job!="prometheus"}
      record: prometheus:memory:actualused
      labels:
        desc: "节点用户实际使用的内存量"
        unit: byte
        job: "prometheus"

    - expr: (1-(node_memory_MemAvailable_bytes{job!="prometheus"} / (node_memory_MemTotal_bytes{job!="prometheus"})))* 100
      record: prometheus:memory:used:percent
      labels:
        desc: "节点的内存使用百分比"
        unit: "%"
        job: "prometheus"

    - expr: ((node_memory_MemAvailable_bytes{job!="prometheus"} / (node_memory_MemTotal_bytes{job!="prometheus"})))* 100
      record: prometheus:memory:free:percent
      labels:
        desc: "节点的内存剩余百分比"
        unit: "%"
        job: "prometheus"
##############################################################################################
#                                   load                                                     #
    - expr: sum by (instance) (node_load1{job!="prometheus"})
      record: prometheus:load:load1
      labels:
        desc: "系统1分钟负载"
        unit: " "
        job: "prometheus"

    - expr: sum by (instance) (node_load5{job!="prometheus"})
      record: prometheus:load:load5
      labels:
        desc: "系统5分钟负载"
        unit: " "
        job: "prometheus"

    - expr: sum by (instance) (node_load15{job!="prometheus"})
      record: prometheus:load:load15
      labels:
        desc: "系统15分钟负载"
        unit: " "
        job: "prometheus"

##############################################################################################
#                                 disk                                                       #
    - expr: node_filesystem_size_bytes{job!="prometheus" ,fstype=~"ext4|xfs"}
      record: prometheus:disk:usage:total
      labels:
        desc: "节点的磁盘总量"
        unit: byte
        job: "prometheus"

    - expr: node_filesystem_avail_bytes{job!="prometheus",fstype=~"ext4|xfs"}
      record: prometheus:disk:usage:free
      labels:
        desc: "节点的磁盘剩余空间"
        unit: byte
        job: "prometheus"

    - expr: node_filesystem_size_bytes{job!="prometheus",fstype=~"ext4|xfs"} - node_filesystem_avail_bytes{job!="prometheus",fstype=~"ext4|xfs"}
      record: prometheus:disk:usage:used
      labels:
        desc: "节点的磁盘使用的空间"
        unit: byte
        job: "prometheus"

    - expr:  (1 - node_filesystem_avail_bytes{job!="prometheus",fstype=~"ext4|xfs"} / node_filesystem_size_bytes{job!="prometheus",fstype=~"ext4|xfs"}) * 100
      record: prometheus:disk:used:percent
      labels:
        desc: "节点的磁盘的使用百分比"
        unit: "%"
        job: "prometheus"

    - expr: irate(node_disk_reads_completed_total{job!="prometheus"}[1m])
      record: prometheus:disk:read:count:rate
      labels:
        desc: "节点的磁盘读取速率"
        unit: "次/秒"
        job: "prometheus"

    - expr: irate(node_disk_writes_completed_total{job!="prometheus"}[1m])
      record: prometheus:disk:write:count:rate
      labels:
        desc: "节点的磁盘写入速率"
        unit: "次/秒"
        job: "prometheus"

    - expr: (irate(node_disk_written_bytes_total{job!="prometheus"}[1m]))/1024/1024
      record: prometheus:disk:read:mb:rate
      labels:
        desc: "节点的设备读取MB速率"
        unit: "MB/s"
        job: "prometheus"

    - expr: (irate(node_disk_read_bytes_total{job!="prometheus"}[1m]))/1024/1024
      record: prometheus:disk:write:mb:rate
      labels:
        desc: "节点的设备写入MB速率"
        unit: "MB/s"
        job: "prometheus"

##############################################################################################
#                                filesystem                                                  #
    - expr:   (1 -node_filesystem_files_free{job!="prometheus",fstype=~"ext4|xfs"} / node_filesystem_files{job!="prometheus",fstype=~"ext4|xfs"}) * 100
      record: prometheus:filesystem:used:percent
      labels:
        desc: "节点的inode的剩余可用的百分比"
        unit: "%"
        job: "prometheus"
#############################################################################################
#                                filefd                                                     #
    - expr: node_filefd_allocated{job!="prometheus"}
      record: prometheus:filefd_allocated:count
      labels:
        desc: "节点的文件描述符打开个数"
        unit: "%"
        job: "prometheus"

    - expr: node_filefd_allocated{job!="prometheus"}/node_filefd_maximum{job!="prometheus"} * 100
      record: prometheus:filefd_allocated:percent
      labels:
        desc: "节点的文件描述符打开百分比"
        unit: "%"
        job: "prometheus"

#############################################################################################
#                                network                                                    #
    - expr: avg by (environment,instance,device) (irate(node_network_receive_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]))
      record: prometheus:network:netin:bit:rate
      labels:
        desc: "节点网卡eth0每秒接收的比特数"
        unit: "bit/s"
        job: "prometheus"

    - expr: avg by (environment,instance,device) (irate(node_network_transmit_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]))
      record: prometheus:network:netout:bit:rate
      labels:
        desc: "节点网卡eth0每秒发送的比特数"
        unit: "bit/s"
        job: "prometheus"

    - expr: avg by (environment,instance,device) (irate(node_network_receive_packets_total{device=~"eth0|eth1|ens33|ens37"}[1m]))
      record: prometheus:network:netin:packet:rate
      labels:
        desc: "节点网卡每秒接收的数据包个数"
        unit: "个/秒"
        job: "prometheus"


    - expr: avg by (environment,instance,device) (irate(node_network_transmit_packets_total{device=~"eth0|eth1|ens33|ens37"}[1m]))
      record: prometheus:network:netout:packet:rate
      labels:
        desc: "节点网卡发送的数据包个数"
        unit: "个/秒"
        job: "prometheus"

    - expr: avg by (environment,instance,device) (irate(node_network_receive_errs_total{device=~"eth0|eth1|ens33|ens37"}[1m]))
      record: prometheus:network:netin:error:rate
      labels:
        desc: "节点设备驱动器检测到的接收错误包的数量"
        unit: "个/秒"
        job: "prometheus"

    - expr: avg by (environment,instance,device) (irate(node_network_transmit_errs_total{device=~"eth0|eth1|ens33|ens37"}[1m]))
      record: prometheus:network:netout:error:rate
      labels:
        desc: "节点设备驱动器检测到的发送错误包的数量"
        unit: "个/秒"
        job: "prometheus"

    - expr: node_tcp_connection_states{job!="prometheus", state="established"}
      record: prometheus:network:tcp:established:count
      labels:
        desc: "节点当前established的个数"
        unit: "个"
        job: "prometheus"

    - expr: node_tcp_connection_states{job!="prometheus", state="time_wait"}
      record: prometheus:network:tcp:timewait:count
      labels:
        desc: "节点timewait的连接数"
        unit: "个"
        job: "prometheus"

    - expr: sum by (environment,instance) (node_tcp_connection_states{job!="prometheus"})
      record: prometheus:network:tcp:total:count
      labels:
        desc: "节点tcp连接总数"
        unit: "个"
        job: "prometheus"
EOF


echo "写入通用规则record-rules配置文件"

cat >> /data/PGAWD/alert/alertmanager.yml << EOF
global:
  resolve_timeout: 5m
route:
  receiver: webhook
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 5m
  group_by: [alertname]
  routes:
  - receiver: webhook
    group_wait: 10s
receivers:
- name: webhook
  webhook_configs:
  - url: http://127.0.0.1:8060/dingtalk/webhook1/send
    send_resolved: true


EOF

echo "写入docker-compose.yml配置文件"

cat >> /data/PGAWD/docker-compose.yml << EOF
version: '3.7'
services:
  prometheus:
    image: prom/prometheus
    restart: "always"
    ports:
      - 9090:9090
    container_name: "prometheus"
    volumes:
      - "/data/PGAWD/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml"
      - "/data/PGAWD/rules:/etc/prometheus/rules"
      - "/data/PGAWD/prometheus/data:/prometheus"
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'                     #设置数据路径   跟上面挂载对应

  #告警模块
  alertmanager:
    image: prom/alertmanager:latest
    restart: "always"
    ports:
      - 9093:9093
    container_name: "alertmanager"
    volumes:
      - "/data/PGAWD/alert/alertmanager.yml:/etc/alertmanager/alertmanager.yml"

  #钉钉插件
  webhook:
    image: timonwong/prometheus-webhook-dingtalk
    restart: "always"
    ports:
      - 8060:8060
    container_name: "webhook"           #token指定钉钉
    command:
      - '--ding.profile=webhook1=https://oapi.dingtalk.com/robot/send?access_token=* 钉钉机器人地址'

  #web界面
  grafana:
    image: grafana/grafana
    restart: "always"
    ports:
      - 3000:3000
    container_name: "grafana"
    volumes:
      - "/data/PGAWD/grafana/grafana.ini:/etc/grafana/grafana.ini"              #配置文件自行拷贝出来
      - "/data/PGAWD/grafana/grafana-storage:/var/lib/grafana"
EOF

echo "写入通用规则node-exporter-compose配置文件"

cat >> /data/PGAWD/node-exporter-compose.yml << EOF
version: '3.7'
services:
  node-exporter:
    image: prom/node-exporter
    restart: "always"
    ports:
      - 9100:9100
    container_name: "node-exporter"
    volumes:
      - "/proc:/host/proc:ro"
      - "/sys:/host/sys:ro"
      - "/:/rootfs:ro"
EOF


echo "创建Grafana数据目录"
mkdir -p /data/PGAWD/grafana/grafana-storage

echo "开始授权"
chmod 777 /data/PGAWD/prometheus/data
chmod 777 /data/PGAWD/grafana/grafana-storage

echo "启动服务"

cd /data/PGAWD
docker-compose -f node-exporter-compose.yml up -d
docker-compose -f docker-compose.yml up -d

echo "我懒直接启动下载镜像拷贝文件了grafana.ini 这玩意从容器里面考出来"
grafanaPath=`find / -print0 | grep -FzZ '/diff/etc/grafana/grafana.ini' | awk '{print $1}'`

cp $grafanaPath /data/PGAWD/grafana/grafana.ini


echo "先卸载在启动"
docker-compose -f node-exporter-compose.yml down
docker-compose -f docker-compose.yml down
docker-compose -f node-exporter-compose.yml up -d
docker-compose -f docker-compose.yml up -d
docker logs -f prometheus
docker ps

2.新建alert-rules.yml 和脚本保持在同一文件夹下

groups:
  - name: prometheus-alert
    rules:
      - alert: prometheus-down
        expr: prometheus:up == 0
        for: 1m
        labels:
          severity: 'critical'
        annotations:
          summary: "instance: {{ $labels.instance }} 宕机了"
          description: "instance: {{ $labels.instance }} \n- job: {{ $labels.job }} 关机了， 时间已经1分钟了。"
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-cpu-high
        expr: prometheus:cpu:total:percent > 80
        for: 3m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} cpu 使用率高于 {{ $value }}"
          description: "instance: {{ $labels.instance }} \n- job: {{ $labels.job }} CPU使用率已经持续一分钟高过80% 。"
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-cpu-iowait-high
        expr: prometheus:cpu:iowait:percent >= 12
        for: 3m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} cpu iowait 使用率高于 {{ $value }}"
          description: "instance: {{ $labels.instance }} \n- job: {{ $labels.job }} cpu iowait使用率已经持续三分钟高过12%"
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-load-load1-high
        expr: (prometheus:load:load1) > (prometheus:cpu:count) * 1.2
        for: 3m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} load1 使用率高于 {{ $value }}"
          description: ""
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-memory-high
        expr: prometheus:memory:used:percent > 85
        for: 3m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} memory 使用率高于 {{ $value }}"
          description: ""
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-disk-high
        expr: prometheus:disk:used:percent > 80
        for: 10m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} disk 使用率高于 {{ $value }}"
          description: ""
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-disk-read:count-high
        expr: prometheus:disk:read:count:rate > 2000
        for: 2m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} iops read 使用率高于 {{ $value }}"
          description: ""
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-disk-write-count-high
        expr: prometheus:disk:write:count:rate > 2000
        for: 2m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} iops write 使用率高于 {{ $value }}"
          description: ""
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-disk-read-mb-high
        expr: prometheus:disk:read:mb:rate > 60
        for: 2m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} 读取字节数 高于 {{ $value }}"
          description: ""
          instance: "{{ $labels.instance }}"
          value: "{{ $value }}"
      - alert: prometheus-disk-write-mb-high
        expr: prometheus:disk:write:mb:rate > 60
        for: 2m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} 写入字节数 高于 {{ $value }}"
          description: ""
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-filefd-allocated-percent-high
        expr: prometheus:filefd_allocated:percent > 80
        for: 10m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} 打开文件描述符 高于 {{ $value }}"
          description: ""
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-network-netin-error-rate-high
        expr: prometheus:network:netin:error:rate > 4
        for: 1m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} 包进入的错误速率 高于 {{ $value }}"
          description: ""
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-network-netin-packet-rate-high
        expr: prometheus:network:netin:packet:rate > 35000
        for: 1m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} 包进入速率 高于 {{ $value }}"
          description: ""
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-network-netout-packet-rate-high
        expr: prometheus:network:netout:packet:rate > 35000
        for: 1m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} 包流出速率 高于 {{ $value }}"
          description: ""
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-network-tcp-total-count-high
        expr: prometheus:network:tcp:total:count > 40000
        for: 1m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} tcp连接数量 高于 {{ $value }}"
          description: ""
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-process-zoom-total-count-high
        expr: prometheus:process:zoom:total:count > 10
        for: 10m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} 僵死进程数量 高于 {{ $value }}"
          description: ""
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"
      - alert: prometheus-time-offset-high
        expr: prometheus:time:offset > 0.03
        for: 2m
        labels:
          severity: info
        annotations:
          summary: "instance: {{ $labels.instance }} {{ $labels.desc }}  {{ $value }} {{ $labels.unit }}"
          description: ""
          value: "{{ $value }}"
          instance: "{{ $labels.instance }}"

3.QA常见问题解决

#查看所有容器
docker ps -a 
#脚本特殊字符处理
doc2unix 
#(容器id/容器name 或者关系 二选一) 排错
docker logs -f 

4.关于界面连接

       实话说，这个东东没有仔细研究，我是根据导入json后直接点击发现连接失败，找到页面编辑发现URL中变量配置，我配了1-8所有变量进行测试，最后选择自己需要的变量进行替换完成列表与详情连接。

5.关于网络通信

      部署使用时请将127替换为监控相关宿主机内网或者外网ip，别一直配127然后找错误。  

6.下载附件

下载附件

喜欢 (1)赏

支付宝[]

分享 (0)