Prometheus安装及配置-摩杜云开发者社区

1、下载及解压安装包

cd /usr/local/src/

export VER="2.13.1"
wget -N https://github.com/prometheus/prometheus/releases/download/v${VER}/prometheus-${VER}.linux-amd64.tar.gz

mkdir -p /data0/prometheus 
groupadd prometheus
useradd -g prometheus prometheus -d /data0/prometheus
 
tar -xvf prometheus-${VER}.linux-amd64.tar.gz
cd /usr/local/src/
mv prometheus-${VER}.linux-amd64 /data0/prometheus/prometheus_server
 
cd /data0/prometheus/prometheus_server/
mkdir -p {data,config,logs,bin} 
mv prometheus promtool bin/
mv prometheus.yml config/
 
chown -R prometheus.prometheus /data0/prometheus

2 、设置环境变量

vim /etc/profile

PATH=/data0/prometheus/prometheus_server/bin:$PATH:$HOME/bin

source /etc/profile

3、检查配置文件

promtool check config /data0/prometheus/prometheus_server/config/prometheus.yml

Checking /data0/prometheus/prometheus_server/config/prometheus.yml
  SUCCESS: 0 rule files found

4、创建prometheus.service 的 systemd unit 文件

4.1、常规服务

sudo tee /etc/systemd/system/prometheus.service <<-'EOF'
[Unit]
Description=Prometheus
Documentation=https://prometheus.io/
After=network.target
 
[Service]
Type=simple
User=prometheus
ExecStart=/data0/prometheus/prometheus_server/bin/prometheus --config.file=/data0/prometheus/prometheus_server/config/prometheus.yml --storage.tsdb.path=/data0/prometheus/prometheus_server/data --storage.tsdb.retention=60d
Restart=on-failure
 
[Install]
WantedBy=multi-user.target
EOF

systemctl enable prometheus.service
systemctl stop prometheus.service
systemctl restart prometheus.service
systemctl status prometheus.service

4.2、使用supervisor管理prometheus_server

yum install -y epel-release supervisor

sudo tee /etc/supervisord.d/prometheus.ini<<-"EOF"
[program:prometheus]
# 启动程序的命令;
command = /data0/prometheus/prometheus_server/bin/prometheus --config.file=/data0/prometheus/prometheus_server/config/prometheus.yml --storage.tsdb.path=/data0/prometheus/prometheus_server/data --storage.tsdb.retention=60d
# 在supervisord启动的时候也自动启动;
autostart = true
# 程序异常退出后自动重启;
autorestart = true
# 启动5秒后没有异常退出，就当作已经正常启动了;
startsecs = 5
# 启动失败自动重试次数，默认是3;
startretries = 3
# 启动程序的用户;
user = prometheus
# 把stderr重定向到stdout，默认false;
redirect_stderr = true
# 标准日志输出;
stdout_logfile=/data0/prometheus/prometheus_server/logs/out-prometheus.log
# 错误日志输出;
stderr_logfile=/data0/prometheus/prometheus_server/logs/err-prometheus.log
# 标准日志文件大小，默认50MB;
stdout_logfile_maxbytes = 20MB
# 标准日志文件备份数;
stdout_logfile_backups = 20
EOF

systemctl daemon-reload
systemctl enable supervisord
systemctl stop supervisord
systemctl restart supervisord
supervisorctl restart prometheus
supervisorctl status

5、prometheus.yml配置文件

#创建Alertmanager告警规则文件
mkdir -p /data0/prometheus/prometheus_server/rules/
touch /data0/prometheus/prometheus_server/rules/node_down.yml
touch /data0/prometheus/prometheus_server/rules/memory_over.yml
touch /data0/prometheus/prometheus_server/rules/disk_over.yml
touch /data0/prometheus/prometheus_server/rules/cpu_over.yml

#prometheus配置文件
cat > /data0/prometheus/prometheus_server/config/prometheus.yml << \EOF
# my global config
global:
  scrape_interval: 15s # 设置抓取(pull)时间间隔，默认是1m
  evaluation_interval: 15s # 设置rules评估时间间隔，默认是1m
  # scrape_timeout is set to the global default (10s).

# 告警管理配置，默认配置
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - 192.168.56.11:9093 # 这里修改为 alertmanagers 的地址

# 加载rules，并根据设置的时间间隔定期评估
rule_files:
# - "first_rules.yml"
# - "second_rules.yml"
  - "/data0/prometheus/prometheus_server/rules/node_down.yml"                 # 实例存活报警规则文件
  - "/data0/prometheus/prometheus_server/rules/memory_over.yml"               # 内存报警规则文件
  - "/data0/prometheus/prometheus_server/rules/disk_over.yml"                 # 磁盘报警规则文件
  - "/data0/prometheus/prometheus_server/rules/cpu_over.yml"                  # cpu报警规则文件

# 抓取(pull)，即监控目标配置
# 默认只有主机本身的监控配置
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    # 可覆盖全局配置设置的抓取间隔，由15秒重写成5秒。
    scrape_interval: 10s

    static_configs:
      - targets: ['localhost:9090', 'localhost:9100']

  - job_name: 'DMC_HOST'
    file_sd_configs:
      - files: ['./hosts.json']  
      # 被监控的主机，可以通过static_configs罗列所有机器，这里通过file_sd_configs参数加载文件的形式读取
      # 被监控的主机，可以json或yaml格式书写，我这里以json格式书写，target里面写监控机器的ip，labels非必须，可以由你自己定
EOF

#file_sd_configs参数形式配置主机列表
cat > /data0/prometheus/prometheus_server/config/hosts.json << \EOF
[
{
"targets": [
  "192.168.56.11:9100",
  "192.168.56.12:9100",
  "192.168.56.13:9100"
],
"labels": {
    "service": "db_node"
    }
},
{
"targets": [
  "192.168.56.14:9100",
  "192.168.56.15:9100",
  "192.168.56.16:9100"
],
"labels": {
    "service": "web_node"
    }
}
]
EOF

# 服务器存活报警
cat > /data0/prometheus/prometheus_server/rules/node_down.yml <<\EOF
groups:
- name: 实例存活告警规则
  rules:
  - alert: 实例存活告警
    expr: up == 0
    for: 1m
    labels:
      user: prometheus
      severity: warning
    annotations:
      description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes."
EOF

# mem报警
cat > /data0/prometheus/prometheus_server/rules/memory_over.yml <<\EOF
groups:
- name: 内存报警规则
  rules:
  - alert: 内存使用率告警
    expr: (node_memory_MemTotal_bytes - (node_memory_MemFree_bytes+node_memory_Buffers_bytes+node_memory_Cached_bytes )) / node_memory_MemTotal_bytes * 100 > 80
    for: 1m
    labels:
      user: prometheus
      severity: warning
    annotations:
      description: "服务器: 内存使用超过80%！(当前值: {{ $value }}%)"
EOF

# disk报警
cat > /data0/prometheus/prometheus_server/rules/disk_over.yml <<\EOF
groups:
- name: 磁盘报警规则
  rules:
  - alert: 磁盘使用率告警
    expr: (node_filesystem_size_bytes - node_filesystem_avail_bytes) / node_filesystem_size_bytes * 100 > 80
    for: 1m
    labels:
      user: prometheus
      severity: warning
    annotations:
      description: "服务器: 磁盘设备: 使用超过80%！(挂载点: {{ $labels.mountpoint }} 当前值: {{ $value }}%)"
EOF

# cpu报警
cat > /data0/prometheus/prometheus_server/rules/cpu_over.yml <<\EOF
groups:
- name: CPU报警规则
  rules:
  - alert: CPU使用率告警
    expr: 100 - (avg by (instance)(irate(node_cpu_seconds_total{mode="idle"}[1m]) )) * 100 > 90
    for: 1m
    labels:
      user: prometheus
      severity: warning
    annotations:
      description: "服务器: CPU使用超过90%！(当前值: {{ $value }}%)"
EOF