创建prometheus数据目录并赋权
[root@k8snode2 ~]# mkdir /data
[root@k8snode2 ~]# chmod 777 /data
创建prometheus配置文件configMap
[root@k8smaster1 prometheus]# cat prometheus-cfg.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus--cfg
namespace: monitor-sa
labels:
app: prometheus
data:
prometheus.yml: |
global:
scrape_interval: 15s
scrape_timeout: 10s
evaluation_interval: 1m
scrape_configs:
- job_name: 'kubernetes-node'
kubernetes_sd_configs:
- role: node
relabel_configs:
- source_labels: [_address_]
regex: '(.*):10250'
replacement: '${1}:9100'
target_label: _address_
action: replace
- action: labelmap
regex: _meta_kubernetes_node_label_(.+)
- job_name: 'kubernetes-node-cadvisor'
kubernetes_sd_configs:
- role: node
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- action: labelmap
regex: _meta_kubernetes_node_label_(.+)
- target_label: _address_
replacement: kubernetes.default.svc:443
- source_labels: [_meta_kubernetes_node_name]
regex: (.+)
target_label: _metrics_path_
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
- job_name: 'kubernetes-apiserver'
kubernetes_sd_configs:
- role: endpoints
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- source_labels: [_meta_kubernetes_namespace,_meta_kubernetes_service_name,_meta_kubernetes_endpoint_port_name]
action: keep
regex: default;kubernetes;https
- job_name: 'kubernetes-service_endpoints'
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [_meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [_meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: _scheme_
regex: (https?)
- source_labels: [_meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: _metrics_path
regex: (.+)
- source_labels: [_address_,_meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
target_label: _address_
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- action: lebelmap
regex: _meta_kubernetes_service_label_(.+)
- source_labels: [_meta_kubernetes_namespace]
action: replace
target_label: kuberentes_namespace
- source_labels: [_meta_kubernetes_service_name]
action: replace
target_label: kubernetes_name
查看执行结果
[root@k8smaster1 prometheus]# kubectl get cm -n monitor-sa
NAME DATA AGE
kube-root-ca.crt 1 19h
prometheus--cfg 1 10s
[root@k8smaster1 prometheus]# kubectl get cm prometheus--cfg -o yaml -n monitor-sa
apiVersion: v1
data:
prometheus.yml: "global:\n scrape_interval: 15s\n scrape_timeout: 10s\n evaluation_interval:
1m\nscrape_configs:\n- job_name: 'kubernetes-node'\n kubernetes_sd_configs:\n
\ - role: node\n relabel_configs:\n - source_labels: [_address_]\n regex:
'(.*):10250'\n replacement: '${1}:9100'\n target_label: _address_\n action:
replace\n - action: labelmap\n regex: _meta_kubernetes_node_label_(.+)\n-
job_name: 'kubernetes-node-cadvisor'\n kubernetes_sd_configs:\n - role: node\n
\ scheme: https\n tls_config:\n ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n
\ bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n relabel_configs:\n
\ - action: labelmap\n regex: _meta_kubernetes_node_label_(.+)\n - target_label:
_address_\n replacement: kubernetes.default.svc:443\n - source_labels: [_meta_kubernetes_node_name]\n
\ regex: (.+)\n target_label: _metrics_path_\n replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor\n-
job_name: 'kubernetes-apiserver'\n kubernetes_sd_configs:\n - role: endpoints\n
\ scheme: https\n tls_config:\n ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n
\ bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n relabel_configs:\n
\ - source_labels: [_meta_kubernetes_namespace,_meta_kubernetes_service_name,_meta_kubernetes_endpoint_port_name]\n
\ action: keep\n regex: default;kubernetes;https\n- job_name: 'kubernetes-service_endpoints'\n
\ kubernetes_sd_configs:\n - role: endpoints\n relabel_configs:\n - source_labels:
[_meta_kubernetes_service_annotation_prometheus_io_scrape]\n action: keep\n
\ regex: true\n - source_labels: [_meta_kubernetes_service_annotation_prometheus_io_scheme]\n
\ action: replace\n target_label: _scheme_\n regex: (https?)\n - source_labels:
[_meta_kubernetes_service_annotation_prometheus_io_path]\n action: replace\n
\ target_label: _metrics_path\n regex: (.+)\n - source_labels: [_address_,_meta_kubernetes_service_annotation_prometheus_io_port]\n
\ action: replace\n target_label: _address_\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n
\ replacement: $1:$2\n - action: lebelmap\n regex: _meta_kubernetes_service_label_(.+)\n
\ - source_labels: [_meta_kubernetes_namespace]\n action: replace\n target_label:
kuberentes_namespace\n - source_labels: [_meta_kubernetes_service_name]\n action:
replace\n target_label: kubernetes_name \n"
kind: ConfigMap
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"v1","data":{"prometheus.yml":"global:\n scrape_interval: 15s\n scrape_timeout: 10s\n evaluation_interval: 1m\nscrape_configs:\n- job_name: 'kubernetes-node'\n kubernetes_sd_configs:\n - role: node\n relabel_configs:\n - source_labels: [_address_]\n regex: '(.*):10250'\n replacement: '${1}:9100'\n target_label: _address_\n action: replace\n - action: labelmap\n regex: _meta_kubernetes_node_label_(.+)\n- job_name: 'kubernetes-node-cadvisor'\n kubernetes_sd_configs:\n - role: node\n scheme: https\n tls_config:\n ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n relabel_configs:\n - action: labelmap\n regex: _meta_kubernetes_node_label_(.+)\n - target_label: _address_\n replacement: kubernetes.default.svc:443\n - source_labels: [_meta_kubernetes_node_name]\n regex: (.+)\n target_label: _metrics_path_\n replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor\n- job_name: 'kubernetes-apiserver'\n kubernetes_sd_configs:\n - role: endpoints\n scheme: https\n tls_config:\n ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n relabel_configs:\n - source_labels: [_meta_kubernetes_namespace,_meta_kubernetes_service_name,_meta_kubernetes_endpoint_port_name]\n action: keep\n regex: default;kubernetes;https\n- job_name: 'kubernetes-service_endpoints'\n kubernetes_sd_configs:\n - role: endpoints\n relabel_configs:\n - source_labels: [_meta_kubernetes_service_annotation_prometheus_io_scrape]\n action: keep\n regex: true\n - source_labels: [_meta_kubernetes_service_annotation_prometheus_io_scheme]\n action: replace\n target_label: _scheme_\n regex: (https?)\n - source_labels: [_meta_kubernetes_service_annotation_prometheus_io_path]\n action: replace\n target_label: _metrics_path\n regex: (.+)\n - source_labels: [_address_,_meta_kubernetes_service_annotation_prometheus_io_port]\n action: replace\n target_label: _address_\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n - action: lebelmap\n regex: _meta_kubernetes_service_label_(.+)\n - source_labels: [_meta_kubernetes_namespace]\n action: replace\n target_label: kuberentes_namespace\n - source_labels: [_meta_kubernetes_service_name]\n action: replace\n target_label: kubernetes_name \n"},"kind":"ConfigMap","metadata":{"annotations":{},"labels":{"app":"prometheus"},"name":"prometheus--cfg","namespace":"monitor-sa"}}
creationTimestamp: "2023-07-11T03:56:29Z"
labels:
app: prometheus
name: prometheus--cfg
namespace: monitor-sa
resourceVersion: "1143434"
uid: 751cf405-3bd4-4043-b22d-99be52f8f4fa
安装 Prometheus
Yaml 文件
查看创建结果
查看Pod运行日志
[root@k8smaster1 ~]# kubectl logs prometheus-server-6f97cf98dd-ndg5w -n monitor-sa
level=info ts=2023-07-12T07:55:15.300092999Z caller=main.go:220 msg="Starting Prometheus" version="(version=2.2.1, branch=HEAD, revision=bc6058c81272a8d938c05e75607371284236aadc)"
level=info ts=2023-07-12T07:55:15.300189532Z caller=main.go:221 build_context="(go=go1.10, user=root@149e5b3f0829, date=20180314-14:15:45)"
level=info ts=2023-07-12T07:55:15.300214841Z caller=main.go:222 host_details="(Linux 3.10.0-1160.el7.x86_64 #1 SMP Mon Oct 19 16:18:59 UTC 2020 x86_64 prometheus-server-6f97cf98dd-ndg5w (none))"
level=info ts=2023-07-12T07:55:15.300233878Z caller=main.go:223 fd_limits="(soft=1048576, hard=1048576)"
level=info ts=2023-07-12T07:55:15.303804654Z caller=main.go:504 msg="Starting TSDB ..."
level=info ts=2023-07-12T07:55:15.307121345Z caller=web.go:382 component=web msg="Start listening for connections" address=0.0.0.0:9090
level=info ts=2023-07-12T07:55:15.311274144Z caller=main.go:514 msg="TSDB started"
level=info ts=2023-07-12T07:55:15.311311545Z caller=main.go:588 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
level=info ts=2023-07-12T07:55:15.312333132Z caller=main.go:398 msg="Stopping scrape discovery manager..."
level=info ts=2023-07-12T07:55:15.312367522Z caller=main.go:411 msg="Stopping notify discovery manager..."
level=info ts=2023-07-12T07:55:15.312381895Z caller=main.go:432 msg="Stopping scrape manager..."
level=info ts=2023-07-12T07:55:15.312410542Z caller=main.go:394 msg="Scrape discovery manager stopped"
level=info ts=2023-07-12T07:55:15.312453813Z caller=main.go:426 msg="Scrape manager stopped"
level=info ts=2023-07-12T07:55:15.312492456Z caller=main.go:407 msg="Notify discovery manager stopped"
level=info ts=2023-07-12T07:55:15.31252679Z caller=manager.go:460 component="rule manager" msg="Stopping rule manager..."
level=info ts=2023-07-12T07:55:15.312550383Z caller=manager.go:466 component="rule manager" msg="Rule manager stopped"
level=info ts=2023-07-12T07:55:15.312564621Z caller=notifier.go:512 component=notifier msg="Stopping notification manager..."
level=info ts=2023-07-12T07:55:15.312602296Z caller=main.go:573 msg="Notifier manager stopped"
level=error ts=2023-07-12T07:55:15.312682464Z caller=main.go:582 err="Error loading config couldn't load configuration (--config.file=/etc/prometheus/prometheus.yml): parsing YAML file /etc/prometheus/prometheus.yml: unknown relabel action \"lebelmap\""
配置service，暴露端口到外网
apiVersion: v1
kind: Service
metadata:
name: prometheus
namespace: monitor-sa
labels:
app: prometheus
spec:
selector:
app: prometheus
component: server
ports:
- targetPort: 9090
查看配置结果
[root@k8smaster1 prometheus]# kubectl get service -n monitor-sa
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
prometheus NodePort 10.106.223.98 <none> 9090:31967/TCP 3m49s
[root@k8smaster1 prometheus]# kubectl get pods -n monitor-sa
NAME READY STATUS RESTARTS AGE
node-exporter-fx995 1/1 Running 0 45h
node-exporter-ktfkn 1/1 Running 0 45h
node-exporter-qrfkj 1/1 Running 0 45h
prometheus-server-6f97cf98dd-4wpg2 1/1 Running 0 52m
[root@k8smaster1 prometheus]#
浏览器访问
查看IPVS映射关系
[root@k8smaster1 prometheus]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 172.17.0.1:30943 rr
-> 192.168.1.172:443 Masq 1 0 0
-> 192.168.1.173:443 Masq 1 0 0
TCP 172.17.0.1:31967 rr
-> 10.244.93.67:9090 Masq 1 0 0
TCP 172.17.0.1:32051 rr
-> 192.168.1.172:80 Masq 1 0 0
-> 192.168.1.173:80 Masq 1 0 0
TCP 192.168.1.170:30943 rr
-> 192.168.1.172:443 Masq 1 0 0
-> 192.168.1.173:443 Masq 1 0 0
TCP 192.168.1.170:31967 rr
-> 10.244.93.67:9090 Masq 1 0 0
TCP 192.168.1.170:32051 rr
-> 192.168.1.172:80 Masq 1 0 0
-> 192.168.1.173:80 Masq 1 0 0
TCP 10.96.0.1:443 rr
-> 192.168.1.170:6443 Masq 1 4 0
TCP 10.96.0.10:53 rr
-> 10.244.249.10:53 Masq 1 0 0
-> 10.244.249.12:53 Masq 1 0 0
TCP 10.96.0.10:9153 rr
-> 10.244.249.10:9153 Masq 1 0 0
-> 10.244.249.12:9153 Masq 1 0 0
TCP 10.103.192.195:80 rr
-> 10.244.185.213:80 Masq 1 0 0
TCP 10.104.75.192:443 rr
-> 192.168.1.172:8443 Masq 1 0 0
-> 192.168.1.173:8443 Masq 1 0 0
TCP 10.106.199.96:80 rr
-> 192.168.1.172:80 Masq 1 0 0
-> 192.168.1.173:80 Masq 1 0 0
TCP 10.106.199.96:443 rr
-> 192.168.1.172:443 Masq 1 0 0
-> 192.168.1.173:443 Masq 1 0 0
TCP 10.106.223.98:9090 rr
-> 10.244.93.67:9090 Masq 1 0 0
TCP 10.109.75.195:80 rr
-> 10.244.185.212:80 Masq 1 0 0
TCP 10.244.93.64:30943 rr
-> 192.168.1.172:443 Masq 1 0 0
-> 192.168.1.173:443 Masq 1 0 0
TCP 10.244.93.64:31967 rr
-> 10.244.93.67:9090 Masq 1 0 0
TCP 10.244.93.64:32051 rr
-> 192.168.1.172:80 Masq 1 0 0
-> 192.168.1.173:80 Masq 1 0 0
UDP 10.96.0.10:53 rr
-> 10.244.249.10:53 Masq 1 0 0
-> 10.244.249.12:53 Masq 1 0 0
[root@k8smaster1 prometheus]#
