k8s部署daemonset logstash-摩杜云开发者社区

实现：收集容器控制台日志

k8s部署daemonset logstash_logstash

部署daemonset logstash，如上图红框位置

找一台机器打镜像

FROM logstash:7.12.1


USER root
WORKDIR /usr/share/logstash 
ADD logstash.yml /usr/share/logstash/config/logstash.yml
ADD logstash.conf /usr/share/logstash/pipeline/logstash.conf 

docker build . -t 10.0.7.12/k8s/logstash:7.9.0
docker push 10.0.7.12/k8s/logstash:7.9.0

logstash.conf文件

cat logstash.conf 
input {
  file {
    path => "/var/log/pods/*/*/*.log"
    start_position => "beginning"
    type => "jsonfile-daemonset-applog"
  }

  file {
    path => "/var/log/*.log"
    start_position => "beginning"
    type => "jsonfile-daemonset-syslog"
  }
}

output {
  if [type] == "jsonfile-daemonset-applog" {
    kafka {
      bootstrap_servers => "${KAFKA_SERVER}"
      topic_id => "${TOPIC_ID}"
      batch_size => 16384  #logstash每次向ES传输的数据量大小,单位为字节
      codec => "${CODEC}" # json编码
   } }

  if [type] == "jsonfile-daemonset-syslog" {
    kafka {
      bootstrap_servers => "${KAFKA_SERVER}"
      topic_id => "${TOPIC_ID}"
      batch_size => 16384
      codec => "${CODEC}" #系统日志不是json格式
  }}
}

在k8s集群部署daemonset

cat daemonset-logstash.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: logstash-elasticsearch
  namespace: kube-system
  labels:
    k8s-app: logstash-logging
spec:
  selector:
    matchLabels:
      name: logstash-elasticsearch
  template:
    metadata:
      labels:
        name: logstash-elasticsearch
    spec:
      tolerations:
      #- key: node-role.kubernetes.io/unschedulable
      - key: node.kubernetes.io/unschedulable
        operator: Exists
        effect: NoSchedule
      imagePullSecrets:
      - name: myregistrykey
      containers:
      - name: logstash-elasticsearch
        image: 10.0.7.12/k8s/logstash:7.9.0
        env:
        - name: "KAFKA_SERVER"
          value: "10.0.7.53:9092,10.0.7.54:9092,10.0.7.55:9092"
        - name: "TOPIC_ID"
          value: "jsonfile-log-topic"
        - name: "CODEC"
          value: "json"
        volumeMounts:
        - name: varlog #定义宿主机系统日志挂载路径
          mountPath: /var/log #宿主机系统日志挂载点
        - name: varlibdockercontainers #定义容器日志挂载路径,和logstash配置文件中的收集路径保持一直
          mountPath: /var/log/pods #containerd挂载路径,此路径与logstash的日志收集路径必须一致
          readOnly: false
      terminationGracePeriodSeconds: 30
      volumes:
      - name: varlog
        hostPath:
          path: /var/log #宿主机系统日志
      - name: varlibdockercontainers
        hostPath:
          path: /var/log/pods #containerd的宿主机日志路径
          
kubectl apply -f damonset-logstash.yaml