Promtail

Promtail 是用来将容器日志发送到 Loki 或者 Grafana 服务上的日志收集工具，该工具主要包括发现采集目标以及给日志流添加上 Label 标签，然后发送给 Loki，另外 Promtail 的服务发现是基于 Prometheus 的服务发现机制实现的。

以下在k8s集群中安装并配置promtail的例子：
为了更好的理解，我创建了两个yaml文件，promtail-daemonset.yaml负责拉取promtail镜像，映射卷，配置k8s认证文件等，promtail以daemonset的方式存在每个node节点；promtail-configmap.yaml负责配置promtail抓取日志的配置文件。

配置如下：
[root@kubeadm loki]# cat promtail-daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: promtail-daemonset
spec:
  selector:
    matchLabels:
      name: promtail
  template:
    metadata:
      labels:
        name: promtail
    spec:
      hostNetwork: true                  #开启hostnetwork映射configmap定义的端口到node节点方便在浏览器调试
      serviceAccount: SERVICE_ACCOUNT
      serviceAccountName: promtail-serviceaccount  #这里的名称要跟下面ServiceAccount定义的一样
      volumes:
      - name: logs
        hostPath:
          path: /var/log
      - name: varlibdockercontainers     #注意，不加这个卷映射的，在抓取pod日志文件时会报promtail logs no such file or directory
        hostPath:
          path: /var/lib/docker/containers
      - name: promtail-config
        configMap:
          name: promtail-config          #这里的name要跟promtail-configmap.yaml定义的name一样
      containers:
      - name: promtail-container
        image: grafana/promtail
        args:
        - -config.file=/etc/promtail/promtail.yaml           #指定从configmap定义的promtail配置文件在pod的存放位置
        - -client.url=http://192.168.37.202:3100/loki/api/v1/push
        volumeMounts:
        - name: logs
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: promtail-config
          mountPath: /etc/promtail

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: promtail-clusterrole
rules:                      #选择了给予cluster-admin的权限，解决启动时pod报权限警告
- apiGroups:
  - '*'
  resources:
  - '*'
  verbs:
  - '*'
- nonResourceURLs:
  - '*'
  verbs:
  - '*'
---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: promtail-serviceaccount

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: promtail-clusterrolebinding
subjects:
    - kind: ServiceAccount
      name: promtail-serviceaccount
      namespace: default
roleRef:
    kind: ClusterRole
    name: promtail-clusterrole
    apiGroup: rbac.authorization.k8s.io
抓取pod容器内的日志configmap配置：
[root@kubeadm loki]# cat promtail-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: promtail-config
data:
  promtail.yaml: |
    clients:
      - url: http://192.168.37.202:3100/loki/api/v1/push   #定义loki的clients
    positions:
      filename: /tmp/positions.yaml      #存放promtail位置信息的yaml文件在pod里的位置，目录路径必须是pod存在的
    server:
      http_listen_port: 8088             #开放promtail的http端口，方便浏览器查看调试
    target_config:
      sync_period: 10s
    scrape_configs:
    - job_name: kubernetes-pods-name
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - source_labels:
        - __meta_kubernetes_pod_label_name
        target_label: __service__             #前后带__表示隐藏不显示这个标签，但loki中可以搜索到
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ^$
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: instance
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container_name
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
    - job_name: kubernetes-pods-app
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: .+
        source_labels:
        - __meta_kubernetes_pod_label_name
      - source_labels:
        - __meta_kubernetes_pod_label_app
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ^$
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: instance
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container_name
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
    - job_name: kubernetes-pods-direct-controllers
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: .+
        separator: ''
        source_labels:
        - __meta_kubernetes_pod_label_name
        - __meta_kubernetes_pod_label_app
      - action: drop
        regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
        source_labels:
        - __meta_kubernetes_pod_controller_name
      - source_labels:
        - __meta_kubernetes_pod_controller_name
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ^$
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: instance
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container_name
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
    - job_name: kubernetes-pods-indirect-controller
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: .+
        separator: ''
        source_labels:
        - __meta_kubernetes_pod_label_name
        - __meta_kubernetes_pod_label_app
      - action: keep
        regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
        source_labels:
        - __meta_kubernetes_pod_controller_name
      - action: replace
        regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
        source_labels:
        - __meta_kubernetes_pod_controller_name
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ^$
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: instance
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container_name
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_uid
        - __meta_kubernetes_pod_container_name
        target_label: __path__
    - job_name: kubernetes-pods-static
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: ^$
        source_labels:
        - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_label_component
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ^$
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: instance
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container_name
      - replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
        - __meta_kubernetes_pod_container_name
        target_label: __path__

上面的配置不太好理解，请参考一下官网关于loki-promtail的配置文档，部分解说原文翻译中文如下：

regex是任何有效的 RE2正则表达式。这是必需的replace，keep，drop，labelmap，labeldrop和 labelkeep行动。正则表达式固定在两端。
确定要执行的重新标记操作：

  • replace：regex与串联的匹配source_labels。然后，设置 target_label于replacement与匹配组的引用（${1}，${2}，…）中replacement可以通过值取代。如果regex 不匹配，则不进行替换。
  • keep：删除regex与串联不匹配的目标source_labels。
  • drop：删除regex与串联的匹配的目标source_labels。
  • hashmod：设置target_label为的modulus哈希值的source_labels。
  • labelmap：regex与所有标签名称匹配。然后匹配标签的值复制到由给定的标签名称replacement与匹配组的参考（${1}，${2}，…）在replacement由他们的价值取代。
  • labeldrop：regex与所有标签名称匹配。任何匹配的标签将从标签集中删除。
  • labelkeep：regex与所有标签名称匹配。任何不匹配的标签都将从标签集中删除
抓取pod内应用程序指定日志目录如下：

前提：pod应用程序日志目录有做持久卷映射到node节点主机目录

[root@kubeadm loki]# cat promtail-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: promtail-config
data:
  promtail.yaml: |
    clients:
      - url: http://192.168.37.202:3100/loki/api/v1/push
    positions:
      filename: /var/log/positions.yaml
    server:
      http_listen_port: 8088
    target_config:
      sync_period: 10s
    scrape_configs:
    - job_name: service
      static_configs:
      - targets:
          - localhost
        labels:
          job: web01
          __path__: /var/log/web01/*/*.log       #抓取node节点/var/log/web01/*/下的所有日志
      - targets:
          - localhost
        labels:
          job: web02
          __path__: /var/log/web02/*/*.log
执行：
[root@kubeadm loki]# kubectl apply -f promtail-daemonset.yaml
[root@kubeadm loki]# kubectl apply -f promtail-configmap.yaml
[root@kubeadm loki]# kubectl get pod
NAME                                READY   STATUS    RESTARTS   AGE
promtail-daemonset-7j88j            1/1     Running   0          4h4m
promtail-daemonset-j5jps            1/1     Running   0          4h4m
promtail-daemonset-lxdtq            1/1     Running   0          4h4m
排错：

查看报错，按照相应提示解决问题

[root@kubeadm loki]# kubectl describe pod/promtail-daemonset-7j88j
[root@kubeadm loki]# kubectl logs pod/promtail-daemonset-7j88j

注意：promtail-daemonset.yaml读取promtail-configmap.yaml文件中内容是首次启动就读进了，promtail-configmap.yaml内容有更新的，需要kubectl delete -f promtail-daemonset.yaml后再重新apply promtail-daemonset.yaml文件。

附：测试环境中的实例：

[root@kubemaster01 yaml]# cat promtail.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: promtail-serviceaccount

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: promtail-clusterrole
rules:
- apiGroups:
  - '*'
  resources:
  - '*'
  verbs:
  - '*'
- nonResourceURLs:
  - '*'
  verbs:
  - '*'

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: promtail-clusterrolebinding
subjects:
    - kind: ServiceAccount
      name: promtail-serviceaccount
      namespace: default
roleRef:
    kind: ClusterRole
    name: promtail-clusterrole
    apiGroup: rbac.authorization.k8s.io

---

apiVersion: v1
kind: ConfigMap
metadata:
  name: promtail-config
data:
  promtail.yaml: |
    clients:
      - url: http://192.168.19.44:31000/loki/api/v1/push
    positions:
      filename: /var/log/positions.yaml
    server:
      http_listen_port: 8088
    target_config:
      sync_period: 10s
    scrape_configs:
    - job_name: service
      static_configs:
      - targets:
          - localhost
        labels:
          job: contract
          __path__: /ubox/logs/contract/*/*.log
      pipeline_stages:
      - match:
          selector: '{job=~".*"}'
          stages:
          - regex:
              expression: '^(?P<time>\S+ \S+) (?P<content>.*)$'
          - timestamp:
              source: time
              format: 2006-01-02 15:04:05
              location: 'Asia/Shanghai'
      - match:
          selector: '{job=~".*"}'
          stages:
          - regex:
              expression: '(?P<IP>\S+?) - (?P<user>\S+?) \[(?P<time>\S+? \+\d{4})\] (?P<content>.*)$'
          - timestamp:
              source: time
              format: 02/Jan/2006:15:04:05 -0700
              location: 'Asia/Shanghai'

---

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: promtail-daemonset
spec:
  selector:
    matchLabels:
      name: promtail
  template:
    metadata:
      labels:
        name: promtail
    spec:
      hostNetwork: true
      serviceAccount: SERVICE_ACCOUNT
      serviceAccountName: promtail-serviceaccount
      volumes:
      - name: logs
        hostPath:
          path: /var/log
      - name: servicelog
        hostPath:
          path: /ubox/logs
      - name: promtail-config
        configMap: 
          name: promtail-config
      containers:
      - name: promtail-container
        image: harbor.dev.uboxol.com/library/promtail:8809de6
        args:
        - -config.file=/etc/promtail/promtail.yaml
        - -client.url=http://192.168.19.44:31000/loki/api/v1/push
        volumeMounts:
        - name: logs
          mountPath: /var/log
        - name: servicelog
          mountPath: /ubox/logs
          readOnly: true
        - name: promtail-config
          mountPath: /etc/promtail
      imagePullSecrets:
      - name: registry-harbor

注：这个文件写了两个配置时间格式的正则，下面是日志的实例：

#日志1：
2020-07-27 13:18:38 Date-info: REQUEST_ID:5f1a6a7e27520：.......
#时间截匹配正则：
expression: '^(?P<time>\S+ \S+) (?P<content>.*)$'

#日志2：
172.16.214.64 - - [27/Jul/2020:12:00:00 +0800] "GET /contracts/index?sdfsdsdfsfsdfsd......
#时间截匹配正则：
expression: '(?P<IP>\S+?) - (?P<user>\S+?) \[(?P<time>\S+? \+\d{4})\] (?P<content>.*)$'
文档更新时间: 2020-07-27 15:45   作者：子木