Docker swarm （五）filebeat日志收集

前提条件：

• swarm环境
• 有 ES、LogStash和kibana
• 只需要通过filebeat发送到ES然后使用kibana展示即可

收集java日志

程序启动要指定一个日志输出文件，并将此文件目录挂载到宿主机，如果不使用此方法，filebeat将没有权限访问默认输出目录，/data/docker_dir/containers/

宿主机目录（多节点则都要创建）：

/data/docker_filebeat/

• conf 存放配置文件
• logs 存放java容器日志
• registry 权限777，filebeat会在目录下创建文件

示例

java容器Dockerfile

最后一条 将日志发送到/root/logs/saas-admin.log

FROM docker.wecode123.com:30080/network/centos:7.5.1804
 
COPY saas-admin.jar saas-admin.jar
 
COPY argvtest.py /home/work/workspace/online_codeus/docs/argvtest.py
COPY format_code.py /home/work/workspace/online_codeus/docs/format_code.py
COPY .terminalfx /root/.terminalfx
COPY magic_box.py /usr/local/python3/lib/python3.6/magic_box.py
COPY wecode.py /usr/local/python3/lib/python3.6/wecode.py
COPY codeus.py /usr/local/python3/lib/python3.6/codeus.py
ADD pinpoint.tar.gz /
 
ENV LANG=en_US.UTF-8
 
ENV JAVA_HOME=/usr/local/jdk
ENV PATH=$JAVA_HOME/bin:$PATH
 
ENV export JAVA_HOME M2_HOME
ENV PYTHON_HOME=/usr/local/python3
ENV export PYTHON_HOME
ENV export CLASSPATH=.
ENV PATH=$JAVA_HOME/bin:$M2_HOME/bin:$PYTHON_HOME/bin:$PATH:$HOME/.local/bin:$HOME/bin
 
ENV PTY_LIB_FOLDER=/root/.terminalfx/libpty
ENV export PTY_LIB_FOLDER
 
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo 'Asia/Shanghai' >/etc/timezone
 
#CMD exec java -jar cloudcompiler-1.0-SNAPSHOT.jar --spring.profiles.active=$CONFIG_FILE
CMD exec java -jar saas-admin.jar --server.port=20240 --spring.profiles.active=qc >> /root/logs/saas-admin.log
           

主要就是最后一条日志输出，其它的根据自己环境定义就行啦

启动使用的是stack

使用volumes将日志挂载到宿主机，两个节点的话都需要创建挂载目录

version: '3'
 
services:
 
  saas-admin-demo:
    image: docker_image
    ports:
      - 30330:20240
    networks:
      - saas-demo
    volumes:
      - /data/docker_filebeat/logs/:/root/logs/
    deploy:
      mode: replicated
      replicas: 2
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 3
      update_config:
        parallelism: 1
        delay: 10s
 
networks:
  saas-demo:
    driver: overlay
           

启动后查看目录中是否有日志，有日志继续下面操作

部署filebeat

配置文件

# cat /data/docker_filebeat/conf/filebeat.yml

直接指向Logstash

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/docker/*.log
 
output.logstash.hosts: ["172.17.202.147:5044"]
           

Logstash文件

# cat app.conf
input {
    beats {
        port => "5044"
    }
}
 
filter {
    mutate {
        rename => { "[host][name]" => "host" }
    }
    multiline {
         pattern => "^202.*-.*-.*"
         negate => true
         what => "previous"
    }
}
 
filter {
    mutate {
        rename => { "[host][name]" => "host" }
    }
    if [type] == "codeus-app02-accesslog" {
        grok {
            match => ["message","%{JAVA_DATE:date} %{JAVA_TIME:time} %{JAVA_PORT:port} %{JAVA_LOGLEVEL:loglevel} %{JAVA_LOGTYPE:logtype} %{JAVA_NUMBER:number} %{JAVA_NULL:null} %{JAVA_DATE_TIME:date_time} %{JAVA_USERID:userID} %{JAVA_USERIP:userIP} %{JAVA_BREXPO:brexpo} %{JAVA_CONNECT:connect}"]
        }
    }
}
 
filter {
    mutate {
        rename => { "[host][name]" => "host" }
    }
    if [type] == "codeus-app02-website-accesslog" {
        grok {
            match => ["message","%{JAVA_DATE:date} %{JAVA_TIME:time} %{JAVA_PORT:port} %{JAVA_LOGLEVEL:loglevel} %{JAVA_LOGTYPE:logtype} %{JAVA_NUMBER:number} %{JAVA_NULL:null} %{JAVA_DATE_TIME:date_time} %{JAVA_USERID:userID} %{JAVA_USERIP:userIP} %{JAVA_BREXPO:brexpo} %{JAVA_CONNECT:connect}"]
        }
    }
}
 
filter {
    mutate {
        rename => { "[host][name]" => "host" }
    }
    if [type] == "codeus-app01-accesslog" {
        grok {
            match => ["message","%{JAVA_DATE:date} %{JAVA_TIME:time} %{JAVA_PORT:port} %{JAVA_LOGLEVEL:loglevel} %{JAVA_LOGTYPE:logtype} %{JAVA_NUMBER:number} %{JAVA_NULL:null} %{JAVA_DATE_TIME:date_time} %{JAVA_USERID:userID} %{JAVA_USERIP:userIP} %{JAVA_BREXPO:brexpo} %{JAVA_CONNECT:connect}"]
        }
    }
}
 
filter {
    mutate {
        rename => { "[host][name]" => "host" }
    }
    if [type] == "codeus-app01-website-accesslog" {
        grok {
            match => ["message","%{JAVA_DATE:date} %{JAVA_TIME:time} %{JAVA_PORT:port} %{JAVA_LOGLEVEL:loglevel} %{JAVA_LOGTYPE:logtype} %{JAVA_NUMBER:number} %{JAVA_NULL:null} %{JAVA_DATE_TIME:date_time} %{JAVA_USERID:userID} %{JAVA_USERIP:userIP} %{JAVA_BREXPO:brexpo} %{JAVA_CONNECT:connect}"]
        }
    }
}
 
output {
#    stdout { codec => rubydebug }
    elasticsearch {
        hosts => "127.0.0.1"
        index => "logstash-%{+YYYY.MM.dd}"
    }
}
           

启动，通过stack，直接复制代码到web页面

# cat stack-filebeat.yml

version: '3'
 
services:
 
  docker-filebeat:
    image: docker.elastic.co/beats/filebeat:7.2.0
    volumes:
      - /data/docker_filebeat/logs/:/var/log/docker:ro
      - /data/docker_filebeat/conf/filebeat.yml:/usr/share/filebeat/filebeat.yml
      - /data/docker_filebeat/registry/:/usr/share/filebeat/data/registry/
    deploy:
      mode: replicated
      replicas: 2
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 3
           

查看filebeat容器日志，出现下面内容表示成功

# docker logs -f filebeat-container-name

在kibana查看

设置中添加索引 

具体日志属于哪个程序可以进一步修改filebeat配置文件，或者根据现有文件中的 log.file.path 索引判断，例如：