ELK之logstash配置
一、logstash直接监听日志文件`
input{
file {
path => "/tmp/logs/info/gateway.log"
start_position => "beginning"
type => "log1"
}
file {
path => "/tmp/logs/debug/boot.log"
start_position => "beginning"
type => "log2"
}
}
output{
if[type] == "log1"{
elasticsearch {
hosts => ["192.167.213.11:9100"]
index => "gateway-%{+YYYY.MM.dd}"
}
}
if[type] == "log2"{
elasticsearch {
hosts => ["192.168.213.11:9100"]
index => "boot-%{+YYYY.MM.dd}"
}
}
}
二、filebeat整合logstash
filebeat.yml
# filebeat.yml
filebeat.prospectors:
- type: log
paths:
- /tmp/logs/info/gate.log
input_type: log
tags: ["log1"]
fields:
type: gate
document_type: gate
multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
multiline.negate: true
multiline.match: after
- type: log
paths:
-/tmp/logs/debug/boot.log
input_type: log
tags: ["log2"]
fields:
type: boot
document_type: boot
multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
multiline.negate: true
multiline.match: after
fields:
env: develop-203.11
output:
logstash:
hosts: ["192.168.203.11:5000"]
注:filebeat的文件由于是yml 所以语法严谨一些
logstash.conf
input{
beats {
port => 5000
codec => "json"
}
}
filter {
grok {
match => {
"message" => "(?<date>(\d*[./-]\d*[./-]\d* \d*:\d*:\d*[.,][0-9]+)) .*%{LOGLEVEL:level} .*\- \[%{USERNAME:requestId}\] \[%{HOSTNAME:ip}\] \[%{USERNAME:channel}\] \[%{USERNAME:serviceName}\] \[%{INT:spendTime}\ms] \[%{INT:code}]"
}
overwrite => ["message"]
}
mutate {
convert => ["spendTime", "integer"]
}
}
output{
if "log1" in [tags]{
elasticsearch {
hosts => ["192.168.213.11:9200"]
index => "gate-%{+YYYY.MM.dd}"
}
}
if "log2" in [tags]{
elasticsearch {
hosts => ["192.168.213.11:9200"]
index => "boot-%{+YYYY.MM.dd}"
}
}
}