elk各種插件的使用（3）

#接上一節的内容

輸入插件： input

192.168.43.60 node60

[root@node60 /]# yum -y install nginx

[root@node60 /]# systemctl restart nginx

[root@node60 /]# cd /var/log/nginx

輸入插件：file

file插件：用于讀取指定日志檔案

常用字段：

• path 日志檔案路徑，可以使用通配符

• exclude 排除采集的日志檔案

• start_position 指定日志檔案什麼位置開始讀，預設從結尾

開始，指定beginning表示從頭開始讀

讀取日志檔案并輸出到檔案

input {

     file {

     path => "/var/log/test/*.log"

     exclude => "error.log"

     start_position => "beginning"

     }

}

filter {

output {

     path => "/tmp/test.log"

   }

[root@node3 bin]# ./logstash -f /opt/elk/logstash/conf.d/test.conf

輸出詳細的資訊

sending logstash logs to /opt/elk/logstash/logs which is now configured via log4j2.properties

[2021-11-05t12:59:22,918][info ][logstash.runner          ] starting logstash {"logstash.version"=>"7.9.3", "jruby.version"=>"jruby 9.2.13.0 (2.5.7) 2020-08-03 9a89c94bcc openjdk 64-bit server vm 25.312-b07 on 1.8.0_312-b07 +indy +jit [linux-x86_64]"}

[2021-11-05t12:59:26,001][warn ][logstash.config.source.multilocal] ignoring the 'pipelines.yml' file because modules or command line options are specified

[2021-11-05t12:59:35,499][info ][org.reflections.reflections] reflections took 304 ms to scan 1 urls, producing 22 keys and 45 values

[2021-11-05t12:59:39,245][info ][logstash.javapipeline    ][main] starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>250, "pipeline.sources"=>["/opt/elk/logstash/conf.d/test.conf"], :thread=>"#<thread:0x1de7aa68 run>"}

[2021-11-05t12:59:42,917][info ][logstash.javapipeline    ][main] pipeline java execution initialization time {"seconds"=>3.58}

[2021-11-05t12:59:44,265][info ][logstash.inputs.file     ][main] no sincedb_path set, generating one based on the "path" setting {:sincedb_path=>"/opt/elk/logstash/data/plugins/inputs/file/.sincedb_54fd3bc452299b50db7e60530cbeaef2", :path=>["/var/log/test/*.log"]}

[2021-11-05t12:59:44,478][info ][logstash.javapipeline    ][main] pipeline started {"pipeline.id"=>"main"}

[2021-11-05t12:59:44,979][info ][logstash.agent           ] pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}

[2021-11-05t12:59:45,174][info ][filewatch.observingtail  ][main][c5d8f8115c91d1d476c4f03565b3fffbf48137506f07684afc6b5d0d32fdc284] start, creating discoverer, watch with file and sincedb collections

[2021-11-05t12:59:47,285][info ][logstash.agent           ] successfully started logstash api endpoint {:port=>9600}

[2021-11-05t12:59:49,633][info ][logstash.outputs.file    ][main][9884c6f8b9b568b3031858d3a5a03ac64b44f4e7e9f56741573a4f5d1a651706] opening file

{:path=>"/tmp/test.log"}

[2021-11-05t13:00:19,623][info ][logstash.outputs.file    ][main][9884c6f8b9b568b3031858d3a5a03ac64b44f4e7e9f56741573a4f5d1a651706] closing file /tmp/test.log

輸入插件 file

示例：配置日志來源

    file {

    path => "/var/log/test/*.log"      #定義日志的類型

    exclude => "error.log"              #排除錯誤日志

    start_position => "beginning"    #收集日志的位置

    tags => "web"                            #web項目

    tags => "nginx"                          #服務nginx

    type => "access"                        日志的類型

    add_field => {

   "project" => "microservice"           #微服務

   "app" => "product"                       #定義生産環境

      }

    }

[root@node3 bin]# ./logstash -f /opt/elk/logstash/conf.d/test1.conf

[2021-11-05t13:12:45,892][info ][logstash.runner          ] starting logstash {"logstash.version"=>"7.9.3", "jruby.version"=>"jruby 9.2.13.0 (2.5.7) 2020-08-03 9a89c94bcc openjdk 64-bit server vm 25.312-b07 on 1.8.0_312-b07 +indy +jit [linux-x86_64]"}

[2021-11-05t13:12:48,741][warn ][logstash.config.source.multilocal] ignoring the 'pipelines.yml' file because modules or command line options are specified

[2021-11-05t13:12:56,050][error][logstash.agent           ] failed to execute action {:action=>logstash::pipelineaction::create/pipeline_id:main, :exception=>"logstash::configurationerror", :message=>"expected one of [ \\t\\r\\n], \"#\", \"{\", [a-za-z0-9_-], '\"', \"'\", \"}\" at line 10, column 45 (byte 314) after input {\n\n    file {\n\n    path => \"/var/log/test/*.log\"      #定義日志的類型\n    exclude => \"error.log\"              #排除錯誤日志\n    start_position => \"beginning\"    #收集日志的位置\n    tags => \"web\"                            #web項目\n    tags => \"nginx\"                          #服務nginx\n    type => \"access\"                        ", :backtrace=>["/opt/elk/logstash/logstash-core/lib/logstash/compiler.rb:32:in `compile_imperative'", "org/logstash/execution/abstractpipelineext.java:183:in `initialize'", "org/logstash/execution/javabasepipelineext.java:69:in `initialize'", "/opt/elk/logstash/logstash-core/lib/logstash/java_pipeline.rb:47:in `initialize'", "/opt/elk/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:52:in `execute'", "/opt/elk/logstash/logstash-core/lib/logstash/agent.rb:357:in `block in converge_state'"]}

[2021-11-05t13:12:57,737][info ][logstash.agent           ] successfully started logstash api endpoint {:port=>9600}