一、安裝elasticsearch
官網:https://www.elastic.co/guide/index.html
https://www.elastic.co/guide/en/elasticsearch/reference/2.3/index.html
參考搭建文檔
Elasticsearch-6.0.0
logstash-6.0.0
kibana-6.0.0
filebeat-6.0.0
https://blog.51cto.com/zero01/2079879 上篇
https://blog.51cto.com/zero01/2082794 下篇
參考 https://www.cnblogs.com/superlinux/p/10591428.html
filebeat寫入kafka :https://www.jianshu.com/p/da8113f58115
1)環境準備
[root@k8s6 ~]# java -version
openjdk version "1.8.0_201"
OpenJDK Runtime Environment (build 1.8.0_201-b09)
OpenJDK 64-Bit Server VM (build 25.201-b09, mixed mode)
1)
[root@k8s6 ~]# ls elktools/
elasticsearch-2.3.5.rpm kibana-4.5.4-1.x86_64.rpm logstash-2.3.4-1.noarch.rpm
View Code
2)rpm包安裝
[root@k8s6 ~]# rpm -ivh elktools/elasticsearch-2.3.5.rpm
3)修改配置檔案
備份配置檔案
cp /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml.bak
修改配置檔案
[root@k8s6 elasticsearch]# diff /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml.bak
17c17
< cluster.name: myes
---
> # cluster.name: my-application
23c23
< node.name: master_node
---
> # node.name: node-1
54c54
< network.host: 192.168.10.22
---
> # network.host: 192.168.0.1
58c58
< http.port: 9200
---
> # http.port: 9200
View Code
補充:
bootstrap.mlockall: false
bootstrap.system_call_filter: false
4)啟動服務
[root@k8s6 elasticsearch]# systemctl start elasticsearch 啟動服務
[root@k8s6 elasticsearch]# netstat -lntup|grep java
tcp6 0 0 192.168.10.22:9200 :::* LISTEN 25874/java
tcp6 0 0 192.168.10.22:9300 :::* LISTEN 25874/java
View Code
5)對elasticsearch的api驗證
[root@k8s6 elasticsearch]# curl http://192.168.10.22:9200
{
"name" : "master_node",
"cluster_name" : "myes",
"version" : {
"number" : "2.3.5",
"build_hash" : "90f439ff60a3c0f497f91663701e64ccd01edbb4",
"build_timestamp" : "2016-07-27T10:36:52Z",
"build_snapshot" : false,
"lucene_version" : "5.5.0"
},
"tagline" : "You Know, for Search"
}
View Code
二、安裝elasticsearch的插件安裝
1)檢視執行的指令和插件位置
[root@k8s6 ~]# ls /usr/share/elasticsearch/
bin lib LICENSE.txt modules NOTICE.txt plugins README.textile
2)安裝head插件
[root@k8s6 ~]# /usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head 從github下載下傳
[root@k8s6 ~]# ls /usr/share/elasticsearch/plugins/head/
Dockerfile Gruntfile.js LICENCE proxy src
Dockerfile-alpine grunt_fileSets.js package.json README.textile test
elasticsearch-head.sublime-project index.html plugin-descriptor.properties _site
2.1)通路測試
http://192.168.10.22:9200/_plugin/head/
3.1)安裝 kopf 插件
[root@k8s6 ~]# /usr/share/elasticsearch/bin/plugin install lmenezes/elasticsearch-kopf
3.2)通路測試
http://192.168.10.22:9200/_plugin/kopf/
4)模拟送出post請求
重新整理連接配接
二、es的叢集測試
1)另一台機器同樣安裝好es,修改配置檔案(主播模式)測試vm會存在問題
[root@node01 elasticsearch]# grep '^[a-Z]' elasticsearch.yml
cluster.name: myes
node.name: node01
path.data: /data/es-data
path.logs: /var/log/elasticsearch
bootstrap.mlockall: false
bootstrap.system_call_filter: false
network.host: 192.168.10.23
http.port: 9200
建立目錄,并授權
[root@node01 elasticsearch]# mkdir -p /data/es-data
[root@node01 elasticsearch]# chown elasticsearch:elasticsearch /data/es-data/
2)單點傳播模式
[root@node01 elasticsearch]# grep '^[a-Z]' elasticsearch.yml
cluster.name: myes
node.name: node01
path.data: /data/es-data
path.logs: /var/log/elasticsearch
bootstrap.mlockall: true
network.host: 192.168.10.23
http.port: 9200
discovery.zen.ping.unicast.hosts: ["192.168.10.22", "192.168.10.23"]
3)啟動服務
強調,一點要關閉防火牆。否則無法弄成叢集
[root@node01 elasticsearch]# /etc/init.d/elasticsearch start
4)查詢叢集的狀态
[root@node01 ~]# curl -XGET 'http://192.168.10.22:9200/_cluster/health?pretty=true'
{
"cluster_name" : "myes",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 2, # 2個節點
"number_of_data_nodes" : 2,
"active_primary_shards" : 5,
"active_shards" : 10,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
View Code
三、在主節點安裝 logstash
1)安裝logstash
[root@k8s6 elktools]# rpm -ivh logstash-2.3.4-1.noarch.rpm
準備中... ################################# [100%]
正在更新/安裝...
1:logstash-1:2.3.4-1 ################################# [100%]
[root@k8s6 elktools]# rpm -qa|grep logstash
logstash-2.3.4-1.noarch
[root@k8s6 elktools]# rpm -ql logstash # 檢視安裝生成了哪些檔案
View Code
2)模拟日志标準輸出:/opt/logstash/bin/logstash -e 'input { stdin{} } output { stdout{} }'
[root@k8s6 elktools]# /opt/logstash/bin/logstash -e 'input { stdin{} } output { stdout{} }'
Settings: Default pipeline workers: 4
Pipeline main started
hello
2019-03-15T04:42:43.106Z k8s6 hello
hello world
2019-03-15T04:42:51.121Z k8s6 hello world
View Code
json格式輸出: /opt/logstash/bin/logstash -e 'input { stdin{} } output { stdout{ codec => rubydebug } }'
[root@k8s6 elktools]# /opt/logstash/bin/logstash -e 'input { stdin{} } output { stdout{ codec => rubydebug } }'
Settings: Default pipeline workers: 4
Pipeline main started
hello world
{
"message" => "hello world",
"@version" => "1",
"@timestamp" => "2019-03-15T04:47:12.509Z",
"host" => "k8s6"
}
View Code
3)指令行模拟輸出到elasticsearch: /opt/logstash/bin/logstash -e 'input { stdin{} } output { elasticsearch{ hosts => ["192.168.10.22:9200"] index => "logstash-%{+YYYY.MM.dd}" } }'
[root@k8s6 elktools]# /opt/logstash/bin/logstash -e 'input { stdin{} } output { elasticsearch{ hosts => ["192.168.10.22:9200"] index => "logstash-%{+YYYY.MM.dd}" } }'
Settings: Default pipeline workers: 4
Pipeline main started
hhhee
View Code
查詢到輸出的值
4)同時進行螢幕輸出并寫入es
[root@k8s6 elktools]# /opt/logstash/bin/logstash -e 'input { stdin{} } output { stdout{ codec => rubydebug } elasticsearch{ hosts => ["192.168.10.22:9200"] index => "logstash-%{+YYYY.MM.dd}" } }'
Settings: Default pipeline workers: 4
Pipeline main started
天天向上
{
"message" => "天天向上",
"@version" => "1",
"@timestamp" => "2019-03-15T05:02:55.388Z",
"host" => "k8s6"
}
View Code
四、使用配置檔案啟動logstash
檢測文法
[root@node01 ~]# /opt/logstash/bin/logstash -t -f /etc/logstash/conf.d/nginx.conf
Configuration OK
1)進入到配置檔案目錄。cd /etc/logstash/conf.d/ ,該目錄由/etc/init.d/logstash 這裡定義的
[root@k8s6 elktools]# cd /etc/logstash/conf.d/
[root@k8s6 conf.d]# ls
[root@k8s6 conf.d]#
View Code
編輯 demo.conf檔案,既輸出在螢幕,也存入es
YYYY.MM.dd 每天
[root@k8s6 conf.d]# cat demo.conf
input {
stdin{}
}
filter{
}
output{
elasticsearch {
hosts => ["192.168.10.22:9200"]
index => "logstash-%{+YYYY.MM.dd}"
}
stdout{ codec => rubydebug }
}
View Code
啟動服務:[root@k8s6 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/demo.conf
2)使用file插件,指定要記錄日志的目錄
[root@k8s6 conf.d]# cat file.conf
input {
file{
path => ["/var/log/messages", "/var/log/secure"]
type => "system-log"
start_position => "beginning"
}
}
filter{
}
output{
elasticsearch {
hosts => ["192.168.10.22:9200"]
index => "system-log-%{+YYYY.MM}"
}
}
file.conf
啟動服務: /opt/logstash/bin/logstash -f /etc/logstash/conf.d/file.conf
五、安裝kibana
1)在主節點安裝kibana
[root@k8s6 elktools]# rpm -ivh kibana-4.5.4-1.x86_64.rpm
檢視安裝到了哪些位置
[root@k8s6 elktools]# rpm -ql kibana
View Code
2)修改配置檔案
[root@k8s6 config]# pwd
/opt/kibana/config
[root@k8s6 config]# grep '^[a-Z]' kibana.yml
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.url: "http://192.168.10.22:9200"
kibana.index: ".kibana"
/opt/kibana/config/kibana.yml
啟動服務:[root@k8s6 config]# /etc/init.d/kibana start
[root@k8s6 config]# netstat -lntup|grep 5601
tcp 0 0 0.0.0.0:5601 0.0.0.0:* LISTEN 31271/node
網頁通路:192.168.10.22:5601
2)配置索引,配置哪個,顯示哪個的日志檔案
比對到了建立,即可
補充知識
import requests
from requests.auth import HTTPBasicAuth
res = requests.get('https://elk0001.eniot.io:9200/_cluster/health/?pretty',verify=False, auth=HTTPBasicAuth('user', 'password'))
print(res.text)
# curl --insecure --user user:password -XGET "https://elk-log0001.eniot.io:9200/_cluster/health/?pretty"