[ELK] Server 安装

一、ELK 相关资料

ELK官网：点击打开链接

ELKstack 中文指南：点击打开链接

二、安装过程

节点1：172.214.5.19

节点2：172.216.18.40

节点3：172.216.33.100

1、Java安装

# yum -y install java-1.8.0

# vi /etc/profile

[ELK] Server 安装

2、elasticsearch安装

[ELK] Server 安装

# yum -y install elasticsearch

# mkdir -p /data/elk/data /data/elk/logs

# chown -R elasticsearch /data/elk

# vi /etc/elasticsearch/elasticsearch.yml # 三个节点分别安装 elasticsearch

[ELK] Server 安装

# vi /etc/security/limits.conf

[ELK] Server 安装

# vi /usr/lib/systemd/system/elasticsearch.service

[ELK] Server 安装

# systemctl daemon-reload

# systemctl enable elasticsearch

# systemctl start elasticsearch

# tailf/data/elk/logs/wxtky-elk.log

# netstat -tunlp | egrep "9200|9300"

3、安装head

# yum -y install git

# cd /data/elk

# git clonegit://github.com/mobz/elasticsearch-head.git

# cd elasticsearch-head

# wget https://nodejs.org/dist/v4.6.1/node-v4.6.1-linux-x64.tar.gz

# tar -zxvf node-v4.6.1-linux-x64.tar.gz

# vi /etc/profile

[ELK] Server 安装

# source /etc/profile

# npm install -g cnpm --registry=https://registry.npm.taobao.org

# npm install –g grunt–cli # npm install grunt 多试几次，我这边用这个成功了

# vi Gruntfile.js +94

[ELK] Server 安装

# vi /etc/elasticsearch/elasticsearch.yml # elk5中安装head要求配置以下两项

[ELK] Server 安装

# vi _site/app.js +4329

[ELK] Server 安装

# cd /data/elk/elasticsearch-head

# grunt server

[ELK] Server 安装

# 开始安装

[ELK] Server 安装

。。。。。。

# 全部安装完毕后再次 grunt server

按照提示打开 head插件页面

4、Kibana 安装

# yum安装

# rpm --importhttps://packages.elastic.co/GPG-KEY-elasticsearch

# vi /etc/yum.repos.d/kibana.repo

【kibana-4.6】

name=Kibana repository for 4.6.x packages

baseurl=https://packages.elastic.co/kibana/4.6/centos

gpgcheck=1

gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch

enabled=1

# systemctl enable kibana

# 配置

# vi /opt/kibana/config/kibana.yml

server.port: 5601

server.host: "0.0.0.0"

elasticsearch.url: "http://172.214.5.19:9200" #节点2修改

kibana.index: ".kibana"

# firewall-cmd --permanent --add-port='5601/tcp'

# 启动

# systemctl daemon-reload

# systemctl start kibana

# 登入测试

http://172.214.5.19:5601

5、kafka安装（节点1&2）

# 解压安装

# cd/usr/local/src

# wget http://mirror.rise.ph/apache/kafka/0.10.2.0/kafka_2.12-0.10.2.0.tgz

# tar -xf kafka_2.12-0.10.2.0.tgz

# mv kafka_2.12-0.10.2.0 /usr/local/

# ln -sv kafka_2.12-0.10.2.0/ kafka

# zookeeper集群配置

# vi /usr/local/kafka/config/zookeeper.properties

dataDir=/data/zookeeper/data

dataLogDir=/data/zookeeper/log

clientPort=2181

tickTime=2000

initLimit=20

syncLimit=10

server.1=172.214.5.19:2888:3888

server.2=172.216.18.40:2888:3888

# mkdir /data/zookeeper

# echo 1 > /data/zookeeper/myid #节点2修改 echo 2 > /data/zookeeper/myid

# vi /usr/local/kafka/config/server.properties

broker.id=1 #节点 2修改

listeners=PLAINTEXT://172.214.5.19:9092 #节点2修改

num.network.threads=3

num.io.threads=8

socket.send.buffer.bytes=102400

socket.receive.buffer.bytes=102400

socket.request.max.bytes=104857600

log.dirs=/data/kafka-logs #该目录会自动新建

num.partitions=16

num.recovery.threads.per.data.dir=1

log.retention.hours=168

log.segment.bytes=1073741824

log.retention.check.interval.ms=300000

zookeeper.connect=172.214.5.19:2181,172.216.18.40:2181

zookeeper.connection.timeout.ms=6000

# 两个节点分别启动

# screen-S zookeeper

# /usr/local/kafka/bin/zookeeper-server-start.sh/usr/local/kafka/config/zookeeper.properties #启集群

# /usr/local/kafka/bin/zookeeper-server-stop.sh #停

# exit

# screen -S kafka

# /usr/local/kafka/bin/kafka-server-start.sh/usr/local/kafka/config/server.properties #启kafka

# /usr/local/kafka/bin/kafka-server-stop.sh #停

# exit

# 两个节点防火墙设置

# firewall-cmd --permanent--add-port={2888/tcp,3888/tcp,2181/tcp,9092/tcp}

# firewall-cmd --reload

# 测试

# 新建一个主题

# /usr/local/kafka/bin/kafka-topics.sh --create--zookeeper localhost:2181 --replication-factor2 --partitions 1 --topic summer

# 查看刚创建的主题

# /usr/local/kafka/bin/kafka-topics.sh --list --zookeeper172.214.5.19:2181

# 查看summer主题的详情

# /usr/local/kafka/bin/kafka-topics.sh --describe --zookeeper172.214.5.19:2181 --topic summer

# 发送消息

# /bin/bash/usr/local/kafka/bin/kafka-console-producer.sh --broker-list 172.214.5.19:9092 --topic summer

this is anews

# 接收消息

# /usr/local/kafka/bin/kafka-console-consumer.sh--zookeeper 172.216.18.40:2181 --topic summer--from-beginning

6、logstash安装（节点1&2）

# yum安装

# rpm --importhttps://packages.elastic.co/GPG-KEY-elasticsearch

# vi /etc/yum.repos.d/logstash.repo

[logstash-2.4]

name=Logstash repository for 2.4.x packages

baseurl=https://packages.elastic.co/logstash/2.4/centos

gpgcheck=1

gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch

enabled=1

# yuminstall -y logstash

# chkconfig logstash on

# syslog 监控

# ssh 172.216.33.100

# vi /etc/rsyslog.conf

*.* @@172.214.5.19:514

# systemctl restart rsyslog

# exit

# 回到节点1

# cd /etc/logstash/conf.d/

# viinput.conf

input {

syslog {

type => "system-syslog"

host => "172.214.5.19"

port => "514"

}

output {

kafka { #输出到kafka

bootstrap_servers => "172.214.5.19:9092,172.216.18.40:9092" #他们就是生产者

topic_id => "system-syslog" #这个将作为主题的名称，将会自动创建

compression_type => "snappy" #压缩类型

}

# vi output.conf

input {

kafka {

zk_connect => "172.214.5.19:2181,172.216.18.40:2181" #消费者们

topic_id => "system-syslog"

codec => plain

reset_beginning => false

consumer_threads => 5

decorate_events => true

}

output {

elasticsearch {

hosts => ["192.168.1.203:9200","192.168.1.204:9200"]

index => "system-syslog-%{+YYYY-MM-dd}" #为了区分之前实验，我这里新生成的所以名字为“test-system-messages-%{+YYYY-MM}”

}

# /etc/init.d/logstash start

# 测试查看kafka是否有system-syslog主题

# /usr/local/kafka/bin/kafka-topics.sh --list --zookeeper172.214.5.19:2181

system-syslog

# URL登入

http://172.214.5.19:9100

# kibana新建索引

# http://172.214.5.19:5601

填写 system-syslog-*

7、nginx负载均衡（172.216.33.100 端）

# 下载安装

# cd /usr/local/src/

# wget http://nginx.org/download/nginx-1.11.12.tar.gz

# tar -zxf nginx-1.11.12.tar.gz

# yum -y install zlib zlib-devel openssl openssl-devel pcre pcre-devel

# cd nginx-1.11.12/

# ./configure --prefix=/usr/local/nginx

# make && make install

# /usr/local/nginx/sbin/nginx-t #测试

# 配置

# firewall-cmd --permanent --add-port='80/tcp'

# firewall-cmd --reload

# vi /usr/local/nginx/conf/nginx.conf

#user nobody;

worker_processes 1;

#error_log logs/error.log;

#error_log logs/error.log notice;

#error_log logs/error.log info;

#pid logs/nginx.pid;

events {

worker_connections 1024;

}

http {

include /usr/local/nginx/conf.d/*.conf; #修改点

include mime.types;

default_type application/octet-stream;

#log_format main '$remote_addr - $remote_user [$time_local] "$request" '

# '$status $body_bytes_sent "$http_referer" '

# '"$http_user_agent" "$http_x_forwarded_for"';

#access_log logs/access.log main;

sendfile on;

#tcp_nopush on;

#keepalive_timeout 0;

keepalive_timeout 65;

#gzip on;

#server { # server注释掉

#listen 80;

# server_name localhost;

#charset koi8-r;

#access_log logs/host.access.log main;

# location / {

# root html;

# index index.html index.htm;

# }

#error_page 404 /404.html;

# redirect server error pages to the static page /50x.html

# error_page 500 502 503 504 /50x.html;

# location = /50x.html {

# root html;

# }

# proxy the PHP scripts to Apache listening on 127.0.0.1:80

#location ~ \.php$ {

# proxy_pass http://127.0.0.1;

# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000

#location ~ \.php$ {

# root html;

# fastcgi_pass 127.0.0.1:9000;

# fastcgi_index index.php;

# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;

# include fastcgi_params;

# deny access to .htaccess files, if Apache's document root

# concurs with nginx's one

#location ~ /\.ht {

# deny all;

}

# another virtual host using mix of IP-, name-, and port-based configuration

#server {

# listen 8000;

# listen somename:8080;

# server_name somename alias another.alias;

# location / {

# root html;

# index index.html index.htm;

# }

# HTTPS server

#server {

# listen 443 ssl;

# server_name localhost;

# ssl_certificate cert.pem;

# ssl_certificate_key cert.key;

# ssl_session_cache shared:SSL:1m;

# ssl_session_timeout 5m;

# ssl_ciphers HIGH:!aNULL:!MD5;

# ssl_prefer_server_ciphers on;

# location / {

# root html;

# index index.html index.htm;

# }

# mkdir /usr/local/nginx/conf.d/

# vi /usr/local/nginx/conf.d/es.conf

upstream es {

server 172.214.5.19:5601 max_fails=3 fail_timeout=30s;

server 172.216.18.40:5601 max_fails=3 fail_timeout=30s;

}

server {

listen 80;

server_name localhost;

location / {

proxy_pass http://es/;

index index.html index.htm;

#auth

auth_basic "ELK Private";

auth_basic_user_file /usr/local/nginx/.htpasswd;

}

# /usr/local/nginx/sbin/nginx-t #测试配置文件正确性

# yum -y install httpd-tools # htpasswd命令包

# htpasswd -cm/usr/local/nginx/.htpasswd elk #新建elk用户

# /usr/local/nginx/sbin/nginx #启

# ps -ef | grepnginx | grep -v grep | awk '{print $2}' | xargs kill -9 #停

# 登入测试

URL:http://172.216.33.100

输入账号密码

[ELK] Server 安装

继续阅读

kafka环境部署(二)

Kafka学习篇（二）——Kafka环境搭建安装JDK安装KafkaKafka命令Kafka配置

大白话带你认识Kafka一、Kafka基础1.Topic 主题二、kafka的集群架构3.Kafka的网络设计finally

kafka使用笔记-librdkafka支持sasl认证

librdkafka consumer封装的一点总结

用c++ 连接kafka我所踩过的坑(Connection refused || desired partition does not exist)

Linux 安装kafka的库librdkafka

librdkafka安装步骤

Kafka：Topic概念与API介绍

Doris SQL日志审计部署，以及sql收集输出kafka，后续血缘分析

Flink Kafka Doris实战demo

延云行业搜索数据库在大数据生态中位置和重要性大数据的挑战大数据技术的现状延云行业搜索数据库

Kafka 和 EMS 消息批量 ack 的实现

Spring Cloud整合Sleuth，当请求完成后，Zipkin没有链路信息

尚硅谷—韩顺平—图解 Java设计模式（结构型）（55～）

30天了解30种技术系列---(10)面向Cloud的搜索引擎 ElasticSearch