Hadoop元件單機版——shell腳本一鍵安裝

在/opt/下存放安裝的各個軟體，然後安裝到/opt/soft/下

#! /bin/bash
echo 'full system install begining....'

#global var 
jdk=true
mysql=false
hadoop=true
hive=true
zookeeper=true
hbase=true
sqoop=true
spark=true
flume=false
kafka=false
flink=false

#rpm -qa | grep java 
#rpm -e --nodeps ...

#setup jdk 1.8.111
if [ "$jdk" = true ];then
	mkdir /opt/soft
	echo 'setup jdk 1.8.111'
	tar -zxf /opt/jdk-8u111-linux-x64.tar.gz -C /opt/soft
	mv /opt/soft/jdk1.8.0_111 /opt/soft/jdk180
	echo '#jdk' >> /etc/profile
	echo 'export JAVAHOME=/opt/soft/jdk180' >> /etc/profile
	echo 'export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar' >> /etc/profile
	echo 'export PATH=$JAVAHOME/bin:$PATH' >> /etc/profile
fi

#setup mysql 5.7
if [ "$mysql" = true ];then
	echo 'setup mysql 5.7'
	var1=rpm -qa|grep mariadb
	yum remove -y $var1
	cd /opt/
	wget  http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm
	rpm  -ivh mysql-community-release-el7-5.noarch.rpm
	yum -y install mysql-server
	chown -R root:root /var/lib/mysql
	chown root /var/lib/mysql
	systemctl restart mysql
	mysql -h localhost -P3306 -uroot  mysql  --default-character-set=utf8 -e "use mysql;update user set password=password('root') where user='root';grant all on *.* to [email protected]'%' identified by 'root';flush privileges;"
	sed -i '22a\character-set-server=utf8' /etc/my.cnf
	systemctl restart mysql
fi

#setup hadoop260
if [ "$hadoop" = true ];then
	echo 'setup hadoop cdh 5.14.2 - hadoop2.6'
	tar -zxf /opt/hadoop-2.6.0-cdh5.14.2.tar.gz -C /opt/soft
	mv /opt/soft/hadoop-2.6.0-cdh5.14.2 /opt/soft/hadoop260
	
	echo "-----------hadoop-env.sh-------------"
	echo "export JAVA_HOME=/opt/soft/jdk180" >> /opt/soft/hadoop260/etc/hadoop/hadoop-env.sh
	
	echo "-----------core-site.xml-------------"
	val1="/opt/soft/hadoop260/etc/hadoop/core-site.xml"
	sed -i '19a\<property><name>hadoop.proxyuser.root.groups</name><value>*</value></property>' $val1
	sed -i '19a\<property><name>hadoop.proxyuser.root.users</name><value>*</value></property>' $val1
	sed -i '19a\<property><name>hadoop.tmp.dir</name><value>/opt/soft/hadoop260/tmp</value></property>' $val1
	sed -i "19a\<property><name>fs.defaultFS</name><value>hdfs://hadoop02:9000</value></property>" $val1
	
	echo "-----------hdfs-site.xml-------------"
	val2="/opt/soft/hadoop260/etc/hadoop/hdfs-site.xml"
	sed -i '19a\<property><name>dfs.permissions</name><value>false</value></property>' $val2
	sed -i '19a\<property><name>dfs.replication</name><value>1</value></property>' $val2
	
	echo "-----------mapred-site.xml-------------"
	cp /opt/soft/hadoop260/etc/hadoop/mapred-site.xml.template /opt/soft/hadoop260/etc/hadoop/mapred-site.xml
	val3="/opt/soft/hadoop260/etc/hadoop/mapred-site.xml"
	sed -i '19a\<property><name>mapreduce.framework.name</name><value>yarn</value></property>' $val3
	
	echo "-----------yarn-site.xml-------------"
	val4="/opt/soft/hadoop260/etc/hadoop/yarn-site.xml"
	sed -i '15a\<property><name>yarn.nodemanager.aux-services</name><value>mapreduce_shuffle</value></property>' $val4
	sed -i '15a\<property><name>yarn.resourcemanager.localhost</name><value>localhost</value></property>' $val4
	
	sed -i 's/localhost/hadoop02/g' /opt/soft/hadoop260/etc/hadoop/slaves
	echo '#hadoop' >> /etc/profile
	echo 'export HADOOP_HOME=/opt/soft/hadoop260' >> /etc/profile
	echo 'export HADOOP_MAPRED_HOME=$HADOOP_HOME' >> /etc/profile
	echo 'export HADOOP_COMMON_HOME=$HADOOP_HOME' >> /etc/profile
	echo 'export HADOOP_HDFS_HOME=$HADOOP_HOME' >> /etc/profile
	echo 'export YARN_HOME=$HADOOP_HOME' >> /etc/profile
	echo 'export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native' >> /etc/profile
	echo 'export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin' >> /etc/profile
	echo 'export HADOOP_INSTALL=$HADOOP_HOME' >> /etc/profile
	
	source /etc/profile
	cd /opt/soft/hadoop260/bin
	hadoop namenode -format
	tar -xf /opt/hadoop-native-64-2.6.0.tar -C /opt/soft/hadoop260/lib/native/
fi



#setup hive
if [ "$hive" = true ];then
	echo 'setup hive110'
	tar -zxf /opt/hive-1.1.0-cdh5.14.2.tar.gz -C  /opt/soft/
	mv /opt/soft/hive-1.1.0-cdh5.14.2/ /opt/soft/hive110
	cp /opt/mysql-connector-java-5.1.38.jar /opt/soft/hive110/lib/
	cd /opt/soft/hive110/conf
	echo "配置hive-site.xml檔案"
	touch hive-site.xml
	echo "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>
	<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>
	<configuration>
		<property>
			<name>hive.metastore.warehouse.dir</name>
			<value>/hive110/warehouse</value>
		</property>
		<property>
			<name>hive.metastore.local</name>
			<value>true</value>
		</property>
		<property>
			<name>javax.jdo.option.ConnectionURL</name>
			<value>jdbc:mysql://hadoop02:3306/hive110?createDatabaseIfNotExist=true</value>
		</property>
		<property>
			<name>javax.jdo.option.ConnectionDriverName</name>
			<value>com.mysql.jdbc.Driver</value>
		</property>
		<property>
			<name>javax.jdo.option.ConnectionUserName</name>
			<value>root</value>
		</property>
		<property>
			<name>javax.jdo.option.ConnectionPassword</name>
			<value>root</value>
		</property>
		<property>
			<name>hive.server2.authentication</name>
			<value>NONE</value>
		</property>
		<property>
			<name>hive.server2.thrift.client.user</name>
			<value>root</value>
		</property>
		<property>
			<name>hive.server2.thrift.client.password</name>
			<value>root</value>
		</property>
	</configuration>" >  /opt/soft/hive110/conf/hive-site.xml

	echo "#hive" >> /etc/profile
	echo 'export HIVE_HOME=/opt/soft/hive110' >> /etc/profile
	echo 'export PATH=$PATH:$HIVE_HOME/bin' >> /etc/profile
	source /etc/profile
	schematool -dbType mysql -initSchema
	cd /opt
fi

#setup zookeeper
if [ "$zookeeper" = true ];then
	echo 'setup zookeeper'
	tar -zxf /opt/zookeeper-3.4.5-cdh5.14.2.tar.gz -C /opt/soft/
	mv /opt/soft/zookeeper-3.4.5-cdh5.14.2 /opt/soft/zk345
	cp /opt/soft/zk345/conf/zoo_sample.cfg /opt/soft/zk345/conf/zoo.cfg
	mkdir -p /opt/soft/zk345/datas
	sed -i '12c dataDir=/opt/soft/zk345/datas' /opt/soft/zk345/conf/zoo.cfg 
	echo "server.0=$1:2287:3387" >> /opt/soft/zk345/conf/zoo.cfg 
	echo '#zookeeper' >> /etc/profile
	echo 'export ZOOKEEPER_HOME=/opt/soft/zk345' >> /etc/profile
	echo 'export PATH=$PATH:$ZOOKEEPER_HOME/bin' >> /etc/profile
fi

#setup hbase

if [ "$hbase" = true ];then
	echo 'setup hbase'
	tar -zxf /opt/hbase-1.2.0-cdh5.14.2.tar.gz -C /opt/soft/
	mv /opt/soft/hbase-1.2.0-cdh5.14.2/ /opt/soft/hbase120
	echo "export JAVA_HOME=/opt/soft/jdk180" >> /opt/soft/hbase120/conf/hbase-env.sh
	echo "export HBASE_MANAGES_ZK=false" >> /opt/soft/hbase120/conf/hbase-env.sh

	val5="/opt/soft/hbase120/conf/hbase-site.xml"
	sed -i "23a\<property><name>hbase.zookeeper.property.clientPort</name><value>2181</value></property>" $val5
	sed -i "23a\<property><name>hbase.zookeeper.property.dataDir</name><value>/opt/soft/zk345/datas</value></property>" $val5
	sed -i "23a\<property><name>hbase.cluster.distributed</name><value>true</value></property>" $val5
	sed -i "23a\<property><name>hbase.rootdir</name><value>hdfs://hadoop02:9000/hbase</value></property>" $val5
	echo "#hbase" >> /etc/profile
	echo 'export HBASE_HOME=/opt/soft/hbase120' >> /etc/profile
	echo 'export PATH=$HBASE_HOME/bin:$HBASE_HOME/sbin:$PATH' >> /etc/profile
	source /etc/profile
fi

#setup sqoop

if [ "$sqoop" = true ];then
	echo 'setup sqoop'
	tar -zxf /opt/sqoop-1.4.6-cdh5.14.2.tar.gz -C /opt/soft/
	mv /opt/soft/sqoop-1.4.6-cdh5.14.2/ /opt/soft/sqoop146
	cp /opt/soft/sqoop146/conf/sqoop-env-template.sh /opt/soft/sqoop146/conf/sqoop-env.sh
	echo 'export HADOOP_COMMON_HOME=/opt/soft/hadoop260' >> /opt/soft/sqoop146/conf/sqoop-env.sh
	echo 'export HADOOP_MAPRED_HOME=/opt/soft/hadoop260' >> /opt/soft/sqoop146/conf/sqoop-env.sh
	echo 'export HIVE_HOME=/opt/soft/hive110' >> /opt/soft/sqoop146/conf/sqoop-env.sh
	echo 'export HBASE_HOME=/opt/soft/hbase120' >> /opt/soft/sqoop146/conf/sqoop-env.sh
	echo 'export ZOOCFGDIR=/opt/soft/zk345/conf' >> /opt/soft/sqoop146/conf/sqoop-env.sh
	echo 'export ZOOKEEPER_HOME=/opt/soft/zk345' >> /opt/soft/sqoop146/conf/sqoop-env.sh
	cp /opt/mysql-connector-java-5.1.38.jar /opt/soft/sqoop146/lib/
	cp /opt/java-json.jar /opt/soft/sqoop146/lib/
	cp /opt/soft/hive110/lib/hive-common-1.1.0-cdh5.14.2.jar /opt/soft/sqoop146/lib/
	cp /opt/soft/hive110/lib/hive-jdbc-1.1.0-cdh5.14.2-standalone.jar /opt/soft/sqoop146/lib/
	echo '#sqoop' >> /etc/profile
	echo 'export SQOOP_HOME=/opt/soft/sqoop146' >> /etc/profile
	echo 'export PATH=$PATH:$SQOOP_HOME/bin' >> /etc/profile
fi

#setup spark 
if [ "$spark" = true ];then
	echo 'setup spark'
	tar -zxf /opt/spark-2.3.4-bin-hadoop2.6.tgz -C /opt/soft/
	mv /opt/soft/spark-2.3.4-bin-hadoop2.6 /opt/soft/spark234
	cp /opt/soft/spark234/conf/slaves.template /opt/soft/spark234/conf/slaves
	cp /opt/soft/spark234/conf/spark-env.sh.template /opt/soft/spark234/conf/spark-env.sh
	echo "export SPARK_MASTER_HOST=192.168.98.135" >>  /opt/soft/spark234/conf/spark-env.sh
	echo "export SPARK_MASTER_PORT=7077" >>  /opt/soft/spark234/conf/spark-env.sh
	echo "export SPARK_WORKER_CORES=2" >>  /opt/soft/spark234/conf/spark-env.sh
	echo "export SPARK_WORKER_MEMORY=3g" >>  /opt/soft/spark234/conf/spark-env.sh
	echo "export SPARK_MASTER_WEBUI_PORT=8888" >>  /opt/soft/spark234/conf/spark-env.sh
	echo "export JAVA_HOME=/opt/soft/jdk180" >> /opt/soft/spark234/sbin/spark-config.sh
fi

安裝完後

Hadoop元件單機版——shell腳本一鍵安裝

Hadoop元件單機版——shell腳本一鍵安裝

繼續閱讀

HK-2000資料采集儀資料庫操作說明

終端環境之tmux

查找檔案中的字元串

尚矽谷—韓順平—圖解 Java設計模式（結構型）（55～）

Storm編譯打包過程中遇到的一些問題及解決方法

MapReduce的幾個企業級經典面試案例MapReduce的幾個企業級經典面試案例

9.spark Core 進階2--Cashe

拒絕使用者登入:/bin/false和/usr/sbin/nologin

淺談企業活動中進行資料分析的重要性

Shell程式設計——sort排序、uniq忽略重複、tr替換壓縮删除、cut指定删除字段、正規表達式元字元sort 指令uniq 指令tr 指令cut 指令正規表達式

Linxu常用指令技巧彙總

《Linux指令行與Shell腳本程式設計大全第2版.布盧姆》pdf

ACS基本配置-權限等級管理

Ambari介紹和架構原理

NOSQL安全攻擊

win10本地scala和spark安裝安裝scala安裝spark