華為雲單機版hadoop2.7.3及外網通路hdfs

關閉防火牆

systemctl stop firewalld.service
systemctl disable firewalld.service
           

配置ip映射

内網IP：192.168.x.xxx

公網IP：115.61.xx.xx

vim /etc/hosts
::1     localhost       localhost.localdomain   localhost6      localhost6.localdomain6
127.0.0.1       localhost       localhost.localdomain   localhost4      localhost4.localdomain4
127.0.0.1       localhost       localhost
127.0.0.1       ecs-1211        ecs-1211
192.168.x.xxx   lkk2
           

使用 hostname 指令修改主機名

hostname lkk2
           

配置ssh免密登入

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys
           

檢視是否可以免密登入

ssh lkk2
           

删除系統自帶OpenJDK

[[email protected]~]# rpm -qa | grep jdk 
java-1.8.0-openjdk-headless-1.8.0.232.b09-0.el7_7.aarch64
java-1.8.0-openjdk-devel-1.8.0.232.b09-0.el7_7.aarch64
java-1.8.0-openjdk-1.8.0.232.b09-0.el7_7.aarch64
copy-jdk-configs-3.3-10.el7_5.noarch
           

[[email protected]~]#  rpm -e --nodeps java-1.8.0-openjdk-headless-1.8.0.232.b09-0.el7_7.aarch64
[[email protected]~]#  rpm -e --nodeps java-1.8.0-openjdk-devel-1.8.0.232.b09-0.el7_7.aarch64
[[email protected]~]#  rpm -e --nodeps java-1.8.0-openjdk-1.8.0.232.b09-0.el7_7.aarch64
           

安裝jdk

mkdir -p /usr/local/java

cd /usr/local/java

上傳 jdk-8u281-linux-aarch64.tar.gz 到/usr/local/java 目錄

tar -zxvf jdk-8u281-linux-aarch64.tar.gz 
           

配置JDK環境變量

vim /etc/profile
JAVA_HOME=/usr/local/java/jdk1.8.0_281
JRE_HOME=/usr/java/jdk1.8.0_281/jre
CLASS_PATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib
PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
export JAVA_HOME JRE_HOME CLASS_PATH PATH
           

使配置檔案生效

source /etc/profile 
           

[[email protected] hadoop]# java -version 
java version "1.8.0_281"
Java(TM) SE Runtime Environment (build 1.8.0_281-b09)
Java HotSpot(TM) 64-Bit Server VM (build 25.281-b09, mixed mode)
           

解壓hadoop

mkdir -p /data/hadoop

上傳 hadoop-2.7.3.tar.gz 到 /data/hadoop

tar -zxvf  hadoop-2.7.3.tar.gz 
           

配置hadoop環境變量

vim /etc/profile
JAVA_HOME=/usr/local/java/jdk1.8.0_281
JRE_HOME=/usr/java/jdk1.8.0_281/jre
HADOOP_HOME=/data/hadoop/hadoop-2.7.3
CLASS_PATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib
PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
export JAVA_HOME JRE_HOME HADOOP_HOME CLASS_PATH PATH
           

使配置檔案生效

source /etc/profile 
           

hadoop配置檔案有

hadoop-env.sh

yarn-env.sh

core-site.xml

hdfs-site.xml

mapred-site.xml

yarn-site.xml

vim hadoop-env.sh

export JAVA_HOME=/usr/local/java/jdk1.8.0_281
           

vim yarn-env.sh

# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
 export JAVA_HOME=/usr/local/java/jdk1.8.0_281
 JAVA_HEAP_MAX=Xmx3072m
           

vim core-site.xml

<property>
		<name>fs.default.name</name>
		<value>hdfs://lkk2:9000</value>
  </property>
  
  <property>
   	<name>hadoop.tmp.dir</name>
   	<value>file:/data/hadoop/data/tmp</value>
  </property>
           

vim hdfs-site.xml

<property>
		<name>dfs.datanode.data.dir</name>
		<value>/data/hadoop/data/hdf/data</value>
		<final>true</final>
	  </property>
	  <property>
		<name>dfs.namenode.name.dir</name>
		<value>/data/hadoop/data/hdf/name</value>
		<final>true</final>
	  </property>
	  <property>
			<name>dfs.replication</name>
			<value>1</value>
	  </property>
	  <property>
		<name>dfs.permissions.enabled</name>
		<value>false</value>
	  </property>
	<property>
		<name>dfs.client.use.datanode.hostname</name>
		<value>true</value>
		<description>only cofig in clients</description>
	</property>
           

vim mapred-site.xml

<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>
  <property>
    <name>mapreduce.jobhistory.address</name>
    <value>lkk2:10020</value>
  </property>
  <property>
    <name>mapreduce.jobhistory.webapp.address</name>
    <value>lkk2:19888</value>
  </property>
           

vim yarn-site.xml

<property>
	<name>yarn.resourcemanager.hostname</name>
	<value>lkk2</value>
</property>
<property>
	<name>yarn.nodemanager.aux-services</name>
	<value>mapreduce_shuffle</value>
</property>
           

vim slaves

lkk2
           

格式化namenode

cd /data/hadoop/hadoop-2.7.3/sbin/

hdfs namenode -format
           

啟動

start-all.sh   
           

http://115.61.xx.xx:50070