Hadoop 2.6.0叢集搭建

yum install gcc 

yum install gcc-c++

yum install make

yum install autoconfautomake libtool cmake

yum install ncurses-devel

yum install openssl-devel

groupadd hadoop  添加一個組

useradd hadoop -g hadoop  添加使用者

安裝protoc(需用root使用者)

1 tar -xvf protobuf-2.5.0.tar.bz2 

2 cd protobuf-2.5.0 

3 ./configure --prefix=/opt/protoc/ 

4 make && make install

編譯hadoop

mvn clean package -Pdist,native -DskipTests -Dtar 

編譯完的hadoop在 /home/hadoop/ocdc/hadoop-2.6.0-src/hadoop-dist/target 路徑下

配置hosts檔案

10.1.245.244 master

10.1.245.243 slave1

指令行輸入 hostname master

免密碼登入：

執行指令生成密鑰： ssh-keygen -t rsa -P ""

進入檔案夾cd  .ssh (進入檔案夾後可以執行ls  -a 檢視檔案)

将生成的公鑰id_rsa.pub 内容追加到authorized_keys（執行指令：cat id_rsa.pub >> authorized_keys）

---------------------------(core-site.xml)------------------------------

<configuration>

 <!--指定hdfs的nameservice為ns1-->

<property>

  <name>fs.defaultFS</name>

  <value>hdfs://master</value>

 </property>

 <property>

  <name>io.file.buffer.size</name>

  <value>131072</value>

<!--指定hadoop資料存放目錄-->

  <name>hadoop.tmp.dir</name>

  <value>/home/hadoop/ocdc/hadoop-2.6.0/tmp</value>

  <description>Abasefor other temporary directories.</description>

  <name>hadoop.proxyuser.spark.hosts</name>

  <value>*</value>

  <name>hadoop.proxyuser.spark.groups</name>

</configuration>

<!--指定zookeeper位址-->

   <name>ha.zookeeper.quorum</name>

   <value>h4:2181,h5:2181,h6:2181</value>

-------------------------------(hdfs-site.xml )-----------------------------------

  <name>dfs.namenode.secondary.http-address</name>

  <value>master:9001</value>

  <property>

   <name>dfs.namenode.name.dir</name>

   <value>/home/hadoop/ocdc/hadoop-2.6.0/name</value>

  <name>dfs.datanode.data.dir</name>

  <value>/home/hadoop/ocdc/hadoop-2.6.0/data</value>

  </property>

  <name>dfs.replication</name>

  <value>3</value>

  <name>dfs.webhdfs.enabled</name>

  <value>true</value>

   <name>dfs.nameservices</name>

   <value>ns1</value>

</property>

<!-- ns1下面有兩個NameNode，分别是nn1，nn2 -->

   <name>dfs.ha.namenodes.ns1</name>

   <value>nn1,nn2</value>

----------------------------------------(yarn-site.xml)--------------------------

<!-- Site specific YARN configuration properties -->

<!-- 指定nodemanager啟動時加載server的方式為shuffle server -->

   <name>yarn.nodemanager.aux-services</name>

   <value>mapreduce_shuffle</value>

   <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>

   <value>org.apache.hadoop.mapred.ShuffleHandler</value>

   <name>yarn.resourcemanager.address</name>

   <value>master:8032</value>

   <name>yarn.resourcemanager.scheduler.address</name>

   <value>master:8030</value>

   <name>yarn.resourcemanager.resource-tracker.address</name>

   <value>master:8035</value>

   <name>yarn.resourcemanager.admin.address</name>

   <value>master:8033</value>

   <name>yarn.resourcemanager.webapp.address</name>

   <value>master:8088</value>

<name>yarn.nodemanager.resource.memory-mb</name>

<value>16384</value>

<!-- 指定resourcemanager位址 -->

   <property>

       <name>yarn.resourcemanager.hostname</name>

            <value>h3</value>

   </property>

-----------------------------(mapred-site.xml)-------------------------------

<name>mapreduce.framework.name</name>

   <value>yarn</value>

  <name>mapreduce.jobhistory.address</name>

  <value>master:10020</value>

  <name>mapreduce.jobhistory.webapp.address</name>

  <value>master:19888</value>

    <name>yarn.scheduler.maximum-allocation-mb</name>

    <value>16384</value>