[hadoop] 搭建自己的hadoop集群

1>  准备工作
       a>  五台centos6.2虚拟机,配置主机名、IP地址、yum源、
              192.168.68.201  master01
              192.168.68.202  master02
              192.168.68.203  slave01
              192.168.68.204  slvae02
              192.168.68.205  slave03   
       b>  准备所需要的软件包
              jdk-6u26-linux-x64-rpm.bin 
              hadoop-0.20.2.tar.gz
2> 配置我自己的hadoop 集群
      a>  修改5台机器的hosts文件

            ＃vim /etc/hosts           

        192.168.68.201  master01
        192.168.68.202  master02
        192.168.68.203  slave01
        192.168.68.204  slvae02
        192.168.68.205  slave03

      b>  配置master无密码登录slave,在master01和master02上执行以下命令:   

             #ssh-keygen
             #ssh-copy-id -i .ssh/id_rsa.pub root@master01
             #ssh-copy-id -i .ssh/id_rsa.pub root@master02
             #ssh-copy-id -i .ssh/id_rsa.pub root@slave01
             #ssh-copy-id -i .ssh/id_rsa.pub root@slave02
             #ssh-copy-id -i .ssh/id_rsa.pub root@slave03

      c>  5台机器分别安装java虚拟机，设置java环境变量

            #./jdk-6u26-linux-x64-rpm.bin      
            #cat >>/etc/profile <<EOF
              export JAVA_HOME=/usr/java/jdk1.6.0_26
              export PATH=\$JAVA_HOME/bin:\$PATH
              EOF 
      d>  在master01上解压hadoop-0.20.2.tar.gz 配置hadoop集群
             #tar -zxvf hadoop-0.20.2.tar.gz     
            #vim hadoop-0.20.2/conf/hadoop-env.sh 

       export JAVA_HOME=/usr/java/jdk1.6.0_26
       export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_NAMENODE_OPTS"
       export HADOOP_SECONDARYNAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_SECONDARYNAMENODE_OPTS"
       export HADOOP_DATANODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_DATANODE_OPTS"
       export HADOOP_BALANCER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_BALANCER_OPTS"
       export HADOOP_JOBTRACKER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_JOBTRACKER_OPTS"

           #vim hadoop-0.20.2/conf/hdfs-site.xml

<configuration>
    <property>
        <name>dfs.http.address</name>
        <value>192.168.68.201:50070</value>
    </property>
    <property>
        <name>dfs.name.dir</name>
        <value>${hadoop.tmp.dir}/dfs/name,/data/name</value>
    </property>
    <property>
        <name>dfs.data.dir</name>
        <value>/data/data</value>
    </property>
    <property> 
        <name>dfs.replication</name> 
        <value>2</value> 
    </property>
</configuration>

         #vim hadoop-0.20.2/conf/mapred-site.xml 

<configuration>
    <property> 
        <name>mapred.job.tracker</name> 
        <value>master02:8021</value> 
    </property>
    <property> 
        <name>mapred.local.dir</name> 
        <value>/data/mapred/local</value> 
    </property> 
    <property> 
        <name>mapred.system.dir</name> 
        <value>/data/mapred/system</value> 
    </property>
    <property> 
        <name>mapred.job.tracker.http.address</name> 
        <value>192.168.68.202:50030</value> 
    </property>
</configuration>

           #vim hadoop-0.20.2/conf/core-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property> 
        <name>fs.default.name</name> 
        <value>hdfs://master01:8020</value> 
    </property>
    <property> 
        <name>hadoop.tmp.dir</name> 
        <value>/data/hadoop/tmp</value> 
    </property>
</configuration>

           #vim hadoop-0.20.2/conf/masters

     master02

          #vim hadoop-0.20.2/conf/slaves           

     slave01
     slave02
     slave03

      e>  将master01上的hadoop-0.20.2 复制到其他机器    

      #scp -r hadoop-0.20.2 root@master02:/root/
      #scp -r hadoop-0.20.2 root@slave01:/root/
      #scp -r hadoop-0.20.2 root@slave02:/root/
      #scp -r hadoop-0.20.2 root@slave03:/root/

     
3>  启动hadoop集群
       a>  在master01上格式化hdfs系统
              #./hadoop-0.20.2/bin/hadoop namenode -format  
       b>  在master01上启动hdfs系统  ,执行jps查看启动的进程 
              #./hadoop-0.20.2/bin/start-dfs.sh

              #jps

                1872 Jps

                1654 NameNode

      c>  在master02上启动MapReduce，执行jps查看启动的进程
             #./hadoop-0.20.2/bin/start-mapred.sh

             #jps   

               1956 Jps
               1737 SecondaryNameNode
               1895 JobTracker

      d>  查看slave01上启动的进程

             #jps

               2418 Jps
               1758 TaskTracker
               1827 DataNode