hadoop 初次 - 启动 , 配置 , 测试

# HDFS后台程序设置 : 名称节点，第二名称节点 和 数据节点 

conf/hdfs-site.xml

<?xml version="1.0"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <!-- Put site-specific property overrides in this file. --> <configuration> <property>    <name>dfs.replication</name>    <value>2</value> </property> <!-- 主 namenode -->     <property>        <name>dfs.name.dir</name>        <value>/home/hadoop/hadoop/namenode</value>     </property> <!-- 从 data -->     <property>        <name>dfs.data.dir</name>        <value>/home/hadoop/hadoop/data</value>     </property> </configuration>

# hadoop 核心配置 ,比如 mapreduce 中普遍的 I/O 配置

core-site.xml 

<?xml version="1.0"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <!-- Put site-specific property overrides in this file. --> <configuration> <property>    <name>fs.default.name</name>    <value>hdfs://10.10.9.88:9000</value> </property> </configuration>

# mapreduce 后台配置 ： jobtracker 和 tasktracker 

# 这特别注意 使用 localhost 别使用 IP 地址 ，具体 为什么不知道 ，但会报错! 

mapred-site.xml

<?xml version="1.0"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <!-- Put site-specific property overrides in this file. --> <configuration> <property>    <name>mapred.job.tracker</name>    <value>localhost:9001</value> </property> <!-- （在主服务器上） mkdir -p /home/hadoop/hadoop/tmp/mapred/system/ --> <property>     <name>mapred.system.dir</name>     <value>/home/hadoop/hadoop/tmp/mapred/system</value>     <final>true</final> </property> </configuration>

# ssh 免登入 注意配置

masters , slaves 纯文本 文件 

在主机 的 nfs 上 

./bin/start-all.sh 

运行测试

./bin/hadoop jar ./hadoop-0.20.2-test.jar TestDFSIO -write -nrFiles 5 -fileSize 25

TestFDSIO.0.0.4 10/08/10 23:40:20 INFO mapred.FileInputFormat: nrFiles = 5 10/08/10 23:40:20 INFO mapred.FileInputFormat: fileSize (MB) = 25 10/08/10 23:40:20 INFO mapred.FileInputFormat: bufferSize = 1000000 10/08/10 23:40:20 WARN conf.Configuration: mapred-default.xml:a attempt to override final parameter: mapred.system.dir; Ignoring. 10/08/10 23:40:21 INFO mapred.FileInputFormat: creating control file: 25 mega bytes, 5 files 10/08/10 23:40:21 WARN conf.Configuration: mapred-default.xml:a attempt to override final parameter: mapred.system.dir; Ignoring. 10/08/10 23:40:21 INFO mapred.FileInputFormat: created control files for: 5 files 10/08/10 23:40:22 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same. 10/08/10 23:40:22 INFO mapred.FileInputFormat: Total input paths to process : 5 10/08/10 23:40:23 INFO mapred.JobClient: Running job: job_201008102337_0002 10/08/10 23:40:24 INFO mapred.JobClient: map 0% reduce 0% 10/08/10 23:40:41 INFO mapred.JobClient: map 40% reduce 0% 10/08/10 23:41:03 INFO mapred.JobClient: map 80% reduce 0% 10/08/10 23:41:06 INFO mapred.JobClient: map 80% reduce 13% 10/08/10 23:41:12 INFO mapred.JobClient: map 80% reduce 26% 10/08/10 23:41:15 INFO mapred.JobClient: map 100% reduce 26% 10/08/10 23:41:24 INFO mapred.JobClient: map 100% reduce 100% 10/08/10 23:41:26 INFO mapred.JobClient: Job complete: job_201008102337_0002 10/08/10 23:41:26 INFO mapred.JobClient: Counters: 18 10/08/10 23:41:26 INFO mapred.JobClient: Job Counters 10/08/10 23:41:26 INFO mapred.JobClient: Launched reduce tasks=1 10/08/10 23:41:26 INFO mapred.JobClient: Launched map tasks=5 10/08/10 23:41:26 INFO mapred.JobClient: Data-local map tasks=5 10/08/10 23:41:26 INFO mapred.JobClient: FileSystemCounters 10/08/10 23:41:26 INFO mapred.JobClient: FILE_BYTES_READ=471 10/08/10 23:41:26 INFO mapred.JobClient: HDFS_BYTES_READ=565 10/08/10 23:41:26 INFO mapred.JobClient: FILE_BYTES_WRITTEN=1126 10/08/10 23:41:26 INFO mapred.JobClient: HDFS_BYTES_WRITTEN=131072076 10/08/10 23:41:26 INFO mapred.JobClient: Map-Reduce Framework 10/08/10 23:41:26 INFO mapred.JobClient: Reduce input groups=5 10/08/10 23:41:26 INFO mapred.JobClient: Combine output records=0 10/08/10 23:41:26 INFO mapred.JobClient: Map input records=5 10/08/10 23:41:26 INFO mapred.JobClient: Reduce shuffle bytes=495 10/08/10 23:41:26 INFO mapred.JobClient: Reduce output records=5 10/08/10 23:41:26 INFO mapred.JobClient: Spilled Records=50 10/08/10 23:41:26 INFO mapred.JobClient: Map output bytes=415 10/08/10 23:41:26 INFO mapred.JobClient: Map input bytes=135 10/08/10 23:41:26 INFO mapred.JobClient: Combine input records=0 10/08/10 23:41:26 INFO mapred.JobClient: Map output records=25 10/08/10 23:41:26 INFO mapred.JobClient: Reduce input records=25 10/08/10 23:41:27 INFO mapred.FileInputFormat: ----- TestDFSIO ----- : write 10/08/10 23:41:27 INFO mapred.FileInputFormat: Date & time: Tue Aug 10 23:41:26 CST 2010 10/08/10 23:41:27 INFO mapred.FileInputFormat: Number of files: 5 10/08/10 23:41:27 INFO mapred.FileInputFormat: Total MBytes processed: 125 10/08/10 23:41:27 INFO mapred.FileInputFormat: Throughput mb/sec: 4.095540775203958 10/08/10 23:41:27 INFO mapred.FileInputFormat: Average IO rate mb/sec: 5.74212121963501 10/08/10 23:41:27 INFO mapred.FileInputFormat: IO rate std deviation: 2.875854060066087 10/08/10 23:41:27 INFO mapred.FileInputFormat: Test exec time sec: 64.975 10/08/10 23:41:27 INFO mapred.FileInputFormat: