HadoopHA【一】 hadoop配置文件详解

HadoopHA【一】 hadoop配置文件详解

• 配置 core-site.xml 文件

点击(此处)折叠或打开

1. <configuration>
   
2. 
   
3.     <property>
   
4.         <name>fs.defaultFS</name>
   
5.         <value>hdfs://hacluster</value>
   
6.         <description> 集群的namenode节点的url </description>
   
7.     </property>
   
8. 
   
9.     <property>
   
10.         <name>ha.zookeeper.quorum</name>
    
11.         <value>1.hadoop.com:2181,2.hadoop.com:2181,3.hadoop.com:2181</value>
    
12.         <description>zookeeper集群的地址和端口,最好保持基数个至少3台 </description>
    
13.         
    
14.     </property>
    
15. 
    
16.     <property>
    
17.         <name>io.file.buffer.size</name>
    
18.         <value>131072</value>
    
19.         <description></description>
    
20.     </property>
    
21. 
    
22.     <property>
    
23.         <name>hadoop.tmp.dir</name>
    
24.         <value>/usr/local/fqlhadoop/datas/hadoop/tmp-hadoop-${user.name}</value>
    
25.         <description>执行namenode format之后，很多路径都依赖他，namenode节点该目录不可以删除</description>
    
26.     </property>
27. 
    
28. </configuration>
29. 
    

配置  hdfs-site.xml

点击(此处)折叠或打开

1. <configuration>
   
2. 
   
3.     <property>
   
4.         <name>dfs.nameservices</name>
   
5.         <value>hacluster</value>
   
6.     </property>
   
7. 
   
8.     <property>
   
9.         <name>dfs.ha.namenodes.hacluster</name>
   
10.         <value>n1,n2</value>
    
11.     </property>
    
12. 
    
13.     <property>
    
14.         <name>dfs.namenode.rpc-address.hacluster.n1</name>
    
15.         <value>1.hadoop.com:8020</value>
    
16.     </property>
    
17. 
    
18.     <property>
    
19.         <name>dfs.namenode.rpc-address.hacluster.n2</name>
    
20.         <value>2.hadoop.com:8020</value>
    
21.         <description> 远程控制端口 在配置客户端访问时 会用到此端口 </description>
    
22.     </property>
    
23. 
    
24.     <property>
    
25.         <name>dfs.namenode.http-address.hacluster.n1</name>
    
26.         <value>1.hadoop.com:8090</value>
    
27.     </property>
    
28. 
    
29.     <property>
    
30.         <name>dfs.namenode.http-address.hacluster.n2</name>
    
31.         <value>2.hadoop.com:8090</value>
    
32.         <description> 页面访问端口 </description>
    
33.     </property>
    
34. 
    
35.     <property>
    
36.         <name>dfs.namenode.shared.edits.dir</name>
    
37.         <value>qjournal://1.hadoop.com:8485;2.hadoop.com:8485;3.hadoop.com:8485/hacluster</value>
    
38.         <description> journalnode共享文件集群 </description>
    
39.     </property>
    
40. 
    
41.     <property>
    
42.         <name>dfs.client.failover.proxy.provider.hacluster</name>
    
43.         <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
    
44.         <description> 故障处理类 </description>
    
45.     </property>
    
46.                                                                                                                                                                                                                        
    
47.     <property>
    
48.         <name>dfs.ha.fencing.methods</name>
    
49.         <value>sshfence(${user.name}:39000)</value>
    
50.         <description> ssh方式进行故障切换 </description>
    
51.     </property>
    
52.                                                                                                                                                                                                                        
    
53.     <property>
    
54.         <name>dfs.ha.fencing.ssh.private-key-files</name>
    
55.         <value>/home/${user.name}/.ssh/id_rsa</value>
    
56.     </property>
    
57.                                                                                                                                                                                                                        
    
58.     <property>
    
59.         <name>dfs.journalnode.edits.dir</name>
    
60.         <value>/home/hacluster/hadoop/dfs.journalnode.edits.dir</value>
    
61.     </property>
    
62. 
    
63. <!-- auto configure failover -->
    
64. 
    
65.     <property>
    
66.         <name>dfs.ha.automatic-failover.enabled</name>
    
67.         <value>true</value>
    
68.         <description> 自动切换 开启 </description>
    
69.     </property>
    
70.                                                                                                                                                                                                                        
    
71.     <property>
    
72.         <name>dfs.replication</name>
    
73.         <value>3</value>
    
74.         <description> 备机数量 </description>
    
75.     </property>
    
76.                                                                                                                                                                                                                        
    
77.     <property>
    
78.         <name>dfs.namenode.name.dir</name>
    
79.         <value>/home/hacluster/hadoop/dfs.namenode.name.dir</value>
    
80.     </property>
    
81.                                                                                                                                                                                                                        
    
82.     <property>
    
83.         <name>dfs.blocksize</name>
    
84.         <value>67108864</value>
    
85.     </property>
    
86.                                                                                                                                                                                                                        
    
87.     <property>
    
88.         <name>dfs.namenode.handler.count</name>
    
89.         <value>100</value>
    
90.     </property>
    
91.                                                                                                                                                                                                                        
    
92.     <property>
    
93.         <name>dfs.datanode.data.dir</name>
    
94.         <value>/data1/hacluster/hadoop/datanode</value>
95.         <description> datanode数据目录 </description>
    
96.     </property>
    
97.                                                                                                                                                                                                                        
    
98. </configuration>

• 配置 yarn-site.xml

点击(此处)折叠或打开

1. <configuration>
   
2. 
   
3. <!-- Site specific YARN configuration properties -->
   
4. 
   
5.    <property>
   
6.        <name>yarn.resourcemanager.scheduler.class</name>
   
7.        <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
   
8.     </property>
   
9. 
   
10.     <property>
    
11.         <name>yarn.resourcemanager.ha.enabled</name>
    
12.         <value>true</value>
    
13.         <description>开启resource manager HA</description>
    
14.     </property>
    
15. 
    
16.     <property>
    
17.         <name>yarn.resourcemanager.cluster-id</name>
    
18.         <value>cluster1</value>
    
19.     </property>
    
20.     <property>
    
21.         <name>yarn.resourcemanager.ha.rm-ids</name>
    
22.         <value>rm1,rm2</value>
    
23.     </property>
    
24.     <property>
    
25.         <name>yarn.resourcemanager.hostname.rm1</name>
    
26.         <value>1.hadoop.com</value>
    
27.     </property>
    
28.     <property>
    
29.         <name>yarn.resourcemanager.hostname.rm2</name>
    
30.         <value>2.hadoop.com</value>
    
31.     </property>
    
32. 
    
33.     <property>
    
34.         <name>yarn.resourcemanager.zk-address</name>
    
35.         <value>1.hadoop.com:2181,2.hadoop.com:2181,3.hadoop.com:2181</value>
    
36.     </property>
    
37. 
    
38. 
    
39. <!--
    
40.     enable RM Restart feature.
    
41. -->
    
42. 
    
43.     <property>
    
44.         <name>yarn.resourcemanager.recovery.enabled</name>
    
45.         <value>true</value>
    
46.         <description>开启自动恢复功能</description>
    
47.     </property>
48. 
    
49.     <property>
50.         <name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
51.         <value>true</value>
52.         <description>开启故障自动切换</description>
    
53.     </property>
55.     <property>
56.         <name>yarn.resourcemanager.store.class</name>
    
57.         <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
    
58.     </property>
    
59. 
    
60.     <property>
    
61.         <name>yarn.resourcemanager.am.max-attempts</name>
    
62.         <value>5</value>
63.     </property>                                                                                                                                                                                                             
64.                                                                                                                                                                                                                        
65.     <property>                                                                                                                                                                                                         
66.         <name>yarn.nodemanager.aux-services</name>                                                                                                                                                                     
67.         <value>mapreduce_shuffle</value>                                                                                                                                                                               
68.     </property>                                                                                                                                                                                                        
69.                                                                                                                                                                                                                       
70.                                                                                                                                                                                                                        
71. </configuration>