hdfs 优化配置文件

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

<configuration>
 <!-- 把两个NameNode的地址组装成一个集群opencluster -->
    <property>
       <name>fs.defaultFS</name>
       <value>hdfs://opencluster</value>
    </property>
    <!-- 指定hadoop运行时产生文件的存储目录 -->
    <property>
      <name>hadoop.tmp.dir</name>
      <value>/bigdata/zzwl/data/ha/tmp</value>
    </property>
    <!-- 指定ZKFC故障自动切换转移 -->
    <property>
         <name>ha.zookeeper.quorum</name>
         <value>bigdata01:2181,bigdata02:2181,bigdata03:2181</value>
    </property>
    <!--修改core-site.xml中的ipc参数,防止出现连接journalnode服务ConnectException-->
    <property>
        <name>ipc.client.connect.max.retries</name>
        <value>100</value>
        <description>Indicates the number of retries a client will make to establish a server connection.</description>
    </property>

</configuration>

1

export JAVA_HOME=/usr/local/java

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

<configuration>
    <!-- 设置dfs副本数，默认3个 -->
    <property>
    <name>dfs.replication</name>
    <value>3</value>
    </property>
    <!-- 完全分布式集群名称 -->
    <property>
      <name>dfs.nameservices</name>
      <value>opencluster</value>
    </property>
    <!-- 集群中NameNode节点都有哪些 -->
    <property>
       <name>dfs.ha.namenodes.opencluster</name>
       <value>nn1,nn2</value>
    </property>
    <!-- nn1的RPC通信地址 -->
    <property>
       <name>dfs.namenode.rpc-address.opencluster.nn1</name>
       <value>bigdata01:8020</value>
    </property>
    <!-- nn2的RPC通信地址 -->
    <property>
       <name>dfs.namenode.rpc-address.opencluster.nn2</name>
       <value>bigdata02:8020</value>
    </property>
    <!-- nn1的http通信地址 -->
    <property>
       <name>dfs.namenode.http-address.opencluster.nn1</name>
       <value>bigdata01:50070</value>
    </property>
    <!-- nn2的http通信地址 -->
    <property>
        <name>dfs.namenode.http-address.opencluster.nn2</name>
        <value>bigdata02:50070</value>
    </property>
    <!-- 指定NameNode元数据在JournalNode上的存放位置 -->
    <property>
        <name>dfs.namenode.shared.edits.dir</name>
        <value>qjournal://bigdata01:8485;bigdata02:8485;bigdata03:8485/opencluster</value>
    </property>
    <!-- 配置隔离机制，即同一时刻只能有一台服务器对外响应 -->
    <property>
        <name>dfs.ha.fencing.methods</name>
        <value>shell(/bin/true)</value>
    </property>

 <!-- 使用隔离机制时需要ssh无秘钥登录-->
    <property>
        <name>dfs.ha.fencing.ssh.private-key-files</name>
        <value>/root/.ssh/id_rsa</value>
    </property>
    <!-- 声明journalnode服务器存储目录-->
    <property>
       <name>dfs.journalnode.edits.dir</name>
       <value>/bigdata/zzwl/data/ha/jn</value>
    </property>
    <!-- 关闭权限检查-->
    <property>
       <name>dfs.permissions.enable</name>
       <value>false</value>
    </property>
    <!-- 访问代理类：client，opencluster，active配置失败自动切换实现方式-->
    <property>
       <name>dfs.client.failover.proxy.provider.opencluster</name>
       <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
    </property>
    <!-- 配置自动故障转移-->
    <property>
       <name>dfs.ha.automatic-failover.enabled</name>
       <value>true</value>
    </property>

<property>
        <name>dfs.namenode.name.dir</name>
        <value>file:/bigdata/zzwl/data/storage/hdfs/name</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:/bigdata/zzwl/data/storage/hdfs/data</value>
    </property>
    <property>
        <name>dfs.blocksize</name>
        <value>512000000</value> <!--大文件系统HDFS块大小为512M，默认为64MB-->
    </property>
    <property>
        <name>dfs.namenode.handler.count</name>
        <value>100</value>
    </property>

1
2
3

bigdata01
bigdata02
bigdata03

1
2

JAVA=$JAVA_HOME/bin/java
JAVA_HEAP_MAX=-Xmx1000m