{"id":826,"date":"2019-01-23T14:54:46","date_gmt":"2019-01-23T05:54:46","guid":{"rendered":"https:\/\/dong1lkim.oboki.net\/?p=826"},"modified":"2019-09-01T22:23:32","modified_gmt":"2019-09-01T13:23:32","slug":"%ea%b0%80%ec%9a%a9%ec%84%b1%ec%9d%84-%ea%b3%a0%eb%a0%a4%ed%95%9c-hadoop-2-x-cluster-%ec%84%a4%ec%b9%98","status":"publish","type":"post","link":"https:\/\/oboki.net\/workspace\/data-engineering\/hadoop\/%ea%b0%80%ec%9a%a9%ec%84%b1%ec%9d%84-%ea%b3%a0%eb%a0%a4%ed%95%9c-hadoop-2-x-cluster-%ec%84%a4%ec%b9%98\/","title":{"rendered":"[Hadoop] \uac00\uc6a9\uc131\uc744 \uace0\ub824\ud55c Hadoop 2.x Cluster \uc124\uce58"},"content":{"rendered":"<h1>\uac00\uc6a9\uc131\uc744 \uace0\ub824\ud55c Hadoop Cluster \uc124\uce58<\/h1>\n<blockquote><p>\n  Namenode\uc640 Resourcemanager\ub97c active node,standby node\ub85c \uac01\uac01 \ub450 \ubc8c\uc529 \uad6c\uc131\ud568\uc73c\ub85c\uc368 \uace0\uac00\uc6a9\uc131\uc744 \uace0\ub824\ud55c Hadoop Cluster\ub97c \uc124\uce58\ud55c\ub2e4.\n<\/p><\/blockquote>\n<hr \/>\n<h2>\uc0ac\uc804 \uc900\ube44<\/h2>\n<h3>\uc758\uc874 \ud328\ud0a4\uc9c0 \uc124\uce58<\/h3>\n<p>namenode\uc640 resourcemanager\uac00 \uc7a5\uc560 \uc0c1\ud669\uc5d0\uc11c standby -&gt; active \uc0c1\ud0dc\ub85c \uc804\ud658\ub420 \ub54c\uc5d0 <code>fuser<\/code> \uba85\ub839\uc744 \uc774\uc6a9\ud574\uc11c \uc7a5\uc560\ub178\ub4dc\uc758 \ud504\ub85c\uc138\uc2a4\ub4e4\uc744 \uc815\ub9ac\ud558\ubbc0\ub85c <code>psmisc<\/code> \ud328\ud0a4\uc9c0\ub97c \uc124\uce58\ud55c\ub2e4.<\/p>\n<pre><code class=\"bash\">yum -y install psmisc\n<\/code><\/pre>\n<h3>JDK<\/h3>\n<p><a href=\"https:\/\/www.oracle.com\/technetwork\/java\/javase\/downloads\/jdk8-downloads-2133151.html\"><a href=\"https:\/\/www.oracle.com\/technetwork\/java\/javase\/downloads\/jdk8-downloads-2133151.html\"><a href=\"https:\/\/www.oracle.com\/technetwork\/java\/javase\/downloads\/jdk8-downloads-2133151.html\">https:\/\/www.oracle.com\/technetwork\/java\/javase\/downloads\/jdk8-downloads-2133151.html<\/a><\/a><\/a> \ud398\uc774\uc9c0\uc5d0\uc11c JDK 8 \ubc84\uc804 \ubc14\uc774\ub108\ub9ac \ub2e4\uc6b4\ub85c\ub4dc\ud558\uace0 <code>\/app\/jdk<\/code> \uacbd\ub85c\uc5d0 \uc555\ucd95\ud574\uc81c \ud558\uc5ec \uc0ac\uc6a9<\/p>\n<p><code>ls -l \/app\/jdk<\/code><\/p>\n<pre><code class=\"ls\">total 0\ndrwxr-xr-x 7 bigdata bigdata 245 Jul  7  2018 1.8.0_181\nlrwxrwxrwx 1 bigdata bigdata  15 Oct  8 18:03 bin -&gt; .\/1.8.0_181\/bin\nlrwxrwxrwx 1 bigdata bigdata  19 Oct  8 18:03 include -&gt; .\/1.8.0_181\/include\nlrwxrwxrwx 1 bigdata bigdata  26 Oct  8 18:03 javafx-src.zip -&gt; .\/1.8.0_181\/javafx-src.zip\nlrwxrwxrwx 1 bigdata bigdata  15 Oct  8 18:03 jre -&gt; .\/1.8.0_181\/jre\nlrwxrwxrwx 1 bigdata bigdata  15 Oct  8 18:03 lib -&gt; .\/1.8.0_181\/lib\nlrwxrwxrwx 1 bigdata bigdata  15 Oct  8 18:03 man -&gt; .\/1.8.0_181\/man\nlrwxrwxrwx 1 bigdata bigdata  19 Oct  8 18:03 src.zip -&gt; .\/1.8.0_181\/src.zip\n<\/code><\/pre>\n<h3>ZooKeeper \uc124\uce58<\/h3>\n<p>zookeeper \uc124\uce58 \uc790\ub8cc \ucc38\uace0<br \/>\n<a href=\"https:\/\/oboki.net\/workspace\/bigdata\/zookeeper\/zookeeper-3-x-%EC%84%A4%EC%B9%98\/\"><a href=\"https:\/\/oboki.net\/workspace\/bigdata\/zookeeper\/zookeeper-3-x-%EC%84%A4%EC%B9%98\/\"><a href=\"https:\/\/oboki.net\/workspace\/bigdata\/zookeeper\/zookeeper-3-x-%EC%84%A4%EC%B9%98\/\">https:\/\/oboki.net\/workspace\/bigdata\/zookeeper\/zookeeper-3-x-%EC%84%A4%EC%B9%98\/<\/a><\/a><\/a><\/p>\n<h3>\/etc\/hosts<\/h3>\n<pre><code class=\"txt\">127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4\n::1         localhost localhost.localdomain localhost6 localhost6.localdomain6\n\n192.168.179.80 node1.df node1.zk\n192.168.179.81 node2.df node2.zk\n192.168.179.82 node3.df node3.zk\n<\/code><\/pre>\n<h3>\uc0ac\uc6a9\uc790 \uc0dd\uc131<\/h3>\n<pre><code class=\"bash\">useradd -u 8020 hadoop -g bigdata\npasswd hadoop\n<\/code><\/pre>\n<h4>\uc0ac\uc6a9\uc790 \ud504\ub85c\ud30c\uc77c<\/h4>\n<p><code>vi ~\/.bash_profile<\/code><\/p>\n<pre><code class=\"bash\">## Java\nexport JAVA_HOME=\/app\/jdk\nexport PATH=$JAVA_HOME\/bin:$PATH\n\n## Hadoop\nexport HADOOP_HOME=\/app\/hadoop\nexport PATH=$HADOOP_HOME\/bin:$PATH:$HADOOP_HOME\/sbin:$PATH\n\n## User\nset -o vi\n<\/code><\/pre>\n<h3>Directory \uc0dd\uc131<\/h3>\n<pre><code class=\"bash\">mkdir -p \/data\/hadoop\/namenode \/data\/hadoop\/datanode \/data\/hadoop\/journalnode \/data\/hadoop\/yarn\nchown hadoop:bigdata \/data\/hadoop -R\nmkdir \/app\/hadoop\nchown hadoop:bigdata \/app\/hadoop\n<\/code><\/pre>\n<h3>SSH \uacf5\uac1c\ud0a4 \uc0dd\uc131 \ubc0f \uad50\ud658<\/h3>\n<p>\ub9c8\uc2a4\ud130 \ub178\ub4dc\ub294 \ubaa8\ub4e0 \uc2ac\ub808\uc774\ube0c\uc5d0 ssh \uc811\uc18d\uc774 \uac00\ub2a5\ud574\uc57c \ud55c\ub2e4. \ubaa8\ub4e0 \ub178\ub4dc\uc5d0\uc11c hadoop \uacc4\uc815\uc73c\ub85c \uc544\ub798 \uba85\ub839 \uc218\ud589\ud55c\ub2e4.<\/p>\n<pre><code class=\"bash\">ssh-keygen -t rsa -P \"\"\nssh node1.df cat ~\/.ssh\/id_rsa.pub &gt;&gt; ~\/.ssh\/authorized_keys\nssh node2.df cat ~\/.ssh\/id_rsa.pub &gt;&gt; ~\/.ssh\/authorized_keys\nssh node3.df cat ~\/.ssh\/id_rsa.pub &gt;&gt; ~\/.ssh\/authorized_keys\n<\/code><\/pre>\n<hr \/>\n<h2>Hadoop \uc124\uce58<\/h2>\n<p>\ubcf8\uaca9\uc801\uc73c\ub85c hadoop \uc124\uce58. \uc5ec\uae30\uc11c\ubd80\ud130\ub294 \uc804\ubd80 hadoop \uacc4\uc815\uc5d0\uc11c \uc9c4\ud589\ud55c\ub2e4.<\/p>\n<h3>\ubc14\uc774\ub108\ub9ac \uc555\ucd95 \ud574\uc81c \ubc0f \ub9c1\ud06c \uc124\uc815<\/h3>\n<pre><code class=\"bash\">wget http:\/\/mirror.navercorp.com\/apache\/hadoop\/common\/hadoop-2.9.2\/hadoop-2.9.2.tar.gz\ntar -xvzf hadoop-2.9.2.tar.gz -C \/app\/hadoop\/.\nmv \/app\/hadoop\/hadoop-2.9.2\/ \/app\/hadoop\/2.9.2\nfind \/app\/hadoop\/2.9.2 -mindepth 1 -maxdepth 1 -type d | xargs -I {} ln -s {} \/app\/hadoop\/.\n<\/code><\/pre>\n<h3>Hadoop Cluster \ud658\uacbd\uc124\uc815<\/h3>\n<p>\ub2e4\uc74c\uacfc \uac19\uc740 \ud30c\uc77c\uc744 \uc218\uc815\ud558\uc5ec Hadoop Cluster\ub97c \uc124\uc815\ud55c\ub2e4.<\/p>\n<ul>\n<li>slaves<\/li>\n<li>core-site.xml<\/li>\n<li>hdfs-site.xml<\/li>\n<li>yarn-site.xml<\/li>\n<li>hadoop-env.sh<\/li>\n<\/ul>\n<p><code>cd $HADOOP_HOME\/hadoop\/etc<\/code><\/p>\n<h4>slaves<\/h4>\n<p><code>vi slaves<\/code><\/p>\n<pre><code class=\"xml\">node1.df\nnode2.df\nnode3.df\n<\/code><\/pre>\n<h4>core-site.xml<\/h4>\n<p><code>vi core-site.xml<\/code><\/p>\n<pre><code class=\"xml\">&lt;?xml version=\"1.0\" encoding=\"UTF-8\"?&gt;\n&lt;?xml-stylesheet type=\"text\/xsl\" href=\"configuration.xsl\"?&gt;\n\n&lt;configuration&gt;\n    &lt;property&gt;\n        &lt;name&gt;fs.defaultFS&lt;\/name&gt;\n        &lt;value&gt;hdfs:\/\/hadoop-cluster&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;hadoop.tmp.dir&lt;\/name&gt;\n        &lt;value&gt;\/tmp\/hadoop&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;ha.zookeeper.quorum&lt;\/name&gt;\n        &lt;value&gt;node1.zk:2181,node2.zk:2181,node3.zk:2181&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;ipc.client.connect.max.retries&lt;\/name&gt;\n        &lt;value&gt;100&lt;\/value&gt;\n        &lt;description&gt;Indicates the number of retries a client will make to establish a server connection.&lt;\/description&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;ipc.client.connect.retry.interval&lt;\/name&gt;\n        &lt;value&gt;10000&lt;\/value&gt;\n        &lt;description&gt;Indicates the number of milliseconds a client will wait for before retrying to establish a server connection.&lt;\/description&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;hadoop.proxyuser.super.hosts&lt;\/name&gt;\n        &lt;value&gt;*&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;hadoop.proxyuser.super.groups&lt;\/name&gt;\n        &lt;value&gt;*&lt;\/value&gt;\n    &lt;\/property&gt;\n     &lt;property&gt;\n        &lt;name&gt;io.comporession.codecs&lt;\/name&gt;\n        &lt;value&gt;org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec&lt;\/value&gt; \n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;io.compression.codec.lzo.class&lt;\/name&gt;\n        &lt;value&gt;com.hadoop.compression.lzo.LzoCodec&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;fs.inmemory.size.mb&lt;\/name&gt;\n        &lt;value&gt;200&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;io.sort.factor&lt;\/name&gt;\n        &lt;value&gt;100&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;io.sort.mb&lt;\/name&gt;\n        &lt;value&gt;500&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;io.file.buffer.size&lt;\/name&gt;\n        &lt;value&gt;131072&lt;\/value&gt;\n    &lt;\/property&gt;\n&lt;\/configuration&gt;\n<\/code><\/pre>\n<h4>hdfs-site.xml<\/h4>\n<p><code>vi hdfs-site.xml<\/code><\/p>\n<pre><code class=\"xml\">&lt;?xml version=\"1.0\" encoding=\"UTF-8\"?&gt;\n&lt;?xml-stylesheet type=\"text\/xsl\" href=\"configuration.xsl\"?&gt;\n\n&lt;configuration&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.replication&lt;\/name&gt;\n        &lt;value&gt;3&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.namenode.name.dir&lt;\/name&gt;\n        &lt;value&gt;\/data\/hadoop\/namenode&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.datanode.name.dir&lt;\/name&gt;\n        &lt;value&gt;\/data\/hadoop\/datanode&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.journalnode.edits.dir&lt;\/name&gt;\n        &lt;value&gt;\/data\/hadoop\/journalnode&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.nameservices&lt;\/name&gt;\n        &lt;value&gt;hadoop-cluster&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.ha.namenodes.hadoop-cluster&lt;\/name&gt;\n        &lt;value&gt;nn1,nn2&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.namenode.rpc-address.hadoop-cluster.nn1&lt;\/name&gt;\n        &lt;value&gt;node1.df:8020&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.namenode.rpc-address.hadoop-cluster.nn2&lt;\/name&gt;\n        &lt;value&gt;node2.df:8020&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.namenode.http-address.hadoop-cluster.nn1&lt;\/name&gt;\n        &lt;value&gt;node1.df:50070&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.namenode.http-address.hadoop-cluster.nn2&lt;\/name&gt;\n        &lt;value&gt;node2.df:50070&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.namenode.shared.edits.dir&lt;\/name&gt;\n        &lt;value&gt;qjournal:\/\/node1.df:8485;node2.df:8485;node3.df:8485\/hadoop-cluster&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.client.failover.proxy.provider.hadoop-cluster&lt;\/name&gt;\n        &lt;value&gt;org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.ha.fencing.methods&lt;\/name&gt;\n        &lt;value&gt;sshfence(hadoop:22)&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.ha.fencing.ssh.private-key-files&lt;\/name&gt;\n        &lt;value&gt;\/home\/hadoop\/.ssh\/id_rsa&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.ha.automatic-failover.enabled&lt;\/name&gt;\n        &lt;value&gt;true&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.webhdfs.enabled&lt;\/name&gt;\n        &lt;value&gt;true&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.support.append&lt;\/name&gt;\n        &lt;value&gt;true&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.support.broken.append&lt;\/name&gt;\n        &lt;value&gt;true&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.permissions&lt;\/name&gt;\n        &lt;value&gt;false&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.block.size&lt;\/name&gt;\n        &lt;value&gt;268435456&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.namenode.handler.count&lt;\/name&gt;\n        &lt;value&gt;20&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.datanode.handler.count&lt;\/name&gt;\n        &lt;value&gt;20&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;tasktracker.http.threads&lt;\/name&gt;\n        &lt;value&gt;400&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;dfs.hosts.exclude&lt;\/name&gt;\n        &lt;value&gt;\/app\/hadoop\/etc\/hadoop\/dfs.exclude&lt;\/value&gt;\n        &lt;final&gt;true&lt;\/final&gt;\n    &lt;\/property&gt;\n&lt;\/configuration&gt;\n<\/code><\/pre>\n<h4>yarn-site.xml<\/h4>\n<p><code>vi yarn-site.xml<\/code><\/p>\n<pre><code class=\"xml\">&lt;?xml version=\"1.0\"?&gt;\n\n&lt;configuration&gt;\n\n    &lt;property&gt;\n        &lt;name&gt;yarn.acl.enable&lt;\/name&gt;\n        &lt;value&gt;0&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.nodemanager.aux-service&lt;\/name&gt;\n        &lt;value&gt;mapreduce_shuffle&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.nodemanager.aux-service.mapreduce_shuffle.class&lt;\/name&gt;\n        &lt;value&gt;org.apache.hadoop.mapred.ShuffledHandler&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.nodemanager.local-dirs&lt;\/name&gt;\n        &lt;value&gt;\/data\/hadoop\/yarn\/nm-local-dir&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.fs-state.uri&lt;\/name&gt;\n        &lt;value&gt;\/data\/hadoop\/yarn\/system\/rmstore&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;!-- for YARN RM HA --&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.recovery.enabled&lt;\/name&gt;\n        &lt;value&gt;true&lt;value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.store.class&lt;\/name&gt;\n        &lt;value&gt;org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.ha.enabled&lt;\/name&gt;\n        &lt;value&gt;true&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.cluster-id&lt;\/name&gt;\n        &lt;value&gt;yarn-cluster&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.ha.rm-ids&lt;\/name&gt;\n        &lt;value&gt;rm1,rm2&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.hostname.rm1&lt;\/name&gt;\n        &lt;value&gt;node1.df&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.hostname.rm2&lt;\/name&gt;\n        &lt;value&gt;node2.df&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.address.rm1&lt;\/name&gt;\n        &lt;value&gt;node1.df:8050&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.address.rm2&lt;\/name&gt;\n        &lt;value&gt;node2.df:8050&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.webapp.address.rm1&lt;\/name&gt;\n        &lt;value&gt;node1.df:8088&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.webapp.address.rm2&lt;\/name&gt;\n        &lt;value&gt;node2.df:8088&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.zk-address&lt;\/name&gt;\n        &lt;value&gt;node1.zk:2181,node2.zk:2181,node3.zk:2181&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.ha.automatic-failover.enabled&lt;\/name&gt;\n        &lt;value&gt;true&lt;\/value&gt;\n        &lt;description&gt;Enable automatic failover; By default, it si enabled only when HA is enabled.&lt;\/description&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.resource-tracker.address.rm1&lt;\/name&gt;\n        &lt;value&gt;node1.df:8025&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.resource-tracker.address.rm2&lt;\/name&gt;\n        &lt;value&gt;node2.df:8025&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.scheduler.address.rm1&lt;\/name&gt;\n        &lt;value&gt;node1.df:8030&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.scheduler.address.rm2&lt;\/name&gt;\n        &lt;value&gt;node2.df:8030&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.connect.retry-interval.ms&lt;\/name&gt;\n        &lt;value&gt;2000&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.resourcemanager.store.class&lt;\/name&gt;\n        &lt;value&gt;org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.nodemanager.disk-health-checker.max-disk-utilizaion-per-disk-percentage&lt;\/name&gt;\n        &lt;value&gt;98.5&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;!--for Spark  --&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.nodemanager.pmem-check-enabled&lt;\/name&gt;\n        &lt;value&gt;false&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.nodemanager.vmem-check-enabled&lt;\/name&gt;\n        &lt;value&gt;false&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;!--for App Memery  --&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.nodemanager.resource.memory-mb&lt;\/name&gt;\n        &lt;value&gt;2048&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.scheduler.maximum-allocation-mb&lt;\/name&gt;\n        &lt;value&gt;4096&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;!-- for Spark Dynamic Setting --&gt;\n    &lt;!-- spark.executor.cores * spark.executor.instances  --&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.nodemanager.resource.cpu-vcores&lt;\/name&gt;\n        &lt;value&gt;2&lt;\/value&gt;\n    &lt;\/property&gt;\n    &lt;property&gt;\n        &lt;name&gt;yarn.scheduler.maximum-allocation-vcores&lt;\/name&gt;\n        &lt;value&gt;2&lt;\/value&gt;\n    &lt;\/property&gt;\n&lt;\/configuration&gt;\n<\/code><\/pre>\n<h4>hadoop-env.sh<\/h4>\n<p><code>vi hadoop-env.sh<\/code><\/p>\n<pre><code class=\"bash\">#export JAVA_HOME=${JAVA_HOME}\nexport JAVA_HOME=\/app\/jdk\nexport HADOOP_SSH_OPTS=\"-p 22\"\n<\/code><\/pre>\n<h3>Namenode format \ubc0f Hadoop Cluster \uc2e4\ud589<\/h3>\n<h4>Namenode format<\/h4>\n<p>\ubaa8\ub4e0 \ub178\ub4dc\uc5d0\uc11c \uc544\ub798 \uba85\ub839 \uc218\ud589<\/p>\n<pre><code class=\"bash\">touch \/app\/hadoop\/etc\/hadoop\/dfs.exclude\nhadoop-daemon.sh start journalnode\n<\/code><\/pre>\n<p>Primary Namenode \uc5d0\uc11c \uc544\ub798 \uba85\ub839 \uc218\ud589<\/p>\n<pre><code class=\"bash\">hdfs namenode -format\nscp -r \/data\/hadoop\/namenode\/current node2.df:\/data\/hadoop\/namenode\/.\nhdfs zkfc -formatZK\n<\/code><\/pre>\n<p>\ubaa8\ub4e0 \ub178\ub4dc\uc5d0\uc11c \uc544\ub798 \uba85\ub839 \uc218\ud589<\/p>\n<pre><code class=\"bash\">hadoop-daemon.sh stop journalnode\n<\/code><\/pre>\n<h4>Hadoop Cluster \uc2e4\ud589<\/h4>\n<p>Primary Namenode \uc5d0\uc11c \uc544\ub798 \uba85\ub839 \uc218\ud589<\/p>\n<pre><code class=\"bash\">start-dfs.sh\nstart-yarn.sh\n<\/code><\/pre>\n<p>Standy-by Resourcemanager node \uc5d0\uc11c \uc544\ub798 \uba85\ub839 \uc218\ud589<\/p>\n<pre><code class=\"bash\">yarn-daemon.sh start resourcemanager\n<\/code><\/pre>\n<hr \/>\n","protected":false},"excerpt":{"rendered":"<p>\uac00\uc6a9\uc131\uc744 \uace0\ub824\ud55c Hadoop Cluster \uc124\uce58 Namenode\uc640 Resourcemanager\ub97c active node,standby node\ub85c \uac01\uac01 \ub450 \ubc8c\uc529 \uad6c\uc131\ud568\uc73c\ub85c\uc368 \uace0\uac00\uc6a9\uc131\uc744 \uace0\ub824\ud55c Hadoop Cluster\ub97c \uc124\uce58\ud55c\ub2e4. \uc0ac\uc804 \uc900\ube44 \uc758\uc874 \ud328\ud0a4\uc9c0 \uc124\uce58 namenode\uc640 resourcemanager\uac00 \uc7a5\uc560 \uc0c1\ud669\uc5d0\uc11c standby -&gt; active \uc0c1\ud0dc\ub85c \uc804\ud658\ub420 \ub54c\uc5d0 fuser \uba85\ub839\uc744 \uc774\uc6a9\ud574\uc11c \uc7a5\uc560\ub178\ub4dc\uc758 \ud504\ub85c\uc138\uc2a4\ub4e4\uc744 \uc815\ub9ac\ud558\ubbc0\ub85c psmisc \ud328\ud0a4\uc9c0\ub97c \uc124\uce58\ud55c\ub2e4. yum -y install psmisc JDK https:\/\/www.oracle.com\/technetwork\/java\/javase\/downloads\/jdk8-downloads-2133151.html \ud398\uc774\uc9c0\uc5d0\uc11c JDK 8 \ubc84\uc804 \ubc14\uc774\ub108\ub9ac \ub2e4\uc6b4\ub85c\ub4dc\ud558\uace0 [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[71],"tags":[107,98,108,50],"class_list":["post-826","post","type-post","status-publish","format-standard","hentry","category-hadoop","tag-cluster","tag-hadoop","tag-108","tag-50"],"_links":{"self":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/826","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/comments?post=826"}],"version-history":[{"count":8,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/826\/revisions"}],"predecessor-version":[{"id":1360,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/826\/revisions\/1360"}],"wp:attachment":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/media?parent=826"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/categories?post=826"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/tags?post=826"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}