{"id":875,"date":"2019-03-05T16:00:48","date_gmt":"2019-03-05T07:00:48","guid":{"rendered":"https:\/\/dong1lkim.oboki.net\/?p=875"},"modified":"2019-09-01T22:23:15","modified_gmt":"2019-09-01T13:23:15","slug":"hadoop-oozie-%ec%84%a4%ec%b9%98","status":"publish","type":"post","link":"https:\/\/oboki.net\/workspace\/data-engineering\/hadoop\/hadoop-oozie-%ec%84%a4%ec%b9%98\/","title":{"rendered":"[Hadoop] Oozie \uc124\uce58"},"content":{"rendered":"<h1>Oozie \uc124\uce58<\/h1>\n<blockquote><p>\n  \uc544\ud30c\uce58 \uc6b0\uc9c0(Apache Oozie)\ub294 \ud558\ub461\uc758 \uc7a1(job)\uc744 \uad00\ub9ac\ud558\uae30 \uc704\ud55c \uc11c\ubc84 \uae30\ubc18\uc758 \uc6cc\ud06c\ud50c\ub85c \uc2a4\ucf00\uc904\ub9c1 \uc2dc\uc2a4\ud15c\uc774\ub2e4.\n<\/p><\/blockquote>\n<h2>\uc0ac\uc804 \uc900\ube44<\/h2>\n<p>\ub2e4\uc74c\uacfc \uac19\uc740 \uc0ac\uc804 \uc900\ube44 \uc791\uc5c5\uc774 \ud544\uc694\ud558\ub2e4.<\/p>\n<h3>Java<\/h3>\n<p><code>java -version<\/code><\/p>\n<pre><code class=\"bash\">java version \"1.8.0_181\"\nJava(TM) SE Runtime Environment (build 1.8.0_181-b13)\nJava HotSpot(TM) 64-Bit Server VM (build 25.181-b13, mixed mode)\n<\/code><\/pre>\n<h3>Maven<\/h3>\n<p><code>mvn -version<\/code><\/p>\n<pre><code class=\"bash\">Apache Maven 3.6.0 (97c98ec64a1fdfee7767ce5ffb20918da4f719f3; 2018-10-25T03:41:47+09:00)\nMaven home: \/app\/maven\nJava version: 1.8.0_181, vendor: Oracle Corporation, runtime: \/app\/jdk\/jdk1.8.0_181\/jre\nDefault locale: en_US, platform encoding: UTF-8\nOS name: \"linux\", version: \"3.10.0-862.el7.x86_64\", arch: \"amd64\", family: \"unix\"\n<\/code><\/pre>\n<h3>Hadoop<\/h3>\n<ul>\n<li><code>core-site.xml<\/code><\/li>\n<\/ul>\n<pre><code class=\"xml\">  &lt;property&gt;\n    &lt;name&gt;hadoop.proxyuser.root.groups&lt;\/name&gt;\n    &lt;value&gt;*&lt;\/value&gt;\n  &lt;\/property&gt;\n  &lt;property&gt;\n    &lt;name&gt;hadoop.proxyuser.root.hosts&lt;\/name&gt;\n    &lt;value&gt;*&lt;\/value&gt;\n  &lt;\/property&gt;\n<\/code><\/pre>\n<ul>\n<li>oozie \ub514\ub809\ud1a0\ub9ac \uc0dd\uc131<\/li>\n<\/ul>\n<pre><code class=\"bash\">hadoop fs -mkdir \/user\/oozie\nhadoop fs -chown oozie:hadoop \/user\/oozie\n<\/code><\/pre>\n<h3>MySQL<\/h3>\n<p><code>mysql --version<\/code><\/p>\n<pre><code class=\"txt\">mysql  Ver 14.14 Distrib 5.7.25, for el7 (x86_64) using  EditLine wrapper\n<\/code><\/pre>\n<p>\ub2e4\uc74c\uacfc \uac19\uc774 \ub370\uc774\ud130\ubca0\uc774\uc2a4\ub97c \ubbf8\ub9ac \uc0dd\uc131\ud574\uc11c \uc0ac\uc6a9\ud55c\ub2e4.<\/p>\n<pre><code class=\"sql\">create database oozie;\ncreate user oozie@localhost identified by 'oozie';\ngrant all privileges on oozie.* to oozie@localhost;\n<\/code><\/pre>\n<h2>Oozie \uc124\uce58<\/h2>\n<h3>oozie \uc18c\uc2a4 \ub2e4\uc6b4\ub85c\ub4dc \ubc0f \ube4c\ub4dc<\/h3>\n<p>\ub2e4\uc74c\uacfc \uac19\uc774 oozie-4.3.1 \uc18c\uc2a4\ub97c \ub2e4\uc6b4\ubc1b\uc544 \uc124\uce58\ub97c \uc9c4\ud589\ud55c\ub2e4. \uc124\uce58 \uacbd\ub85c\ub294 <code>\/app\/oozie<\/code><\/p>\n<pre><code class=\"bash\">wget http:\/\/apache.mirror.cdnetworks.com\/oozie\/4.3.1\/oozie-4.3.1.tar.gz\ntar -xvzf oozie-4.3.1.tar.gz -C \/app\/oozie\/.\nmv \/app\/oozie\/oozie-4.3.1 \/app\/oozie\/4.3.1\ncd \/app\/oozie\/4.3.1\nvi pom.xml\n<\/code><\/pre>\n<p><code>vi pom.xml<\/code> \ud30c\uc77c\uc5d0\uc11c hadoop.version \ud544\ub4dc \uc911 2.4.0 \uc774\ub77c\uace0 \ub41c \ubd80\ubd84\ub9cc 2.7.3 \uc73c\ub85c \ubcc0\uacbd\ud55c\ub2e4.<\/p>\n<p>\uc0ac\uc6a9\ud558\ub294 \ud558\ub461\uc740 2.9.2 \ubc84\uc804\uc778\ub370 \uc774\ub807\uac8c \uae30\uc785\ud588\ub354\ub2c8 \uc548 \ub428. \ubaa8\ub4e0 \ubc84\uc804\uc73c\ub85c \ud14c\uc2a4\ud2b8\ud574\ubcf4\uc9c0\ub294 \ubabb\ud588\uace0 \uc77c\ub2e8 2.7.3 \uc73c\ub85c \ud588\uc744 \ub54c \uc6f9 \ucf58\uc194 \ub744\uc6b0\ub294 \uac83 \uae4c\uc9c0\ub294 \uc131\uacf5\ud568.<\/p>\n<pre><code class=\"bash\">cd \/app\/oozie\/4.3.1\/bin\n.\/mkdistro.sh -DskipTests\n<\/code><\/pre>\n<p>\ube4c\ub4dc \ub3c4\uc911\uc5d0 \uc5d0\ub7ec\uac00 \ub9ce\uc774 \ub098\uc624\ub294\ub370 \uc5b4\uca0c\ub4e0 \ubc11\uc5d0\ucc98\ub7fc oozie distro \uc0dd\uc131\ub9cc \ub418\uba74 \ub420\ub4ef<\/p>\n<pre><code class=\"txt\">Oozie distro created, DATE[2019.03.05-00:07:54GMT] VC-REV[unavailable], available at [\/app\/oozie\/4.3.1\/distro\/target]\n<\/code><\/pre>\n<p><code>\/app\/oozie<\/code> \uacbd\ub85c\uc5d0 \uc2ec\ubcfc\ub9ad \ub9c1\ud06c\ub97c \uac78\uc5b4\ub450\uace0 \ud658\uacbd \ubcc0\uc218\ub97c \uc124\uc815\ud55c \ub4a4 \uc0ac\uc6a9\ud55c\ub2e4.<\/p>\n<pre><code class=\"bash\">cd \/app\/oozie\nln -s 4.3.1\/distro\/target\/oozie-4.3.1-distro\/oozie-4.3.1\/* .\nexport PATH=\/app\/oozie\/bin:$PATH\n<\/code><\/pre>\n<h3>oozie-site.xml<\/h3>\n<p><code>\/app\/oozie\/conf\/oozie-site.xml<\/code> \ud30c\uc77c\uc744 \uc5f4\uc5b4, \ubbf8\ub9ac \uc0dd\uc131\ud55c mysql \ub370\uc774\ud130\ubca0\uc774\uc2a4 \uc815\ubcf4\ub97c \uc544\ub798\uc640 \uac19\uc774 \uc785\ub825\ud55c\ub2e4.<\/p>\n<pre><code class=\"xml\">&lt;configuration&gt;\n    &lt;property&gt;  \n        &lt;name&gt;oozie.service.JPAService.jdbc.driver&lt;\/name&gt;  \n        &lt;value&gt;com.mysql.jdbc.Driver&lt;\/value&gt;  \n    &lt;\/property&gt;  \n\n    &lt;property&gt;  \n        &lt;name&gt;oozie.service.JPAService.jdbc.url&lt;\/name&gt;  \n        &lt;value&gt;jdbc:mysql:\/\/localhost:3306\/oozie&lt;\/value&gt;  \n    &lt;\/property&gt;  \n\n    &lt;property&gt;  \n        &lt;name&gt;oozie.service.JPAService.create.db.schema&lt;\/name&gt;  \n        &lt;value&gt;true&lt;\/value&gt;  \n    &lt;\/property&gt;  \n\n    &lt;property&gt;  \n        &lt;name&gt;oozie.service.JPAService.jdbc.username&lt;\/name&gt;  \n        &lt;value&gt;oozie&lt;\/value&gt;  \n    &lt;\/property&gt;  \n\n    &lt;property&gt;  \n        &lt;name&gt;oozie.service.JPAService.jdbc.password&lt;\/name&gt;  \n        &lt;value&gt;oozie&lt;\/value&gt;  \n    &lt;\/property&gt;\n&lt;\/configuration&gt;\n<\/code><\/pre>\n<h3>\ub77c\uc774\ube0c\ub7ec\ub9ac \ucd94\uac00<\/h3>\n<p><code>libext<\/code> \ub514\ub809\ud1a0\ub9ac\ub97c \uc0dd\uc131\ud574\uc11c \ucd94\uac00 \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc704\uce58\uc2dc\ud0a8\ub2e4.<\/p>\n<ul>\n<li>hadoop<\/li>\n<li>mysql jdbc driver<\/li>\n<li>extJS for web console<\/li>\n<\/ul>\n<pre><code class=\"bash\">mkdir \/app\/oozie\/libext\ncd \/app\/oozie\/libext\nfind \/app\/hadoop\/share\/hadoop -name \"*.jar\" | xargs  -I {} cp {} .\ncp \/app\/hive\/lib\/mysql-connector-java-5.1.47.jar .\nwget http:\/\/archive.cloudera.com\/gplextras\/misc\/ext-2.2.zip\n<\/code><\/pre>\n<p><code>mysql connector<\/code>\ub294 \uae30\uc874\uc5d0 <code>hive<\/code>\uc5d0\uc11c \uc0ac\uc6a9\ud588\ub358 \uac83\uc744 \uc0ac\uc6a9\ud588\ub294\ub370 \uc5c6\ub294 \uacbd\uc6b0, \uc544\ub798 url\uc5d0\uc11c \ub2e4\uc6b4\ubc1b\uc744 \uc218 \uc788\ub2e4.<\/p>\n<p><a href=\"https:\/\/dev.mysql.com\/downloads\/connector\/j\/5.1.html\"><a href=\"https:\/\/dev.mysql.com\/downloads\/connector\/j\/5.1.html\">https:\/\/dev.mysql.com\/downloads\/connector\/j\/5.1.html<\/a><\/a><\/p>\n<p>\uc544\ub798 \uba85\ub839\uc73c\ub85c <code>oozie webapp<\/code>\uc744 \uc0dd\uc131\ud55c\ub2e4.<\/p>\n<pre><code class=\"bash\">oozie-setup.sh prepare-war \n<\/code><\/pre>\n<p><code>extJS<\/code> \ub97c \ub2e4\uc6b4\ubc1b\uc544\ub193\uc9c0 \uc54a\uc558\uc73c\uba74 \uc544\ub798\uc640 \uac19\uc774 \ub728\uace0 \uc6f9\ube0c\ub77c\uc6b0\uc800\uc5d0\uc11c \uc811\uadfc\uc774 \uc548 \ub428. extJS\ub294 \ub77c\uc774\uc13c\uc2a4\uac00 \ub2ec\ub77c\uc11c \uae30\ubcf8 \uc81c\uacf5\ud558\uc9c0 \uc54a\ub294\ub2e4\uace0 \ud55c\ub2e4.<\/p>\n<pre><code class=\"txt\">INFO: Oozie webconsole disabled, ExtJS library not specified\n<\/code><\/pre>\n<p>libext \uacbd\ub85c\uc5d0 <code>extJS<\/code> \ub97c \ubc1b\uc544\ub1a8\uae30 \ub54c\ubb38\uc5d0 \uc544\ub798\uc640 \uac19\uc774 \ub85c\uadf8\uac00 \ub0a8\uc74c.<\/p>\n<pre><code class=\"txt\">New Oozie WAR file with added 'ExtJS library, JARs' at \/app\/oozie\/oozie-server\/webapps\/oozie.war\n<\/code><\/pre>\n<p>\uc774\uc5b4\uc11c sharelib \uba85\ub839\uc73c\ub85c hadoop \ud30c\uc77c\uc2dc\uc2a4\ud15c\uc5d0 \uacf5\uc720 \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc5c5\ub85c\ub4dc\ud55c\ub2e4.<\/p>\n<pre><code class=\"bash\">oozie-setup.sh sharelib create -fs hdfs:\/\/192.168.179.144:8020\n<\/code><\/pre>\n<pre><code class=\"java\">Exception in thread \"main\" java.lang.NoSuchMethodError: org.apache.hadoop.security.authentication.util.KerberosUtil.hasKerberosTicket(Ljavax\/security\/auth\/Subject;)Z\n    at org.apache.hadoop.security.UserGroupInformation.&lt;init&gt;(UserGroupInformation.java:670)\n    at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:865)\n    at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:824)\n    at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:693)\n    at org.apache.hadoop.fs.FileSystem$Cache$Key.&lt;init&gt;(FileSystem.java:3426)\n    at org.apache.hadoop.fs.FileSystem$Cache$Key.&lt;init&gt;(FileSystem.java:3416)\n    at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3258)\n    at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:470)\n    at org.apache.oozie.tools.OozieSharelibCLI.run(OozieSharelibCLI.java:172)\n    at org.apache.oozie.tools.OozieSharelibCLI.main(OozieSharelibCLI.java:67)\n<\/code><\/pre>\n<p>\uc704 \uc5d0\ub7ec\ub294 hadoop-auth \uc5d0\uc11c hasKerberosTicket \uba54\uc18c\ub4dc\uac00 \uc5c6\uc5b4\uc11c \ub728\ub294\uac74\ub370 <code>\/app\/oozie\/lib\/hadoop-auth-2.7.3.jar<\/code> \ud30c\uc77c\uc744 \uc5c6\uc560\uc8fc\uba74 \ub41c\ub2e4.<\/p>\n<pre><code class=\"bash\">mv \/app\/oozie\/lib\/hadoop-auth-2.7.3.jar ~\/.\n<\/code><\/pre>\n<p>\ub2e4\uc2dc sharelib<\/p>\n<pre><code class=\"bash\">oozie-setup.sh sharelib create -fs hdfs:\/\/192.168.179.144:8020\n<\/code><\/pre>\n<p>\ubc84\uc804\uc5d0 \ub530\ub77c\uc11c \ub2e4\ub978 \uc5d0\ub7ec\uac00 \ub9ce\uc774 \ub0a0 \uc218\ub3c4 \uc788\ub2e4. \ud55c\ub540\ud55c\ub540 \ud558\ub098\uc529 jar \uc704\uce58\uc2dc\ucf1c\uac00\uba74\uc11c \ud558\ub294\uac8c \uc81c\uc77c \uc88b\uc744\uac83\uac19\uc9c0\ub9cc \uc704\uc5d0\uc11c \ud55c\uac83\ucc98\ub7fc \ub2e4 \uc62e\uaca8\ub193\uace0 NoSuchMethodError \ubc1c\uc0dd\ud55c \uac83\uc5d0 \ub300\ud574\uc11c\ub9cc \uace0\uccd0\uc8fc\ub294\uac8c \uc81c\uc77c \ube60\ub97c \uac83 \uac19\ub2e4.<\/p>\n<h3>\uba54\ud0c0\ub370\uc774\ud130\ubca0\uc774\uc2a4 \uc0dd\uc131<\/h3>\n<p>\uc544\ub798 \uba85\ub839\uc73c\ub85c mysql\uc5d0 \uba54\ud0c0\ub370\uc774\ud130\ub97c \uc0dd\uc131\ud55c\ub2e4.<\/p>\n<pre><code class=\"bash\">ooziedb.sh create -sqlfile ooziedb_init_schema.sql -run\n<\/code><\/pre>\n<pre><code class=\"java\">Exception in thread \"main\" java.lang.NoSuchMethodError: org.hsqldb.DatabaseURL.parseURL(Ljava\/lang\/String;ZZ)Lorg\/hsqldb\/persist\/HsqlProperties;\n    at org.hsqldb.jdbc.JDBCDriver.getConnection(Unknown Source)\n    at org.hsqldb.jdbc.JDBCDriver.connect(Unknown Source)\n    at java.sql.DriverManager.getConnection(DriverManager.java:664)\n    at java.sql.DriverManager.getConnection(DriverManager.java:270)\n    at org.apache.openjpa.jdbc.sql.DerbyDictionary.closeDataSource(DerbyDictionary.java:179)\n    at org.apache.openjpa.jdbc.conf.JDBCConfigurationImpl.preClose(JDBCConfigurationImpl.java:969)\n    at org.apache.openjpa.lib.conf.ConfigurationImpl.close(ConfigurationImpl.java:368)\n    at org.apache.openjpa.jdbc.meta.MappingTool$1.run(MappingTool.java:941)\n    at org.apache.openjpa.lib.conf.Configurations.launchRunnable(Configurations.java:762)\n    at org.apache.openjpa.lib.conf.Configurations.runAgainstAllAnchors(Configurations.java:752)\n    at org.apache.openjpa.jdbc.meta.MappingTool.main(MappingTool.java:934)\n    at org.apache.oozie.tools.OozieDBCLI.createUpgradeDB(OozieDBCLI.java:1101)\n    at org.apache.oozie.tools.OozieDBCLI.createDB(OozieDBCLI.java:189)\n    at org.apache.oozie.tools.OozieDBCLI.run(OozieDBCLI.java:127)\n    at org.apache.oozie.tools.OozieDBCLI.main(OozieDBCLI.java:78)\n<\/code><\/pre>\n<p>\uc704 \uc5d0\ub7ec\ub294 <code>\/app\/oozie\/lib\/hsqldb-1.8.0.10.jar<\/code> \ud30c\uc77c\uc744 \uc5c6\uc560\ubc84\ub9ac\uba74 \ub41c\ub2e4. \uc5c6\uc560\ubc84\ub9ac\uace0 \ub2e4\uc2dc \uc2dc\ub3c4.<\/p>\n<pre><code class=\"bash\">mv \/app\/oozie\/lib\/hsqldb-1.8.0.10.jar ~\/.\nooziedb.sh create -sqlfile ooziedb_init_schema.sql -run\n\n\uc544\ub798\uc640 \uac19\uc774 \ud504\ub86c\ud504\ud2b8\ub418\uba74 \uc124\uce58 \uc644\ub8cc.\n\n```txt\n  setting CATALINA_OPTS=\"$CATALINA_OPTS -Xmx1024m\"\nValidate DB Connection\nDONE\nDB schema does not exist\nCheck OOZIE_SYS table does not exist\nDONE\nCreate SQL schema\nDONE\nCreate OOZIE_SYS table\nDONE\n\nOozie DB has been created for Oozie version '4.3.1'\n\nThe SQL commands have been written to: ooziedb_init_schema.sql\n<\/code><\/pre>\n<h2>Oozie \uae30\ub3d9 \ubc0f \uc885\ub8cc<\/h2>\n<h3>\uae30\ub3d9<\/h3>\n<pre><code class=\"bash\">oozied.sh start\n<\/code><\/pre>\n<p><code><a href=\"http:\/\/192.168.179.144:11000\/oozie\/\">http:\/\/192.168.179.144:11000\/oozie\/<\/a><\/code> url\uc5d0\uc11c web console \uc811\uadfc\uc774 \uac00\ub2a5\ud558\ub2e4.<\/p>\n<p><img decoding=\"async\" src=\"\/workspace\/wp-content\/uploads\/2019\/03\/web_console.png\" alt=\"\" \/><\/p>\n<h3>\uc885\ub8cc<\/h3>\n<pre><code class=\"bash\">oozied.sh stop\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>Oozie \uc124\uce58 \uc544\ud30c\uce58 \uc6b0\uc9c0(Apache Oozie)\ub294 \ud558\ub461\uc758 \uc7a1(job)\uc744 \uad00\ub9ac\ud558\uae30 \uc704\ud55c \uc11c\ubc84 \uae30\ubc18\uc758 \uc6cc\ud06c\ud50c\ub85c \uc2a4\ucf00\uc904\ub9c1 \uc2dc\uc2a4\ud15c\uc774\ub2e4. \uc0ac\uc804 \uc900\ube44 \ub2e4\uc74c\uacfc \uac19\uc740 \uc0ac\uc804 \uc900\ube44 \uc791\uc5c5\uc774 \ud544\uc694\ud558\ub2e4. Java java -version java version &#8220;1.8.0_181&#8221; Java(TM) SE Runtime Environment (build 1.8.0_181-b13) Java HotSpot(TM) 64-Bit Server VM (build 25.181-b13, mixed mode) Maven mvn -version Apache Maven 3.6.0 (97c98ec64a1fdfee7767ce5ffb20918da4f719f3; 2018-10-25T03:41:47+09:00) Maven home: \/app\/maven Java [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[71],"tags":[98,99,50],"class_list":["post-875","post","type-post","status-publish","format-standard","hentry","category-hadoop","tag-hadoop","tag-oozie","tag-50"],"_links":{"self":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/875","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/comments?post=875"}],"version-history":[{"count":2,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/875\/revisions"}],"predecessor-version":[{"id":1352,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/875\/revisions\/1352"}],"wp:attachment":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/media?parent=875"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/categories?post=875"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/tags?post=875"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}