{"id":1544,"date":"2020-05-24T22:41:59","date_gmt":"2020-05-24T13:41:59","guid":{"rendered":"https:\/\/oboki.net\/?p=1544"},"modified":"2020-06-23T00:04:54","modified_gmt":"2020-06-22T15:04:54","slug":"airflow-1-10-10-%ec%8b%b1%ea%b8%80-%ec%84%a4%ec%b9%98","status":"publish","type":"post","link":"https:\/\/oboki.net\/workspace\/python\/airflow-1-10-10-%ec%8b%b1%ea%b8%80-%ec%84%a4%ec%b9%98\/","title":{"rendered":"[Airflow] 1.10.10 \uc2f1\uae00 \uc124\uce58"},"content":{"rendered":"

rbac<\/code> \uae30\ub2a5\uc744 \uc774\uc6a9\ud558\uba74 DAG-level Access Control<\/code> \uc774 \uac00\ub2a5\ud558\ub2e4\uace0 \ud55c\ub2e4. \uc0c8\ub86d\uac8c \ucd94\uac00\ub41c tag \uae30\ub2a5\uc5d0 \ub354\ud574\uc838\uc11c tag-level \uad8c\ud55c \uad00\ub9ac\ub3c4 \ub410\uc73c\uba74 \uc88b\uc558\uaca0\uc9c0\ub9cc..<\/del> rbac \uae30\ub2a5\uc744 \uc774\uc6a9\ud558\uba74 \uae30\ubcf8\uc801 \uc81c\uacf5\ub418\ub294 \ub864 \ubfd0\ub9cc \uc544\ub2c8\ub77c Customized role \uc744 \ud1b5\ud574 \uac00\ub2a5\ud574\uc9c8 \uc0ac\uc6a9\uc790 \uad8c\ud55c \uad00\ub9ac\ub97c \uae30\ub300\ud558\uba70 \ucd5c\uc2e0 1.10.10 \ubc84\uc804\uc744 \uc124\uce58\ud574\ubcf8\ub2e4.<\/p>\n

\uae30\uc874\uc5d0 LDAP \uc5f0\ub3d9\ud574\uc11c \uc0ac\uc6a9\ud558\ub358 Airflow \uc2dc\uc2a4\ud15c\uc744 \uc5c5\uadf8\ub808\uc774\ub4dc \ud574\uc11c \uc0ac\uc6a9\ud560 \uc218 \uc788\uc744\uc9c0 \ud655\uc778\ud574\ubcf4\uae30 \uc704\ud574 OpenLDAP \ud658\uacbd\uc744 \uac04\ub2e8\ud558\uac8c \uad6c\ucd95\ud55c \ub4a4 rbac \uc774 \uc815\uc0c1 \ub3d9\uc791 \ud558\ub294\uc9c0 \ud655\uc778 \uba3c\uc800 \ud574\ubcf8 \ub4a4 \uc774\ub7f0\uc800\ub7f0 \uae30\ub2a5\uc744 \uc0ac\uc6a9\ud574\ubcfc \uc608\uc815\uc774\uba70 CentOS7 \uc11c\ubc84\uc5d0 \uc124\uce58\ud55c\ub2e4.<\/p>\n

\n

OpenLDAP \uc124\uce58; https:\/\/oboki.net\/workspace\/system\/linux\/openldap-%ec%84%a4%ec%b9%98\/<\/a><\/p>\n<\/blockquote>\n

\ud658\uacbd\ubcc0\uc218 \uc124\uc815<\/h2>\n

\ub2e4\uc74c\uacfc \uac19\uc774 \ud658\uacbd\ubcc0\uc218\ub97c \uc124\uc815\ud588\uae30 \ub54c\ubb38\uc5d0 mysql, python \uc740 \uba3c\uc800 \uc900\ube44\uac00 \ub418\uc5b4\uc57c \ud55c\ub2e4.<\/p>\n

export PATH=\\\n$HOME\/usr\/mysql\/bin:\\\n$HOME\/usr\/python\/bin:\\\n$JAVA_HOME\/bin:\\\n$PATH\nexport LD_LIBRARY_PATH=\\\n$HOME\/usr\/mysql\/lib:\\\n$HOME\/usr\/python\/lib:\\\n$LD_LIBRARY_PATH\nexport AIRFLOW_HOME=$HOME\/usr\/airflow<\/code><\/pre>\n
Python \uc124\uce58<\/h2>\n
\ud30c\uc774\uc36c \uc124\uce58\uc5d0 \uc55e\uc11c \uc0ac\uc804 \uc900\ube44\ud574\uc57c\ud558\ub294 \uc2dc\uc2a4\ud15c \ub77c\uc774\ube0c\ub7ec\ub9ac\uac00 \ub9ce\uc544\uc11c \ud30c\uc774\uc36c \uc124\uce58 \uc2a4\ud06c\ub9bd\ud2b8\ub97c \ub530\ub85c \uc801\uc5b4\ub460.<\/p>\n
yum -y install gcc openssl-devel zlib-devel libffi-devel bzip2-devel sqlite-devel ncurses-devel gdbm-devel tk-devel xz-devel readline-devel openldap-devel\ncurl -O https:\/\/www.python.org\/ftp\/python\/3.7.7\/Python-3.7.7.tgz && tar xf Python-3.7.7.tgz && cd Python-3.7.7 \n.\/configure --prefix=$HOME\/usr\/python\/3.7.7 --enable-shared && make -j 4 && make install\ncd $HOME\/usr\/python && ln -s .\/3.7.7\/* .\nwhich python3\npip3 install --upgrade setuptools pip wheel<\/code><\/pre>\n
\ub370\uc774\ud130\ubca0\uc774\uc2a4 \uc0dd\uc131<\/h2>\n
airflow:airflow@localhost:3306\/airflow<\/code> \uc640 \uac19\uc774 \ub370\uc774\ud130\ubca0\uc774\uc2a4\ub97c \uc0dd\uc131\ud588\uace0 explicit_defaults_for_timestamp<\/code> \uac12\uc744 \ud65c\uc131\ud654\ud588\ub2e4. (\uc138\uc158 \ub2e8\uc704\ub85c \uc124\uc815\ud574\ub3c4 \ub418\uc9c0\ub9cc \ud3b8\uc758\uc0c1 global \ud30c\ub77c\ubbf8\ud130\ub97c \uc801\uc6a9)<\/p>\n
create database airflow;\ncreate user airflow@'%' identified by 'airflow';\ngrant all privileges on airflow.* to airflow@'%';\nflush privileges;\nset global explicit_defaults_for_timestamp = 1;<\/code><\/pre>\n
Airflow \uc124\uce58<\/h2>\n
Airflow \ud328\ud0a4\uc9c0 \uc124\uce58<\/h3>\n
pip3 install -r requirements.txt<\/code><\/pre>\n
airflow 1.10.10 \uc124\uce58 \uae30\uc900\uc73c\ub85c requirements.txt<\/code>\ub294 \ub2e4\uc74c\uacfc \uac19\ub2e4.<\/p>\n
cat requirements.txt<\/code><\/p>\n
alembic==1.4.2\namqp==2.5.2\napache-airflow==1.10.10\napispec==1.3.3\nargcomplete==1.11.1\nattrs==19.3.0\nBabel==2.8.0\nbilliard==3.6.3.0\ncached-property==1.5.1\ncattrs==0.9.2\ncelery==4.4.2\ncertifi==2020.4.5.1\ncffi==1.14.0\nchardet==3.0.4\nclick==7.1.2\ncolorama==0.4.3\ncolorlog==4.0.2\nconfigparser==3.5.3\ncroniter==0.3.31\ncryptography==2.9.2\ndefusedxml==0.6.0\ndill==0.3.1.1\ndnspython==1.16.0\ndocutils==0.16\nemail-validator==1.1.0\nFlask==1.1.2\nFlask-Admin==1.5.4\nFlask-AppBuilder==2.3.4\nFlask-Babel==1.0.0\nFlask-Caching==1.3.3\nFlask-JWT-Extended==3.24.1\nFlask-Login==0.4.1\nFlask-OpenID==1.2.5\nFlask-SQLAlchemy==2.4.1\nflask-swagger==0.2.13\nFlask-WTF==0.14.3\nfuncsigs==1.0.2\nfuture==0.18.2\ngraphviz==0.14\ngunicorn==19.10.0\nidna==2.9\nimportlib-metadata==1.6.0\niso8601==0.1.12\nitsdangerous==1.1.0\nJinja2==2.10.3\njson-merge-patch==0.2\njsonschema==3.2.0\nkombu==4.6.8\nlazy-object-proxy==1.4.3\nldap3==2.7\nlockfile==0.12.2\nMako==1.1.2\nMarkdown==2.6.11\nMarkupSafe==1.1.1\nmarshmallow==2.21.0\nmarshmallow-enum==1.5.1\nmarshmallow-sqlalchemy==0.23.0\nmysqlclient==1.4.6\nnumpy==1.18.4\npandas==0.25.3\npendulum==1.4.4\nprison==0.1.3\npsutil==5.7.0\npyasn1==0.4.8\npyasn1-modules==0.2.8\npycparser==2.20\nPygments==2.6.1\nPyJWT==1.7.1\nPyMySQL==0.9.3\npyrsistent==0.16.0\npython-daemon==2.1.2\npython-dateutil==2.8.1\npython-editor==1.0.4\npython-ldap==3.2.0\npython3-openid==3.1.0\npytz==2020.1\npytzdata==2019.3\nPyYAML==5.3.1\nrequests==2.23.0\nsetproctitle==1.1.10\nsix==1.14.0\nSQLAlchemy==1.3.17\nSQLAlchemy-JSONField==0.9.0\nSQLAlchemy-Utils==0.36.5\ntabulate==0.8.7\ntenacity==4.12.0\ntermcolor==1.1.0\ntext-unidecode==1.2\nthrift==0.13.0\ntyping-extensions==3.7.4.2\ntzlocal==1.5.1\nunicodecsv==0.14.1\nurllib3==1.25.9\nvine==1.3.0\nWerkzeug==0.16.1\nWTForms==2.3.1\nzipp==3.1.0\nzope.deprecation==4.4.0<\/code><\/pre>\n
\ud328\ud0a4\uc9c0 \uc124\uce58\uac00 \uc644\ub8cc\ub418\uba74, \ud30c\uc774\uc36c \ubc14\uc774\ub108\ub9ac \uacbd\ub85c\uc5d0 airflow \uc2a4\ud06c\ub9bd\ud2b8\uac00 \uc0dd\uc131\ub418\uba74\uc11c airflow<\/code> \uba85\ub839\uc744 \uc218\ud589\ud560 \uc218 \uc788\ub2e4. \ucd08\uae30 \uc124\uc815 \ud30c\uc77c\ub4e4\uc744 \ubd88\ub7ec\uc624\uae30 \uc704\ud574 initdb<\/code> \uc218\ud589\ud558\uba74<\/p>\n
airflow initdb<\/code><\/pre>\n
$AIRFLOW_HOME<\/code> \uacbd\ub85c\uc5d0 \ub85c\uadf8, sqlite db \uadf8\ub9ac\uace0 \uc124\uc815\ud30c\uc77c\uc774 \uc0dd\uc131\ub418\ub294\ub370 vim $AIRFLOW_HOME\/airflow.cfg<\/code> \ud30c\uc77c\uc5d0\uc11c \uc544\ub798 \ud56d\ubaa9\ub4e4\uc744 \uc218\uc815\ud55c\ub2e4.<\/p>\n
[core]\nexecutor = LocalExecutor\nsql_alchemy_conn = mysql+pymysql:\/\/airflow:airflow@localhost:3306\/airflow\n\n[webserver]\nauthenticate = True\nrbac = True<\/code><\/pre>\n
\ubcc0\uacbd\ub41c repo DB \ubc0f \ucd94\uac00 \uc124\uc815\ud30c\uc77c\uc744 \ub85c\ub4dc\ud558\uae30 \uc704\ud574 \ub2e4\uc2dc\ud55c\ubc88 initdb<\/code> \uc218\ud589\ud574\uc8fc\uba74<\/p>\n
airflow initdb<\/code><\/pre>\n
rbac \ud65c\uc131\ud654\ub85c FAB \uad00\ub828 \uc124\uc815 \ud30c\uc77c\uc774 \uc544\ub798\uc640 \uac19\uc774 \uc0dd\uc131\ub41c\ub2e4.<\/p>\n
vim $AIRFLOW_HOME\/webserver_config.py<\/code> \ud30c\uc77c\uc5d0 LDAP \uc815\ubcf4\ub97c \uc785\ub825\ud574\uc8fc\uace0<\/p>\n
# from flask_appbuilder.security.manager import AUTH_DB\nfrom flask_appbuilder.security.manager import AUTH_LDAP\n\n#AUTH_TYPE = AUTH_DB\nAUTH_TYPE = AUTH_LDAP\nAUTH_LDAP_SERVER = "ldap:\/\/centos:389"\nAUTH_LDAP_USE_TLS = False #\uac1c\ubc1c\ud658\uacbd\uc5d0 SSL \uc124\uc815\uc774 \uc548 \ub41c \uacbd\uc6b0; ldaps \uac00 \uc544\ub2cc ldap \ud504\ub85c\ud1a0\ucf5c\nAUTH_LDAP_SEARCH = "ou=People,dc=centos"\nAUTH_LDAP_BIND_USER = "uid=oboki,ou=People,dc=centos"\nAUTH_LDAP_BIND_PASSWORD = "oboki"<\/code><\/pre>\n
\ub2e4\uc74c \uba85\ub839\uc73c\ub85c \uc6f9\uc11c\ubc84\uc640 \uc2a4\ucf00\uc904\ub7ec\ub97c \uc2e4\ud589\ud558\uba74 \uae30\ubcf8\uc801\uc740 \uc124\uc815\uc740 \ub05d<\/p>\n
airflow webserver -D\nairflow scheduler<\/code><\/pre>\n
\uc6f9\uc11c\ubc84\uc5d0\uc11c \ub2e4\uc74c\uacfc \uac19\uc740 \ub85c\uadf8\uac00 \ub098\uc624\uba74<\/p>\n
{manager.py:710} WARNING - No user yet created, use flask fab command to do it.<\/code><\/pre>\n
AUTH_DB \uc640\ub294 \ub2e4\ub974\uac8c LDAP \uc2dc\uc2a4\ud15c\uacfc \uc5f0\ub3d9\ub41c \uc0ac\uc6a9\uc790\ub97c \ub4f1\ub85d\ud574\uc8fc\uc5b4\uc57c \ud558\ub294\ub370 \ud328\uc2a4\uc6cc\ub4dc\ub294 ldap \uc5d0\uc11c \uad00\ub9ac\ub97c \ud558\uae30\ub54c\ubb38\uc5d0 --use_random_password<\/code> \uc635\uc158\uc73c\ub85c \uc784\uc758\uc758 \ud328\uc2a4\uc6cc\ub4dc \uac12\uc744 \uc800\uc7a5\ud574\ub450\uba74 \ub41c\ub2e4. (\ud328\uc2a4\uc6cc\ub4dc\ub97c \uc9c0\uc815\ud558\ub354\ub77c\ub3c4 \uc0c1\uad00\uc5c6\uace0 \uc5b4\ucc28\ud53c \uc0ac\uc6a9\ud558\uc9c0 \uc54a\uc744 \uac12)<\/p>\n
airflow create_user -r Admin  -u oboki  -e oboki@example.com  -f oboki -l " " --use_random_password<\/code><\/pre>\n
Admin \uc0ac\uc6a9\uc790\ub9cc \ub4f1\ub85d\ud558\uba74 \uc774\ud6c4 \ub2e4\ub978 \uc0ac\uc6a9\uc790\ub4e4\uc740 Web UI \uc774\uc6a9\ud574\uc11c \uad00\ub9ac\ud560 \uc218 \uc788\ub2e4.<\/p>\n","protected":false},"excerpt":{"rendered":"
rbac \uae30\ub2a5\uc744 \uc774\uc6a9\ud558\uba74 DAG-level Access Control \uc774 \uac00\ub2a5\ud558\ub2e4\uace0 \ud55c\ub2e4. \uc0c8\ub86d\uac8c \ucd94\uac00\ub41c tag \uae30\ub2a5\uc5d0 \ub354\ud574\uc838\uc11c tag-level \uad8c\ud55c \uad00\ub9ac\ub3c4 \ub410\uc73c\uba74 \uc88b\uc558\uaca0\uc9c0\ub9cc.. rbac \uae30\ub2a5\uc744 \uc774\uc6a9\ud558\uba74 \uae30\ubcf8\uc801 \uc81c\uacf5\ub418\ub294 \ub864 \ubfd0\ub9cc \uc544\ub2c8\ub77c Customized role \uc744 \ud1b5\ud574 \uac00\ub2a5\ud574\uc9c8 \uc0ac\uc6a9\uc790 \uad8c\ud55c \uad00\ub9ac\ub97c \uae30\ub300\ud558\uba70 \ucd5c\uc2e0 1.10.10 \ubc84\uc804\uc744 \uc124\uce58\ud574\ubcf8\ub2e4. \uae30\uc874\uc5d0 LDAP \uc5f0\ub3d9\ud574\uc11c \uc0ac\uc6a9\ud558\ub358 Airflow \uc2dc\uc2a4\ud15c\uc744 \uc5c5\uadf8\ub808\uc774\ub4dc \ud574\uc11c \uc0ac\uc6a9\ud560 \uc218 \uc788\uc744\uc9c0 \ud655\uc778\ud574\ubcf4\uae30 \uc704\ud574 OpenLDAP \ud658\uacbd\uc744 […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[199,10],"tags":[],"class_list":["post-1544","post","type-post","status-publish","format-standard","hentry","category-airflow","category-python"],"_links":{"self":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/1544","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/comments?post=1544"}],"version-history":[{"count":0,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/1544\/revisions"}],"wp:attachment":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/media?parent=1544"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/categories?post=1544"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/tags?post=1544"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}