https:\/\/ims.tmaxsoft.com\/tody\/ims\/issue\/issueView.do?issueId<\/a>=<\/code>) \uc5d0 \ubcc0\uc218\ub85c \ub300\uc785\ud574\uc11c \uac01 \uc774\uc288 \ud398\uc774\uc9c0\uc5d0 \uc811\uadfc\ud560 \uc218 \uc788\ub2e4.)<\/p>\n_LOGIN_ID = "IMS_USERNAME"\n_LOGIN_PASS = "IMS_PASSWORD"\n_URL_LOGIN = "https:\/\/ims.tmaxsoft.com\/tody\/auth\/login.do"\n_URL_ISSVIEW = "https:\/\/ims.tmaxsoft.com\/tody\/ims\/issue\/issueView.do?issueId="\n_SLCT_LOGIN_INPUT = "body > form > table > tbody > tr > td > table > tbody > tr:nth-child(2) > td:nth-child(1) > table > tbody > tr > td:nth-child(2) > table > tbody > tr > td:nth-child(2) > table > tbody > tr > td:nth-child(3) > input"\n_SLCT_SUBJ = "body > div:nth-of-type(2) > table > tbody > tr > td:nth-of-type(2) > table > tbody > tr:nth-of-type(2) > td > table:nth-of-type(1) > tbody > tr > td > table > tbody > tr > td:nth-of-type(2)"\n_SLCT_DTLS = "#IssueDescriptionDiv > table > tbody > tr > td"\n_SLCT_IINFO = "#issueInfoTable"\n\ndef IndexIssueList(issueList):\n import re\n from selenium import webdriver\n from bs4 import BeautifulSoup\n\n driver = webdriver.PhantomJS('.\/webdriver\/phantomjs',\n service_log_path='.\/logs\/ghostdriver.log')\n driver.implicitly_wait(1)\n\n driver.get(_URL_LOGIN)\n driver.find_element_by_name('id').send_keys(_LOGIN_ID)\n driver.find_element_by_name('password').send_keys(_LOGIN_PASS)\n driver.find_element_by_css_selector(_SLCT_LOGIN_INPUT).click()\n\n iss = []\n for il in issueList:\n try:\n data = {}\n driver.get(_URL_ISSVIEW+str(il))\n html = driver.page_source\n soup = BeautifulSoup(html,'html.parser')\n\n subj = soup.select(_SLCT_SUBJ)[0].text.strip()\n data.update({'Subject':subj})\n\n iinfo = soup.select(_SLCT_IINFO)[0].findAll('tr')\n for b in iinfo:\n key,val = '',''\n for i in b.findAll('td',attrs={'class':re.compile(r'title|data')}):\n if i['class'][0] == 'title': key = i.text.strip()\n else: val = i.text.strip()\n data.update({key:re.sub('[\\n\\t ]+',' ',val)})\n\n if data['Closed Date'] == '':\n data['Closed Date']='1970\/01\/01 09:00:00'\n if data['Date of final order'] == '':\n data['Date of final order']='1970\/01\/01 09:00:00'\n\n dtls = soup.select(_SLCT_DTLS)\n data.update({'Issue Details':re.sub('[\\n\\t ]+',' ',dtls[0].text.strip())})\n\n comments,tmp = soup.findAll('div',attrs={'class':'commDescTR data'}),""\n for c in comments: tmp += c.text.strip()+'\\n'\n data.update({'Action Log':tmp})\n iss.append(data)\n except IndexError as error:\n continue\n\n driver.close()\n\n for i in range(len(iss)):\n iss[i].update({'Issue Number':re.search(r'\\d+',iss[i]['Issue Number']).group()})\n\n actions = [\n {\n "_index": "issue-v0.1.4",\n "_type": "_doc",\n "_id": iss[i]['Issue Number'],\n "_source": iss[i]\n }\n for i in range(len(iss))\n ]\n\n from elasticsearch import Elasticsearch\n from elasticsearch import helpers\n es_client = Elasticsearch(["localhost:9200"],timeout=300)\n res = helpers.bulk(es_client, actions)<\/code><\/pre>\n\uc704 \ucf54\ub4dc\uc5d0\uc11c driver.close()<\/code> \ubd80\ubd84\uae4c\uc9c0\uac00 IMS \uc6f9\uc5d0\uc11c \ub370\uc774\ud130\ub97c \uc218\uc9d1\ud558\ub294 \ubd80\ubd84\uc774\uace0, \uadf8 \uc544\ub798\uac00 elasticsearch \uc5d0 bulk indexing \uc73c\ub85c \ub370\uc774\ud130\ub97c \uc0c9\uc778\ud558\ub294 \ub2e8\uacc4\uc774\ub2e4. \uc218\uc9d1 \ub2e8\uacc4\uc5d0\uc11c\ub294 \uc218\uc9d1 \ub300\uc0c1 \uc6f9 \ud398\uc774\uc9c0\uc5d0 \ub530\ub77c \ub2e4\ub974\uac8c \uad6c\ud604\ub420 \uac83\uc774\uae30\ub54c\ubb38\uc5d0 \uadf8\ub54c\uadf8\ub54c \ub2ec\ub77c\uc9c8 \ubd80\ubd84\uc774\uc9c0\ub9cc \uc77c\ub2e8 \ud55c \ud398\uc774\uc9c0\uc758 \uc815\ubcf4\ub97c \ub515\uc154\ub108\ub9ac \ud615\ud0dc\ub85c \ub9cc\ub4e4\uace0 \uc774 \ub515\uc154\ub108\ub9ac\ub4e4\uc5d0 \ub300\ud55c \ub9ac\uc2a4\ud2b8\ub97c \ub9cc\ub4e4\uc5b4 \ub4a4\uc5d0\uc11c \uc21c\ud68c\ud560 \uc218 \uc788\ub3c4\ub85d \ud55c\ub2e4.<\/p>\n\ub2e4\uc74c\uacfc \uac19\uc774 \ubc8c\ud06c \uc791\uc5c5\uc744 \ud560 \uac83\ub4e4\uc744 \ud558\ub098\uc758 \ub9ac\uc2a4\ud2b8 actions \ub85c \ub9cc\ub4e4\uc5b4\uc8fc\uba74\ub418\ub294\ub370 _id<\/code> \ub294 Issue Number \uc640 \uac19\uac8c \ub9cc\ub4e4\uc5b4\uc11c \ucd94\ud6c4 \uc5c5\ub370\uc774\ud2b8 \uc791\uc5c5\uc774 \uc77c\uc5b4\ub0a0 \ub54c \ubc88\uac70\ub86d\uac8c id \ub97c \ucc3e\uc544\uc11c \uc5c5\ub370\uc774\ud2b8\ud558\uc9c0 \uc54a\uc544\ub3c4 \ub418\ub3c4\ub85d \ud55c\ub2e4.<\/p>\n actions = [\n {\n "_index": "issue-v0.1.4",\n "_type": "_doc",\n "_id": iss[i]['Issue Number'],\n "_source": iss[i]\n }\n for i in range(len(iss))\n ]<\/code><\/pre>\n\ucd08\uae30 \uc801\uc7ac \ubc0f \uc804\uccb4 \uc5c5\ub370\uc774\ud2b8<\/h2>\n
\uae30\uc874\uc5d0 \uc874\uc7ac \ud558\ub294 _id<\/code> \uc640 \uac19\uc740 \uac12\uc73c\ub85c \ubb38\uc11c\uac00 \uc0c9\uc778\ub418\uba74 update \ucc98\ub9ac\uac00 \ub418\uae30 \ub54c\ubb38\uc5d0 \ucd08\uae30 \uc801\uc7ac\ub294 \ubcc4\ub3c4\ub85c \ub9cc\ub4e4 \ud544\uc694\ub294 \uc5c6\uc774 updateAll \ud558\ub098\uc758 \ud568\uc218\ub85c \ub9cc\ub4e4\uc5c8\ub2e4. MaxFromIMS<\/code> \ud568\uc218\uc5d0\uc11c \ud604\uc7ac \uac00\uc7a5 \ucd5c\uadfc\uc5d0 \ub4f1\ub85d\ub41c IMS \ubc88\ud638\ub97c \uc54c\uc544\ub0b4\uace0 \uc774\ub97c 1\ubc88 \uc774\uc288\ubd80\ud130 _BATCH_SIZE<\/code> \ub2e8\uc704\ub85c \ub04a\uc5b4 \ub124 \uac1c\uc758 \uc4f0\ub808\ub4dc\uc5d0 \uc791\uc5c5\uc744 \ubd84\ubc30\uc2dc\ucf1c\uc900\ub2e4. \uac01 \uc4f0\ub808\ub4dc\ub294 \uc55e\uc11c \ub9cc\ub4e0 IndexIssueList \uc791\uc5c5\uc744 \uc218\ud589\ud558\uba70 _BATCH_SIZE \ud06c\uae30 \ub9cc\ud07c\uc758 \uc774\uc288 \uac2f\uc218\ub97c \uc0c9\uc778\ud558\uace0 \uc885\ub8cc\ub41c\ub2e4.<\/p>\n_URL_ISSLIST = "https:\/\/ims.tmaxsoft.com\/tody\/ims\/issue\/issueList.do"\n_SLCT_ISSLIST = "#IssueListForm"\n_BATCH_SIZE = 50\n_KEEP_ALIVE_LIMIT = '30s'\n\ndef MaxFromIMS():\n from selenium import webdriver\n from bs4 import BeautifulSoup\n\n driver = webdriver.PhantomJS('.\/webdriver\/phantomjs',\n service_log_path='.\/logs\/ghostdriver.log')\n driver.implicitly_wait(1)\n\n driver.get(_URL_LOGIN)\n driver.find_element_by_name('id').send_keys(_LOGIN_ID)\n driver.find_element_by_name('password').send_keys(_LOGIN_PASS)\n driver.find_element_by_css_selector(_SLCT_LOGIN_INPUT).click()\n\n driver.get(_URL_ISSLIST)\n html = driver.page_source\n soup = BeautifulSoup(html,'html.parser')\n\n max_iss = soup.select(_SLCT_ISSLIST)[0].findAll('table',attrs={'bgcolor':'#FFFFFF'})[0].findAll('tr')[1].findAll('td')[0].text.strip()\n\n driver.close()\n\n return int(max_iss)\n\ndef UpdateAll():\n import threading\n from time import sleep\n\n start = 1\n end = MaxFromIMS()\n jobs = list(range(start,end+1))\n while jobs:\n buf = []\n if threading.active_count() < 5:\n while len(buf) < _BATCH_SIZE:\n if len(jobs) == 0: break\n buf.append(jobs.pop(0))\n threading.Thread(target=IndexIssueList,args=(buf,)).start()\n sleep(1)<\/code><\/pre>\n\ubbf8\uc885\ub8cc \uc774\uc288\uc5d0 \ub300\ud55c \uc7ac\uc218\uc9d1<\/h2>\n
IMS \uc2dc\uc2a4\ud15c\uc740 \ucd08\ub2e8\uc704\uc758 \uac31\uc2e0 \uc791\uc5c5\uc774 \uc774\ub904\uc9c0\uc9c0\ub294 \uc54a\uace0 \uadf8\uc815\ub3c4\ub85c \uac80\uc0c9\uc5d4\uc9c4\uc5d0\uc11c \ubc18\uc601\ud574\uc904 \ud544\uc694\ub294 \uc5c6\uc9c0\ub9cc, \uc77c \ub2e8\uc704\ub85c \ubcf4\uba74 \uc81c\ubc95 \ub9ce\uc740 \uac31\uc2e0\uc774 \uc774\ub904\uc9c0\uace0 \uc788\ub2e4. \uc2dc\uc2dc\uac01\uac01 \ubcc0\ud558\ub294 \ub370\uc774\ud130\ub97c \uac80\uc0c9\uc5d4\uc9c4\uc5d0 \ucd5c\uc2e0\ud654 \ud558\uc5ec \ubc18\uc601\ud558\uae30 \uc704\ud574 \ud574\ub2f9 \uc2dc\uc2a4\ud15c\uc758 DB\uc5d0 \uc9c1\uc811 \uc811\uadfc\ud560 \uc218 \uc788\ub2e4\uba74 \uac00\uc7a5 \ud6a8\uc728\uc801\uc774\uaca0\uc9c0\ub9cc \uadf8\ub7f4 \uad8c\ud55c\uc740 \uc5c6\uae30\ub54c\ubb38\uc5d0 \ud558\ub8e8 \ud55c\ubc88 \uc815\ub3c4 \uac31\uc2e0\ub420 \uac00\ub2a5\uc131\uc774 \uc788\ub294 \ubb38\uc11c\ub4e4\uc744 \ub2e4\uc2dc \uc7ac\uc218\uc9d1\ud574\uc8fc\ub294 \uac83\uc73c\ub85c.. \uc774 \uc7ac\uc218\uc9d1 \ub300\uc0c1\uc744 \uc804\uccb4\ub85c \ud558\uae30\uc5d0\ub294 \ubd80\ub2f4\uc2a4\ub7fd\uae30 \ub54c\ubb38\uc5d0 \ucd5c\uc18c\ud654\ud558\uae30 \uc704\ud574 \uc885\ub8cc\ub418\uc9c0 \uc54a\uc740 \uc774\uc288\ub97c Scroll API \uc774\uc6a9\ud574\uc11c \ud30c\uc545\ud55c\ub2e4.<\/p>\n
def GetIssNumberNotClosed():\n body = { \n "_source":["Issue Number","Status"],\n "query" : { \n "bool":{\n "must_not":[\n {\n "term":{\n "Status.keyword":"Closed"\n }\n },\n {\n "term":{\n "Status.keyword":"Closed_P"\n }\n },\n {\n "term":{\n "Status.keyword":"Rejected"\n }\n },\n {\n "term":{\n "Status.keyword":"Rejected (\uc5c6\uc74c)"\n }\n },\n {\n "term":{\n "Status.keyword":"Prevented"\n }\n },\n {\n "term":{\n "Status.keyword":"Resolved"\n }\n }\n ]\n }\n },\n "size":10\n }\n\n from elasticsearch import Elasticsearch\n es_client = Elasticsearch(["localhost:9200"],timeout=300)\n response = es_client.search(\n index = 'issue-v0.1.4',\n scroll = _KEEP_ALIVE_LIMIT,\n size = 100,\n body = body\n )\n\n sid = response['_scroll_id']\n fetched = len(response['hits']['hits'])\n fcnt = fetched\n\n nums = []\n for i in range(fetched):\n nums.append(int(response['hits']['hits'][i]['_source']['Issue Number']))\n\n while(fetched>0):\n response = es_client.scroll(scroll_id=sid, scroll=_KEEP_ALIVE_LIMIT)\n sid = response['_scroll_id']\n fetched = len(response['hits']['hits'])\n for i in range(fetched):\n nums.append(int(response['hits']['hits'][i]['_source']['Issue Number']))\n\n return nums\n\ndef UpdateNotClosed():\n import threading\n jobs = GetIssNumberNotClosed()\n while jobs:\n buf = []\n if threading.active_count() < 5:\n while len(buf) < 50:\n if len(jobs) == 0: break\n buf.append(jobs.pop(0))\n threading.Thread(target=IndexIssueList,args=(buf,)).start()<\/code><\/pre>\n\uc0c8\ub85c\uc6b4 \uc774\uc288 \uc218\uc9d1<\/h2>\n
\uc0c8\ub85c\uc6b4 \uc774\uc288\ub294 \uc218\uc9d1\ub41c \uc774\uc288 \ubc88\ud638\uc758 \ucd5c\ub300 \uac12\uc5d0\uc11c\ubd80\ud130 IMS\uc5d0 \ub4f1\ub85d\ub41c \uc774\uc288\ubc88\ud638\uc758 \ucd5c\ub300\uac12\uae4c\uc9c0\uc778\ub370 MaxFromES \uc5d0\uc11c \uc218\uc9d1\ud55c \ucd5c\ub300 \ubc88\ud638\ub97c \ubc1b\uc544\ub0b4\uace0 \uc5ec\uae30\uc11c\ubd80\ud130 MaxFromIMS \uae4c\uc9c0 \uc218\uc9d1\ud558\ub294 Renew \ud568\uc218\ub97c \uc0dd\uac01\ud574\ubcfc \uc218 \uc788\ub2e4. \uc774\ub294 \uc2dc\uac04\ub2f9 \uba87 \uac1c\uc529\ub9cc \ub4f1\ub85d\ub418\uc5b4\uc11c \uc9e7\uc740 \uc8fc\uae30\ub85c \uc218\uc9d1\ud574\ub3c4 \ub420 \uac83 \uac19\ub2e4.<\/p>\n
def MaxFromES():\n from elasticsearch import Elasticsearch\n\n es = Elasticsearch("localhost:9200",timeout=10)\n body = {\n "size":0,\n "aggs" : {\n "max_iss" : { "max" : { "field" : "Issue Number" } }\n }\n }\n res = es.search(index="issue-v0.1.4", body=body)\n\n return int(res['aggregations']['max_iss']['value'])\n\ndef Renew():\n from_number = MaxFromES()+1\n to_number = MaxFromIMS()+1\n IndexIssueList(range(from_number,to_number))<\/code><\/pre>\n\uc2e4\ud589\ud30c\uc77c\ub85c \ub9cc\ub4e4\uae30 \uc704\ud574 \uba54\uc778 \ud568\uc218\uc5d0\uc11c\ub294 \uc544\ub798\uc640 \uac19\uc544 \ucee4\ub9e8\ub4dc\ub77c\uc778 \uc785\ub825\uc744 \ubc1b\uc544\uc11c \ub3cc\ub824\ubcfc \uc218 \uc788\ub2e4.<\/p>\n
#!\/Library\/Frameworks\/Python.framework\/Versions\/3.7\/bin\/python3\n\nif __name__ == '__main__':\n import sys\n mode = sys.argv[1]\n\n if mode == 'renew': Renew()\n elif mode == 'updateNotClosed': UpdateNotClosed()\n elif mode == 'updateAll': UpdateAll()\n elif mode == 'single': IndexIssueList([int(sys.argv[2])])\n elif mode == 'daemon':\n from datetime import datetime\n from time import sleep\n while True:\n sleep(60)\n now = datetime.now()\n if now.hour % 24 == 17 and now.minute % 60 == 30: updateNotClosed()\n if now.minute % 10 == 0: renew()<\/code><\/pre>\n\uc804\uccb4 \uc18c\uc2a4\ucf54\ub4dc<\/h2>\n
\uc704\uc5d0 \ubd80\ubd84\ub4e4 \ud569\uccd0\ub193\uc740 \uac83\uc5d0, IndexIssueList \ud568\uc218\uc5d0 print('try to crawl issue', il)<\/code> \ubd80\ubd84\ub9cc \ucd94\uac00\ud588\ub2e4. \ucd08\uae30 \uc801\uc7ac\uc2dc\uc5d0\ub294 .\/crawl.py updateAll<\/code> \uacfc \uac19\uc774 \uc2e4\ud589\ud574\uc8fc\uba74 \ub418\uace0 \ub098\uba38\uc9c0\ub3c4 \ube44\uc2b7\ud558\uac8c \uc0ac\uc6a9\ud558\uba74 \ub41c\ub2e4.<\/p>\n#!\/Library\/Frameworks\/Python.framework\/Versions\/3.7\/bin\/python3\n\n_LOGIN_ID = "id"\n_LOGIN_PASS = "pw"\n_URL_LOGIN = "https:\/\/ims.tmaxsoft.com\/tody\/auth\/login.do"\n_URL_ISSVIEW = "https:\/\/ims.tmaxsoft.com\/tody\/ims\/issue\/issueView.do?issueId="\n_SLCT_LOGIN_INPUT = "body > form > table > tbody > tr > td > table > tbody > tr:nth-child(2) > td:nth-child(1) > table > tbody > tr > td:nth-child(2) > table > tbody > tr > td:nth-child(2) > table > tbody > tr > td:nth-child(3) > input"\n_SLCT_SUBJ = "body > div:nth-of-type(2) > table > tbody > tr > td:nth-of-type(2) > table > tbody > tr:nth-of-type(2) > td > table:nth-of-type(1) > tbody > tr > td > table > tbody > tr > td:nth-of-type(2)"\n_SLCT_DTLS = "#IssueDescriptionDiv > table > tbody > tr > td"\n_SLCT_IINFO = "#issueInfoTable"\n\ndef IndexIssueList(issueList):\n import re\n from selenium import webdriver\n from bs4 import BeautifulSoup\n\n driver = webdriver.PhantomJS('.\/webdriver\/phantomjs',\n service_log_path='.\/logs\/ghostdriver.log')\n driver.implicitly_wait(1)\n\n driver.get(_URL_LOGIN)\n driver.find_element_by_name('id').send_keys(_LOGIN_ID)\n driver.find_element_by_name('password').send_keys(_LOGIN_PASS)\n driver.find_element_by_css_selector(_SLCT_LOGIN_INPUT).click()\n\n iss = []\n for il in issueList:\n print('try to crawl issue', il)\n try:\n data = {}\n driver.get(_URL_ISSVIEW+str(il))\n html = driver.page_source\n soup = BeautifulSoup(html,'html.parser')\n\n subj = soup.select(_SLCT_SUBJ)[0].text.strip()\n data.update({'Subject':subj})\n\n iinfo = soup.select(_SLCT_IINFO)[0].findAll('tr')\n for b in iinfo:\n key,val = '',''\n for i in b.findAll('td',attrs={'class':re.compile(r'title|data')}):\n if i['class'][0] == 'title': key = i.text.strip()\n else: val = i.text.strip()\n data.update({key:re.sub('[\\n\\t ]+',' ',val)})\n\n if data['Closed Date'] == '':\n data['Closed Date']='1970\/01\/01 09:00:00'\n if data['Date of final order'] == '':\n data['Date of final order']='1970\/01\/01 09:00:00'\n\n dtls = soup.select(_SLCT_DTLS)\n data.update({'Issue Details':re.sub('[\\n\\t ]+',' ',dtls[0].text.strip())})\n\n comments,tmp = soup.findAll('div',attrs={'class':'commDescTR data'}),""\n for c in comments: tmp += c.text.strip()+'\\n'\n data.update({'Action Log':tmp})\n iss.append(data)\n except IndexError as error:\n logger.info(error)\n continue\n\n driver.close()\n\n for i in range(len(iss)):\n iss[i].update({'Issue Number':re.search(r'\\d+',iss[i]['Issue Number']).group()})\n\n actions = [\n {\n "_index": "issue-v0.1.4",\n "_type": "_doc",\n "_id": iss[i]['Issue Number'],\n "_source": iss[i]\n }\n for i in range(len(iss))\n ]\n\n from elasticsearch import Elasticsearch\n from elasticsearch import helpers\n es_client = Elasticsearch(["localhost:9200"],timeout=300)\n res = helpers.bulk(es_client, actions)\n\n_URL_ISSLIST = "https:\/\/ims.tmaxsoft.com\/tody\/ims\/issue\/issueList.do"\n_SLCT_ISSLIST = "#IssueListForm"\n_BATCH_SIZE = 50\n_KEEP_ALIVE_LIMIT = '30s'\n\ndef MaxFromIMS():\n from selenium import webdriver\n from bs4 import BeautifulSoup\n\n driver = webdriver.PhantomJS('.\/webdriver\/phantomjs',\n service_log_path='.\/logs\/ghostdriver.log')\n driver.implicitly_wait(1)\n\n driver.get(_URL_LOGIN)\n driver.find_element_by_name('id').send_keys(_LOGIN_ID)\n driver.find_element_by_name('password').send_keys(_LOGIN_PASS)\n driver.find_element_by_css_selector(_SLCT_LOGIN_INPUT).click()\n\n driver.get(_URL_ISSLIST)\n html = driver.page_source\n soup = BeautifulSoup(html,'html.parser')\n\n max_iss = soup.select(_SLCT_ISSLIST)[0].findAll('table',attrs={'bgcolor':'#FFFFFF'})[0].findAll('tr')[1].findAll('td')[0].text.strip()\n\n driver.close()\n\n return int(max_iss)\n\ndef UpdateAll():\n import threading\n from time import sleep\n\n start = 1\n end = MaxFromIMS()\n jobs = list(range(start,end+1))\n while jobs:\n buf = []\n if threading.active_count() < 5:\n while len(buf) < _BATCH_SIZE:\n if len(jobs) == 0: break\n buf.append(jobs.pop(0))\n threading.Thread(target=IndexIssueList,args=(buf,)).start()\n sleep(1)\n\ndef GetIssNumberNotClosed():\n body = { \n "_source":["Issue Number","Status"],\n "query" : { \n "bool":{\n "must_not":[\n {\n "term":{\n "Status.keyword":"Closed"\n }\n },\n {\n "term":{\n "Status.keyword":"Closed_P"\n }\n },\n {\n "term":{\n "Status.keyword":"Rejected"\n }\n },\n {\n "term":{\n "Status.keyword":"Rejected (\uc5c6\uc74c)"\n }\n },\n {\n "term":{\n "Status.keyword":"Prevented"\n }\n },\n {\n "term":{\n "Status.keyword":"Resolved"\n }\n }\n ]\n }\n },\n "size":10\n }\n\n from elasticsearch import Elasticsearch\n es_client = Elasticsearch(["localhost:9200"],timeout=300)\n response = es_client.search(\n index = 'issue-v0.1.4',\n scroll = _KEEP_ALIVE_LIMIT,\n size = 100,\n body = body\n )\n\n sid = response['_scroll_id']\n fetched = len(response['hits']['hits'])\n fcnt = fetched\n\n nums = []\n for i in range(fetched):\n nums.append(int(response['hits']['hits'][i]['_source']['Issue Number']))\n\n while(fetched>0):\n response = es_client.scroll(scroll_id=sid, scroll=_KEEP_ALIVE_LIMIT)\n sid = response['_scroll_id']\n fetched = len(response['hits']['hits'])\n for i in range(fetched):\n nums.append(int(response['hits']['hits'][i]['_source']['Issue Number']))\n\n return nums\n\ndef UpdateNotClosed():\n import threading\n jobs = GetIssNumberNotClosed()\n while jobs:\n buf = []\n if threading.active_count() < 5:\n while len(buf) < 50:\n if len(jobs) == 0: break\n buf.append(jobs.pop(0))\n threading.Thread(target=IndexIssueList,args=(buf,)).start()\n\ndef MaxFromES():\n from elasticsearch import Elasticsearch\n\n es = Elasticsearch("localhost:9200",timeout=10)\n body = {\n "size":0,\n "aggs" : {\n "max_iss" : { "max" : { "field" : "Issue Number" } }\n }\n }\n res = es.search(index="issue-v0.1.4", body=body)\n\n return int(res['aggregations']['max_iss']['value'])\n\ndef Renew():\n from_number = MaxFromES()+1\n to_number = MaxFromIMS()+1\n IndexIssueList(range(from_number,to_number))\n\nif __name__ == '__main__':\n import sys\n mode = sys.argv[1]\n\n if mode == 'renew': Renew()\n elif mode == 'updateNotClosed': UpdateNotClosed()\n elif mode == 'updateAll': UpdateAll()\n elif mode == 'single': IndexIssueList([int(sys.argv[2])])\n elif mode == 'daemon':\n from datetime import datetime\n from time import sleep\n while True:\n sleep(60)\n now = datetime.now()\n if now.hour % 24 == 17 and now.minute % 60 == 30: updateNotClosed()\n if now.minute % 10 == 0: renew()<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"[ElasticSearch] \uac80\uc0c9\uc5d4\uc9c4 \ub9cc\ub4e4\uae30 2 – IMS \ub370\uc774\ud130 \ud06c\ub864\ub9c1 Selenium, BeautifulSoup \ub97c \uc774\uc6a9\ud574\uc11c IMS \uc774\uc288 \ud398\uc774\uc9c0 Crawler \ub97c \ub9cc\ub4e0\ub2e4. \uae30\ucd08\uc801\uc778 \ubd80\ubd84\uc740 [\uc5ec\uae30]()\ub97c \ucc38\uace0. \ub2e8\uc704 \ud06c\ub864\ub7ec \uc218\uc9d1\ud560 \uc774\uc288 \ubc88\ud638\uc5d0 \ub300\ud55c \ub9ac\uc2a4\ud2b8\ub97c \ub9cc\ub4e4\uc5b4 \ud55c\ubc88\uc5d0 \uc218\uc9d1\ud558\uace0 Elasticsearch\uc5d0 \uc778\ub371\uc2f1 \ud558\ub294 IndexIssueList \ud568\uc218\ub97c \ub9cc\ub4e4\uc5c8\ub2e4. (IMS \ub294 \uc774\uc288 \ubc88\ud638\ub97c URL(https:\/\/ims.tmaxsoft.com\/tody\/ims\/issue\/issueView.do?issueId=) \uc5d0 \ubcc0\uc218\ub85c \ub300\uc785\ud574\uc11c \uac01 \uc774\uc288 \ud398\uc774\uc9c0\uc5d0 \uc811\uadfc\ud560 \uc218 \uc788\ub2e4.) _LOGIN_ID = "IMS_USERNAME" […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[12,10],"tags":[],"class_list":["post-1134","post","type-post","status-publish","format-standard","hentry","category-elasticsearch","category-python"],"_links":{"self":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/1134","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/comments?post=1134"}],"version-history":[{"count":5,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/1134\/revisions"}],"predecessor-version":[{"id":1151,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/1134\/revisions\/1151"}],"wp:attachment":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/media?parent=1134"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/categories?post=1134"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/tags?post=1134"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}