{"id":368,"date":"2018-08-22T12:37:58","date_gmt":"2018-08-22T03:37:58","guid":{"rendered":"http:\/\/dong1lkim.oboki.net\/?p=368"},"modified":"2019-09-01T22:21:05","modified_gmt":"2019-09-01T13:21:05","slug":"fluentd-usecase-http","status":"publish","type":"post","link":"https:\/\/oboki.net\/workspace\/data-engineering\/fluentd\/fluentd-usecase-http\/","title":{"rendered":"[Fluentd] Usecase &#8211; http"},"content":{"rendered":"<h1>Fluentd Use Case<\/h1>\n<h2>http \ud50c\ub7ec\uadf8\uc778 \ud65c\uc6a9\ud558\uae30<\/h2>\n<blockquote><p>\n  \uc9c0\uc815\ud55c Listener Port \ub97c \ud1b5\ud574 POST \ub418\ub294 \ub370\uc774\ud130\ub97c \uc218\uc9d1\ud55c\ub2e4.\n<\/p><\/blockquote>\n<h3>openAPI<\/h3>\n<p>openAPI \ub97c \uc774\uc6a9\ud574 REST \ud558\uac8c \ubc1b\uc544\uc628 \ub370\uc774\ud130\ub97c \ub2e4\uc2dc Fluentd http listener \uc5d0 \uc804\ub2ec\ud574\uc90c\uc73c\ub85c\uc368 \ub370\uc774\ud130\ub97c \uc218\uc9d1\ud55c\ub2e4.<\/p>\n<h4>\ub370\uc774\ud130 \uc18c\uc2a4<\/h4>\n<p><code>vi \/app\/python\/sources\/JayWalkingAll.py<\/code><\/p>\n<pre><code class=\"python\">JayWalkingAll\n#!\/app\/python\/bin\/python\n\nfrom urllib.request import urlopen\nfrom urllib.parse import urlencode,quote_plus,unquote\nimport urllib\nimport requests\nimport json\n\nurl = 'http:\/\/apis.data.go.kr\/B552061\/jaywalking\/getRestJaywalking'\nmy_service_key = 'some_key_value_here'\nycd = [2015052]\nscd = [50]\n#ycd = [2017081\n#      ,2016052\n#      ,2015052\n#      ,2014117\n#      ,2013114]\n#scd = [11\n#      ,26\n#      ,27\n#      ,28\n#      ,29\n#      ,30\n#      ,31\n#      ,36\n#      ,41\n#      ,42\n#      ,43\n#      ,44\n#      ,45\n#      ,46\n#      ,47\n#      ,48\n#      ,50]\n\nfor i in range (len(ycd)):\n    for j in range (len(scd)):\n        #print (ycd[i],scd[j])\n        queryParams = '?' + urlencode({ quote_plus('servicekey') : my_service_key,\n            quote_plus('searchYearCd') : ycd[i],\n            quote_plus('siDo') : scd[j]\n        })\n        request = urllib.request.Request(url+unquote(queryParams))\n        #print ('Your Request:\\n'+url+unquote(queryParams))\n        request.get_method = lambda: 'GET'\n        response_body = urlopen(request).read()\n        print (response_body.decode('utf-8'))\n\n        #print (type(response_body))\n        #print (type(str(response_body)))\n        #print ((str(response_body.decode('utf-8'))))\n\n        headers = {'Content-type': 'application\/json'}\n        #req = requests.request('PUT','http:\/\/localhost:47474\/JayWalking',json=json.dumps(str(response_body.decode('utf-8'))))\n        import ast\n        json_response_body = ast.literal_eval(str(response_body.decode('utf-8')))\n        for i in range(len(json_response_body['searchResult']['frequentzone'])):\n            data_all = json_response_body['searchResult']['frequentzone'][i]\n            coordinates = list(ast.literal_eval(data_all['geometry'])['coordinates'][0])\n            for j in range(len(coordinates)):\n                data = data_all\n                latitude,longitude = (str(coordinates[j]).replace('[','').replace(']','').replace(' ','')).split(',')\n                print (latitude)\n                print (longitude)\n                #data.update({'location':str(coordinates[j]).replace('[','').replace(']','').replace(' ','')})\n                data.update({'latitude':latitude})\n                data.update({'longitude':longitude})\n                data.update({'location_id':j})\n                print(data)\n                req = requests.request('PUT','http:\/\/localhost:47474\/JayWalking',json=data)\n                #data = {'auth_token': 'auth1', 'widget': 'id1', 'title': 'Something1', 'text': 'Some text', 'moreinfo': 'Subtitle'}\n                print(req.json)\n<\/code><\/pre>\n<h4>fluentd config<\/h4>\n<pre><code class=\"xml\"># Http Input &amp; Stdout Output\n## Input\n&lt;source&gt;\n  @type http\n  port 47474\n  &lt;parse&gt;\n    @type json\n    types latitude:float,longitude:float\n  &lt;\/parse&gt;\n&lt;\/source&gt;\n\n## Filter\n&lt;filter **JayWalking*.**&gt;\n  @type record_transformer\n  enable_ruby\n  &lt;record&gt;\n    location ${record[\"longitude\"]},${record[\"latitude\"]}\n  &lt;\/record&gt;\n  remove_keys longitude,latitude\n&lt;\/filter&gt;\n\n## Output\n&lt;match **JayWalking**&gt;\n  @type copy\n  &lt;store&gt;\n    @type elasticsearch\n    hosts 192.168.179.81:9200,192.168.179.82:9200,\n    logstash_format true\n    logstash_prefix jaywalking\n    logstash_dateformat %Y%m\n    include_tag_key true\n    tag_key @log_name\n    flush_interval 10s \n  &lt;\/store&gt;\n  &lt;store&gt;\n    @type stdout\n  &lt;\/store&gt;\n&lt;\/match&gt;\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>Fluentd Use Case http \ud50c\ub7ec\uadf8\uc778 \ud65c\uc6a9\ud558\uae30 \uc9c0\uc815\ud55c Listener Port \ub97c \ud1b5\ud574 POST \ub418\ub294 \ub370\uc774\ud130\ub97c \uc218\uc9d1\ud55c\ub2e4. openAPI openAPI \ub97c \uc774\uc6a9\ud574 REST \ud558\uac8c \ubc1b\uc544\uc628 \ub370\uc774\ud130\ub97c \ub2e4\uc2dc Fluentd http listener \uc5d0 \uc804\ub2ec\ud574\uc90c\uc73c\ub85c\uc368 \ub370\uc774\ud130\ub97c \uc218\uc9d1\ud55c\ub2e4. \ub370\uc774\ud130 \uc18c\uc2a4 vi \/app\/python\/sources\/JayWalkingAll.py JayWalkingAll #!\/app\/python\/bin\/python from urllib.request import urlopen from urllib.parse import urlencode,quote_plus,unquote import urllib import requests import json url = &#8216;http:\/\/apis.data.go.kr\/B552061\/jaywalking\/getRestJaywalking&#8217; my_service_key [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[11],"tags":[142],"class_list":["post-368","post","type-post","status-publish","format-standard","hentry","category-fluentd","tag-fluented"],"_links":{"self":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/368","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/comments?post=368"}],"version-history":[{"count":6,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/368\/revisions"}],"predecessor-version":[{"id":1243,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/368\/revisions\/1243"}],"wp:attachment":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/media?parent=368"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/categories?post=368"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/tags?post=368"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}