{"id":366,"date":"2018-08-22T12:37:39","date_gmt":"2018-08-22T03:37:39","guid":{"rendered":"http:\/\/dong1lkim.oboki.net\/?p=366"},"modified":"2019-09-01T22:21:05","modified_gmt":"2019-09-01T13:21:05","slug":"fluentd-usecase-tail","status":"publish","type":"post","link":"https:\/\/oboki.net\/workspace\/data-engineering\/fluentd\/fluentd-usecase-tail\/","title":{"rendered":"[Fluentd] Usecase &#8211; tail"},"content":{"rendered":"<h1>Fluentd Use Case<\/h1>\n<h2>tail \ud50c\ub7ec\uadf8\uc778 \ud65c\uc6a9\ud558\uae30<\/h2>\n<blockquote><p>\n  in_tail \ud50c\ub7ec\uadf8\uc778\uc740 \ud14d\uc2a4\ud2b8 \ud30c\uc77c\uc744 \uc77d\uc5b4 \uc218\uc9d1\ud560 \uc218 \uc788\ub294 \uae30\ubcf8 \ud50c\ub7ec\uadf8\uc778\uc774\uba70, \ub85c\uadf8\ud30c\uc77c\uacfc \uac19\uc774 \uac31\uc2e0\ub418\ub294 \ud30c\uc77c\uc744 \uc99d\ubd84\ub9cc \uacc4\uc18d\ud574\uc11c \uc218\uc9d1\ud560 \uc218\ub3c4 \uc788\ub2e4.\n<\/p><\/blockquote>\n<h3>csv \ub370\uc774\ud130<\/h3>\n<p>\ubd84\uc11d \ub370\uc774\ud130\uc14b\uc774\ub098 \uae30\ucd08 \ub370\uc774\ud130\uc14b\uacfc \uac19\uc774 \uc77c\ud68c\uc131 \ub610\ub294 \ubc30\uce58\uc131\uc73c\ub85c \uc218\uc9d1\ud558\ub294 csv \ud14d\uc2a4\ud2b8 \ud30c\uc77c\uc744 \uc218\uc9d1\ud558\ub294 \ubc29\ubc95\uc774\ub2e4.<\/p>\n<h4>\uc218\uc9d1 \ub370\uc774\ud130 \uc18c\uc2a4<\/h4>\n<p><code>head -n 5 accidentology.csv<\/code><\/p>\n<pre><code class=\"csv\">timestamp,Date,Hour,Dept,Com,Address,Zip code,Corner,Segment,Address1,Address2,Vehicle 1 description,Vehicle 1,Vehicle 2 Description,Vehicle 2,Vehicle 3 Description,Vehicle 3,Person 1 Category,Person 1 Status,Person Tag,Person 2 Category,Person 2 Status,Person 2 Tag,Person 3 Cat,Person 3 Status,Person 3 Tag,Person 4 Cat,Person 4 Status,Person 4 Tag,fullAddress,latitude,longitude,season,involvedCount,periodOfDay\n16\/10\/2012 10:15,16\/10\/2012,10:15,75,101,\"1, PLACE DU CHATELET\",,1_75101_6801,,\"1, PLACE DU CHATELET, 75001 Paris\",AVENUE VICTORIA,Motorbike,RESPONSIBLE,,,,,Cond,Injured,RESPONSIBLE,,,,,,,,,,\"1, PLACE DU CHATELET, 75001 Paris\",48.8575996,2.3467545,autumn,1,morning\n15\/03\/2012 17:10,15\/03\/2012,17:10,75,101,\"41, RUE DE RIVOLI\",,1_75101_6850,,\"41, RUE DE RIVOLI, 75001 Paris\",BOULEVARD DE SEBASTOPOL,Bicycle,RESPONSIBLE,,,,,Cond,Injured,RESPONSIBLE,Troll,Safe,RESPONSIBLE,,,,,,,\"41, RUE DE RIVOLI, 75001 Paris\",48.85842,2.3477402,winter,1,afternoon\n19\/04\/2013 11:10,19\/04\/2013,11:10,75,101,\"41, RUE DE RIVOLI\",,1_75101_6850,,\"41, RUE DE RIVOLI, 75001 Paris\",BOULEVARD DE SEBASTOPOL,Van,RESPONSIBLE,,,,,Cond,Safe,RESPONSIBLE,Pedestrian,Injured,RESPONSIBLE,,,,,,,\"41, RUE DE RIVOLI, 75001 Paris\",48.85842,2.3477402,spring,1,morning\n12\/01\/2012 22:30,12\/01\/2012,22:30,75,101,\"19, RUE ETIENNE MARCEL\",,1_75101_6945,,\"19, RUE ETIENNE MARCEL, 75001 Paris\",RUE DE TURBIGO,Motor Scooter,RESPONSIBLE,Motor Scooter,VICTIM,,,Cond,Safe,RESPONSIBLE,Cond,Safe,VICTIM,Pedestrian,Injured,RESPONSIBLE,,,,\"19, RUE ETIENNE MARCEL, 75001 Paris\",48.8638538,2.3483359,winter,2,evening\n<\/code><\/pre>\n<h4>fluentd config<\/h4>\n<p>csv \ud14d\uc2a4\ud2b8 \ud30c\uc77c\uc744 \uc77d\uc5b4\ub4e4\uc778 \ub4a4, csv paser \ub97c \uc774\uc6a9\ud574 \ucd5c\ucd08 \ud30c\uc2f1\uc744 \uc218\ud589\ud55c\ub2e4. \uc774\ud6c4 longitude,latitude \uceec\ub7fc\uc744 location \uc774\ub77c\ub294 \ud544\ub4dc\ub85c \ubcc0\ud615\ud55c \ub4a4 elasticsearch \uc5d0 \uc800\uc7a5\ud55c\ub2e4.<\/p>\n<pre><code class=\"xml\"># CSV, accidentology\n## Input\n&lt;source&gt;\n  @type tail\n  path \/home\/fluentd\/accident*\n  read_from_head true\n  pos_file \/data\/fluentd\/pos\/accident.pos\n  tag accident\n  &lt;parse&gt;\n    @type csv \n    keys timestamp,Date,Hour,Dept,Com,Address,Zip code,Corner,Segment,Address1,Address2,Vehicle 1 description,Vehicle 1,Vehicle 2 Description,Vehicle 2,Vehicle 3 Description,Vehicle 3,Person 1 Category,Person 1 Status,Person Tag,Person 2 Category,Person 2 Status,Person 2 Tag,Person 3 Cat,Person 3 Status,Person 3 Tag,Person 4 Cat,Person 4 Status,Person 4 Tag,fullAddress,latitude,longitude,season,involvedCount,periodOfDay\n    types latitude:float,longitude:float\n    time_key timestamp\n    time_format %d\/%m\/%Y %H:%M\n  &lt;\/parse&gt;\n&lt;\/source&gt;\n\n## Filter\n&lt;filter accident*.**&gt;\n  @type record_transformer\n  enable_ruby\n  &lt;record&gt;\n    location ${record[\"latitude\"]},${record[\"longitude\"]}\n  &lt;\/record&gt;\n  remove_keys longitude,latitude\n&lt;\/filter&gt;\n\n## Output\n&lt;match accident*.**&gt;\n  @type copy\n  &lt;store&gt;\n    @type elasticsearch\n    hosts 192.168.179.81:9200,192.168.179.82:9200,\n    type_name accident\n    logstash_format true\n    logstash_prefix accidentology\n    logstash_dateformat %Y\n    include_tag_key true\n    tag_key @log_name\n    flush_interval 10s \n  &lt;\/store&gt;\n  &lt;store&gt;\n    @type stdout\n  &lt;\/store&gt;\n&lt;\/match&gt;\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>Fluentd Use Case tail \ud50c\ub7ec\uadf8\uc778 \ud65c\uc6a9\ud558\uae30 in_tail \ud50c\ub7ec\uadf8\uc778\uc740 \ud14d\uc2a4\ud2b8 \ud30c\uc77c\uc744 \uc77d\uc5b4 \uc218\uc9d1\ud560 \uc218 \uc788\ub294 \uae30\ubcf8 \ud50c\ub7ec\uadf8\uc778\uc774\uba70, \ub85c\uadf8\ud30c\uc77c\uacfc \uac19\uc774 \uac31\uc2e0\ub418\ub294 \ud30c\uc77c\uc744 \uc99d\ubd84\ub9cc \uacc4\uc18d\ud574\uc11c \uc218\uc9d1\ud560 \uc218\ub3c4 \uc788\ub2e4. csv \ub370\uc774\ud130 \ubd84\uc11d \ub370\uc774\ud130\uc14b\uc774\ub098 \uae30\ucd08 \ub370\uc774\ud130\uc14b\uacfc \uac19\uc774 \uc77c\ud68c\uc131 \ub610\ub294 \ubc30\uce58\uc131\uc73c\ub85c \uc218\uc9d1\ud558\ub294 csv \ud14d\uc2a4\ud2b8 \ud30c\uc77c\uc744 \uc218\uc9d1\ud558\ub294 \ubc29\ubc95\uc774\ub2e4. \uc218\uc9d1 \ub370\uc774\ud130 \uc18c\uc2a4 head -n 5 accidentology.csv timestamp,Date,Hour,Dept,Com,Address,Zip code,Corner,Segment,Address1,Address2,Vehicle 1 description,Vehicle 1,Vehicle 2 Description,Vehicle [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[11],"tags":[],"class_list":["post-366","post","type-post","status-publish","format-standard","hentry","category-fluentd"],"_links":{"self":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/366","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/comments?post=366"}],"version-history":[{"count":5,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/366\/revisions"}],"predecessor-version":[{"id":1244,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/posts\/366\/revisions\/1244"}],"wp:attachment":[{"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/media?parent=366"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/categories?post=366"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/oboki.net\/workspace\/wp-json\/wp\/v2\/tags?post=366"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}