nginx通路日志 logstash 配置檔案執行個體2

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

<code>日志格式：</code>

<code>log_format elk "$http_clientip | $http_x_forwarded_for | $time_local | $request | $status | $body_bytes_sent | "</code>

<code>                  </code><code>" $request_body | $content_length | $http_referer | $http_user_agent | "</code>

<code>                  </code><code>"$http_cookie | $remote_addr | $hostname | $upstream_addr | $upstream_response_time | $request_time";</code>

<code>日志執行個體：</code>

<code>36.110.211.42 | 10.10.130.101 | 23/Jun/2017:17:51:01 +0800 | GET /lvyou/dongjing/ HTTP/1.1 | 200 | 73181 | - | 0 | - | Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Safari/537.36 | JSESSION_O2O=0000F0jMLw1MnT6SFCvhcqW3oP9:19h7oe5dr; SessionID=10.10.130.101.1498210484146456; sCityCode=SZX; sCityName=%E6%B7%B1%E5%9C%B3; vac_ss_sid=4441002; vac_ss_uid=4441 | 10.10.130.100 | www1-n01 | 10.10.130.237:80 | 0.808 | 0.809</code>

<code>logstash執行個體：</code>

<code>input {  </code>

<code>        </code><code>file {  </code>

<code>                </code><code>type =&gt; "www1_access"  </code>

<code>                </code><code>path =&gt; ["/usr/local/elk/elklog/nginxlog/log0/www1.log","/usr/local/elk/elklog/nginxlog/log1/www1.log"]  </code>

<code>        </code><code>}</code>

<code>        </code><code>file {</code>

<code>                </code><code>type =&gt; "flight1_access"</code>

<code>                </code><code>path =&gt; ["/usr/local/elk/elklog/nginxlog/log0/flight1.log","/usr/local/elk/elklog/nginxlog/log1/flight1.log"]</code>

<code>                </code><code>type =&gt; "m_access"</code>

<code>                </code><code>path =&gt; ["/usr/local/elk/elklog/nginxlog/log0/m.log"]</code>

<code>}  </code>

<code>filter {</code>

<code>ruby {</code>

<code>init =&gt; "@kname = ['http_clientip','http_x_forwarded_for','time_local','request','status','body_bytes_sent','request_body','content_length','http_referer','http_user_agent','http_cookie','remote_addr','hostname','upstream_addr','upstream_response_time','request_time']"</code>

<code>code =&gt; "new_event = LogStash::Event.new(Hash[@kname.zip(event.get('message').split(' | '))])</code>

<code>new_event.remove('@timestamp')</code>

<code>event.append(new_event)"</code>

<code>}</code>

<code>if [request] {</code>

<code>init =&gt; "@kname = ['method','uri','verb']"</code>

<code>code =&gt; "new_event = LogStash::Event.new(Hash[@kname.zip(event.get('request').split(' '))])</code>

<code>event.append(new_event)</code>

<code>"</code>

<code>if [uri] {</code>

<code>init =&gt; "@kname = ['url_path','url_args']"</code>

<code>code =&gt; "new_event = LogStash::Event.new(Hash[@kname.zip(event.get('uri').split('?'))])</code>

<code>kv {</code>

<code>prefix =&gt; "url_"</code>

<code>source =&gt; "url_args"</code>

<code>field_split =&gt; "&amp; "</code>

<code>remove_field =&gt; [ "url_args","uri","request" ]</code>

<code>mutate {</code>

<code>convert =&gt; ["body_bytes_sent" , "integer", "content_length", "integer", "upstream_response_time", "float","request_time", "float"]</code>

<code>        </code><code>grok {</code>

<code>              </code><code>match =&gt; [ </code>

<code>"message", "%{IP:clientip} \| %{USER} \| %{HTTPDATE:timestamp}"</code>

<code> </code><code>]</code>

<code>date {</code>

<code>match =&gt; [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]</code>

<code>locale =&gt; "en"</code>

<code>        </code><code>geoip </code>

<code>{</code>

<code>        </code><code>source =&gt; "clientip"</code>

<code>mutate {    </code>

<code>                </code><code>remove_field =&gt; "timestamp"      </code>

<code>                </code><code>remove_field =&gt; "http_clientip"      </code>

<code>useragent {</code>

<code>    </code><code>source =&gt; "http_user_agent"</code>

<code>    </code><code>target =&gt; "useragent"</code>

<code>  </code><code>} </code>

<code>output {</code>

<code>        </code><code>redis {  </code>

<code>                </code><code>host =&gt; "10.10.45.200"  </code>

<code>                </code><code>data_type =&gt; "list"  </code>

<code>                </code><code>key =&gt; "elk_frontend_access:redis"  </code>

<code>                </code><code>port=&gt;"5379"  </code>

<code>        </code><code>}  </code>

<code>注意：分隔符為“空格+table+空格”</code>