exchange日志收集

将exchange伺服器日志收集到logstash中，exchange日志包括AD域控伺服器日志（主要收集登入日志）和exchange服務的transport日志和iis日志

AD域日志收集

在AD域控伺服器上安裝winlogbeat，配置檔案如下：

#======================= Winlogbeat specific options ===========================
# The supported keys are name (required), tags, fields, fields_under_root

winlogbeat.event_logs:
  - name: Application
    tags: ["winlog"]
    fields_under_root: true
    ignore_older: 72h
    fields:
      log_type: winlog_application
    

  - name: System
    tags: ["winlog"]
    fields_under_root: true
    ignore_older: 72h
    fields:
      log_type: winlog_system


  - name: Security
    tags: ["winlog"]
    fields_under_root: true
    ignore_older: 72h
    fields:
      log_type: winlog_security
    processors:
      - script:
          lang: javascript
          id: security
          file: ${path.home}/module/security/config/winlogbeat-security.js


#----------------------------- Logstash output --------------------------------
output.logstash:
  # The Logstash hosts
  hosts: ["192.168.100.1:15044"] #logstash服務的IP位址，安裝前請修改此IP
  
  # 若有多個logstash執行個體，可在hosts中添加多個IP，中間用逗号分隔
  # 如：["192.168.100.1:15044","192.168.100.2:15044"],并将loadbalance設定為true
  # loadbalance: true
  
#----------------------------- kafka output --------------------------------
#output.kafka:
#  hosts: ["127.0.0.1:9092"]
#  topic: 'ri-%{[log_type]}'
#  required_acks: 1

#================================ Processors =====================================
# Configure processors to enhance or manipulate events generated by the beat.
processors:
  - add_host_metadata: 
      netinfo.enabled: true
  - add_cloud_metadata: ~
           

logstash中配置如下：

input {
    beats { 
        port => "15044"
    }
}

filter {
    if "winlog" in [tags] {
        mutate {
            replace => [ 
                "agent_id","%{[agent][id]}",
                "agent_version","%{[agent][type]}-%{[agent][version]}",
                "hostname","%{[agent][hostname]}",
                "level","%{[log][level]}",
                "os_kernel","%{[host][os][kernel]}",
                "os_platform","%{[host][os][platform]}",
                "os_name","%{[host][os][name]} %{[host][os][version]}",
                "os_mac","%{[host][mac]}",
                "os_ip","%{[host][ip]}" 
            ]       
            remove_field =>["host","ecs","event","log","@version","tags","agent"]    
        }
    }         
}

output {
    if [log_type] == "winlog_security" {
        #stdout{}
        udp {
             host => "127.0.0.1"
             port => "15004"
        }
    }
}
           

exchange郵件服務transport日志和iis日志收集

在exchange郵件伺服器上安裝winfilebeat插件，配置檔案如下：

#=========================== Filebeat inputs =============================

filebeat.inputs:

- type: log

  enabled: true
  paths:
    - C:\Program Files\Microsoft\Exchange Server\V15\TransportRoles\Logs\MessageTracking\MSGTRK2*.LOG
  exclude_lines: ['^#']
  #multiline.pattern: '^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z,'
  #multiline.negate: true
  #multiline.match: after
  #multiline.timeout: 10s
  tags: ["exchange"]
  fields_under_root: true
  fields:
    log_type: mail_transportlog
  ignore_older: 72h
  tail_files: true
  close_inactive: 1m
  close_timeout: 3h
  clean_inactive: 75h
  
  
- type: log

  enabled: true
  paths:
    - C:\inetpub\logs\LogFiles\*\*.log
  exclude_lines: ['^#']
  #multiline.pattern: '^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z,'
  #multiline.negate: true
  #multiline.match: after
  #multiline.timeout: 10s
  tags: ["exchange"]
  fields_under_root: true
  fields:
    log_type: mail_iislog
  ignore_older: 72h
  tail_files: true
  close_inactive: 1m
  close_timeout: 3h
  clean_inactive: 75h


#============================= Filebeat modules ===============================

filebeat.config.modules:
  # Glob pattern for configuration loading
  path: ${path.config}/modules.d/*.yml

  # Set to true to enable config reloading
  reload.enabled: false

  # Period on which files under path should be checked for changes
  #reload.period: 10s

#----------------------------- Logstash output --------------------------------
output.logstash:
  # The Logstash hosts
  hosts: ["192.168.100.1:15044"]


#================================ Processors =====================================

# Configure processors to enhance or manipulate events generated by the beat.

processors:
  - add_host_metadata: 
      netinfo.enabled: true
  - add_cloud_metadata: ~

           

logstash中接收配置如下：

input {
    beats { 
        port => "15044"
    }
}

filter {
    if [log_type] == "mail_transportlog" { 
        csv {
           columns => ["date_time","client_ip","client_hostname","server_ip","server_hostname","source_context","connector_id","source_type","event_id","internal_message_id","message_id","network_message_id","recipient_address","recipient_status","total_bytes","recipient_count","related_recipient_address","reference","message_subject","sender_address","return_path","message_info","directionality","tenant_id","original_client_ip","original_server_ip","custom_data"]
        }
        mutate {
           remove_field => [ "@timestamp","ecs","input","log","@version","agent","host","tags"]
        }
    }    

    else if [log_type] == "mail_iislog" {
        grok {
            match => { "message" => "%{TIMESTAMP_ISO8601:date_time} %{IPORHOST:server_ip} %{WORD:method} %{URIPATH:path} %{NOTSPACE:args} %{NUMBER:port} %{NOTSPACE:username} %{IPORHOST:client_ip} %{NOTSPACE:user_agent} %{NOTSPACE:referrer} %{NUMBER:response} %{NUMBER:sc_status} %{NUMBER:sc_substatus} %{NUMBER:time_taken}"}
        }
        mutate {
           remove_field => [ "@timestamp","ecs","input","log","@version","agent","host","tags"]
        }
    }               
}

output {
    if [log_type] == "mail_transportlog" { 
       #stdout{}
       udp {
             host => "127.0.0.1"
             port => "15001"
        }
    }

    if [log_type] == "mail_iislog" {
       #stdout{}
       udp {
             host => "127.0.0.1"
             port => "15002"
        }
    }
}

           

參考博文：

1. https://www.jianshu.com/p/69ae6de2fdc4
2. https://www.bianchengquan.com/article/153237.html