flume配置参考

1 redis source,memory channel,json intercepter and file_roll sink配置例子

agent.sources = redis redis1

agent.channels = memory

agent.sinks = fileSink

agent.sources.redis.type = com.flumeng.plugins.source.RedisLPopSource

agent.sources.redis.host = 192.168.121.128

agent.sources.redis.port = 6379

agent.sources.redis.eventNamePrefix = event

agent.sources.redis.timeout = 3000

agent.sources.redis.charset = utf-8

agent.sources.redis.interceptors = i1

agent.sources.redis.interceptors.i1.type = com.flumeng.plugins.source.JSONInterceptor$Builder

agent.sources.redis1.type = com.flumeng.plugins.source.RedisLRangeSource

agent.sources.redis1.host = 192.168.121.128

agent.sources.redis1.port = 6379

agent.sources.redis1.eventNamePrefix = event

agent.sources.redis1.timeout = 3000

agent.sources.redis1.charset = utf-8

agent.sources.redis1.eventbatchSize = 2000

agent.sources.redis1.interceptors = i2

agent.sources.redis1.interceptors.i2.type = com.flumeng.plugins.source.JSONInterceptor$Builder

agent.channels.memory.type = memory

agent.channels.memory.capacity = 1000000

agent.channels.memory.transactionCapacity = 100000

agent.sinks.fileSink.type = file_roll

agent.sinks.fileSink.channel = memory

agent.sinks.fileSink.sink.directory = /mnt/hgfs/share/result/

agent.sinks.fileSink.sink.rollInterval = 0

agent.sinks.fileSink.batchSize=500

agent.sources.redis.channels = memory

agent.sources.redis1.channels = memory

2 redis source,memory channel,json intercepter and avro sink配置例子

agent.sources = redis redis1

agent.channels = memory

agent.sinks = fileSink

agent.sources.redis.type = com.flumeng.plugins.source.RedisLPopSource

agent.sources.redis.host = 192.168.121.128

agent.sources.redis.port = 6379

agent.sources.redis.eventNamePrefix = event

agent.sources.redis.timeout = 3000

agent.sources.redis.charset = utf-8

agent.sources.redis.interceptors = i1

agent.sources.redis.interceptors.i1.type = com.flumeng.plugins.source.JSONInterceptor$Builder

agent.sources.redis1.type = com.flumeng.plugins.source.RedisLRangeSource

agent.sources.redis1.host = 192.168.121.128

agent.sources.redis1.port = 6379

agent.sources.redis1.eventNamePrefix = event

agent.sources.redis1.timeout = 3000

agent.sources.redis1.charset = utf-8

agent.sources.redis1.eventbatchSize = 2000

agent.sources.redis1.interceptors = i2

agent.sources.redis1.interceptors.i2.type = com.flumeng.plugins.source.JSONInterceptor$Builder

agent.channels.memory.type = memory

agent.channels.memory.capacity = 1000000

agent.channels.memory.transactionCapacity = 100000

agent.sinks.fileSink.type = file_roll

agent.sinks.fileSink.channel = memory

agent.sinks.fileSink.sink.directory = /mnt/hgfs/share/result/

agent.sinks.fileSink.sink.rollInterval = 0

agent.sinks.fileSink.batchSize=500

agent.sources.redis.channels = memory

agent.sources.redis1.channels = memory

3 avro source--memery channel ---HDFS sink---interceptors---serializers

agent.sources = avroSource

agent.channels = memory

agent.sinks = hdfsSink

agent.sources.avroSource.type = avro

agent.sources.avroSource.bind = 0.0.0.0

agent.sources.avroSource.port = 4141

agent.sources.avroSource.interceptors = i1

agent.sources.avroSource.interceptors.i1.type=regex_extractor

agent.sources.avroSource.interceptors.i1.regex = (\\d\\d\\d\\d-\\d\\d-\\d\\d\\s\\d\\d:\\d\\d)

agent.sources.avroSource.interceptors.i1.serializers = s1

agent.sources.avroSource.interceptors.i1.serializers.s1.type = org.apache.flume.interceptor.RegexExtractorInterceptorMillisSerializer

agent.sources.avroSource.interceptors.i1.serializers.s1.name = timestamp

agent.sources.avroSource.interceptors.i1.serializers.s1.pattern = yyyy-MM-dd HH:mm

agent.channels.memory.type = memory

agent.channels.memory.capacity = 1000000

agent.channels.memory.transactionCapacity = 100000

agent.sinks.hdfsSink.type = hdfs

agent.sinks.hdfsSink.channel = memory

agent.sinks.hdfsSink.hdfs.path = hdfs://littleNameservice/test/outputAnalysis/y=%Y/m=%m/d=%d/h=%H

agent.sinks.hdfsSink.hdfs.rollInterval =0 

agent.sinks.hdfsSink.hdfs.rollSize =268435456

agent.sinks.hdfsSink.hdfs.rollCount =0

#agent.sinks.hdfsSink.hdfs.filePrefix =%Y_%m_%d_%H

agent.sinks.hdfsSink.hdfs.filePrefix =event

agent.sinks.hdfsSink.hdfs.fileSuffix =.txt

agent.sinks.hdfsSink.hdfs.inUseSuffix =.tmp

agent.sinks.hdfsSink.hdfs.batchSize = 10000

agent.sinks.hdfsSink.hdfs.idleTimeout = 600

agent.sinks.hdfsSink.hdfs.fileType = DataStream

agent.sources.avroSource.channels = memory

4 avro source---memery channel---hbase sink---serializer---interceptor

agent.sources = avroSource

agent.channels = memory

agent.sinks = hbaseSink

agent.sources.avroSource.type = avro

agent.sources.avroSource.bind = 0.0.0.0

agent.sources.avroSource.port = 4141

agent.sources.avroSource.interceptors = i1

agent.sources.avroSource.interceptors.i1.type=com.jj.flume.interceptor.json.EventHbaseInterceptor$Builder

agent.channels.memory.type = memory

agent.channels.memory.capacity = 1000000

agent.channels.memory.transactionCapacity = 100000

agent.sinks.hbaseSink.type = hbase

agent.sinks.hbaseSink.table = t_event

agent.sinks.hbaseSink.columnFamily = f_event

agent.sinks.hbaseSink.serializer = com.jj.flume.sink.hbase.RegexHbaseEventSerializerJJ

agent.sinks.hbaseSink.channel = memory

agent.sources.avroSource.channels = memory