Friday, June 28, 2013

Flume - This is the flume.conf in Flume NG 1.3.1

# To save logs data to HDFS.

agent1.channels = ch1
# Define a memory channel called ch1 on agent1
agent1.channels.ch1.type = memory
agent1.channels.ch1.capacity = 100000
agent1.channels.ch1.transactionCapacity = 1000

# Define an Avro source called avro-source1 on agent1 and tell it
# to bind to xxx.xxx.xxx.xxx:41414. Connect it to channel ch1.
agent1.sources = avro-source1
agent1.sources.avro-source1.channels = ch1
agent1.sources.avro-source1.type = avro
agent1.sources.avro-source1.bind = 1xx.xxx.111.01
agent1.sources.avro-source1.port = 41414

agent1.sinks = hdfs-sink1
agent1.sinks.hdfs-sink1.type = hdfs
agent1.sinks.hdfs-sink1.channel = ch1
agent1.sinks.hdfs-sink1.hdfs.path = hdfs://xxx.xxx.xxx.xxx:9000/home/hadoop/data/flume/%Y%m%d/%H
agent1.sinks.hdfs-sink1.hdfs.filePrefix = ch1
agent1.sinks.hdfs-sink1.hdfs.inUseSuffix = .txt
agent1.sinks.hdfs-sink1.hdfs.fileType = DataStream
agent1.sinks.hdfs-sink1.hdfs.rollCount = 0
agent1.sinks.hdfs-sink1.hdfs.rollInterval = 1200
agent1.sinks.hdfs-sink1.hdfs.writeFormat = text
agent1.sinks.hdfs-sink1.hdfs.rollSize = 0
agent1.sinks.hdfs-sink1.hdfs.rollCount=1000000
agent1.sinks.hdfs-sink1.hdfs.batchSize = 10
agent1.sinks.hdfs-sink1.hdfs.threadsPoolSize=10



----------------------------------------------------
http://www.nextree.co.kr/p2704/

No comments:

Post a Comment