Thursday, April 9, 2015

Flume - How to install Flume NG 1.5.2

■ Download Flume NG 1.5.2(/usr/local/src/)
$ git clone https://git-wip-us.apache.org/repos/asf/flume.git flume
$ cd flume
$ git checkout trunk

or
$ wget http://mirror.nexcess.net/apache/flume/1.5.2/apache-flume-1.5.2-bin.tar.gz

■ Decompress it
$ tar xvf ./apache-flume-1.5.2-bin.tar.gz
$ mv ./apache-flume-1.5.2-bin ../flume

■ Change owner file
$ chown -R hadoop.hadoop ./flume/


■ Change the file's name
$ cd  /usr/local/flume/conf
$ mv ./flume-env.sh.template ./flume-env.sh
$ mv ./flume-conf.properties.template ./flume.conf

Set environment variables 
$ vim  /usr/local/flume/conf/flume-env.sh

JAVA_HOME=/usr/local/java
FLUME_CLASSPATH="/usr/local/flume/lib"

Set configuration file 
$ vim  /usr/local/flume/conf/flume-env.sh

agent1.channels = ch1
# Define a memory channel called ch1 on agent1
agent1.channels.ch1.type = memory
agent1.channels.ch1.capacity = 100000
agent1.channels.ch1.transactionCapacity = 1000

# Define an Avro source called avro-source1 on agent1 and tell it
# to bind to xxx.xxx.xxx.xxx:41414. Connect it to channel ch1.
agent1.sources = avro-source1
agent1.sources.avro-source1.channels = ch1
agent1.sources.avro-source1.type = avro
agent1.sources.avro-source1.bind = 192.168.11.25
agent1.sources.avro-source1.port = 41414

agent1.sinks = hdfs-sink1
agent1.sinks.hdfs-sink1.type = hdfs
agent1.sinks.hdfs-sink1.channel = ch1
agent1.sinks.hdfs-sink1.hdfs.path = hdfs://192.168.11.23:9000/home/hadoop/data/flume/%Y%m%d/%H
agent1.sinks.hdfs-sink1.hdfs.filePrefix = ch1
agent1.sinks.hdfs-sink1.hdfs.inUseSuffix = .txt
agent1.sinks.hdfs-sink1.hdfs.fileType = DataStream
agent1.sinks.hdfs-sink1.hdfs.rollCount = 0
agent1.sinks.hdfs-sink1.hdfs.rollInterval = 1200
agent1.sinks.hdfs-sink1.hdfs.writeFormat = text
agent1.sinks.hdfs-sink1.hdfs.rollSize = 0
agent1.sinks.hdfs-sink1.hdfs.rollCount=1000000
agent1.sinks.hdfs-sink1.hdfs.batchSize = 10
agent1.sinks.hdfs-sink1.hdfs.threadsPoolSize=10

■ Execute Flume
$ bin/flume-ng agent -n agent1 -c conf -f conf/flume.conf
or
$ bin/flume-ng agent -n agent1 --conf ./conf/ -f conf/flume.conf -Dflume.root.logger=DEBUG,console

No comments:

Post a Comment