Flume - How to install Flume NG 1.5.2

■ Download Flume NG 1.5.2(/usr/local/src/)
$ git clone https://git-wip-us.apache.org/repos/asf/flume.git flume
$ cd flume
$ git checkout trunk
or
$ wget http://mirror.nexcess.net/apache/flume/1.5.2/apache-flume-1.5.2-bin.tar.gz

■ Decompress it
$ tar xvf ./apache-flume-1.5.2-bin.tar.gz
$ mv ./apache-flume-1.5.2-bin ../flume

■ Change owner file
$ chown -R hadoop.hadoop ./flume/


■ Change the file's name
$ cd  /usr/local/flume/conf
$ mv ./flume-env.sh.template ./flume-env.sh
$ mv ./flume-conf.properties.template ./flume.conf

■ Set environment variables 
$ vim  /usr/local/flume/conf/flume-env.sh

JAVA_HOME=/usr/local/java
FLUME_CLASSPATH="/usr/local/flume/lib"

■ Set configuration file 
$ vim  /usr/local/flume/conf/flume-env.sh

agent1.channels = ch1
# Define a memory channel called ch1 on agent1
agent1.channels.ch1.type = memory
agent1.channels.ch1.capacity = 100000
agent1.channels.ch1.transactionCapacity = 1000

# Define an Avro source called avro-source1 on agent1 and tell it
# to bind to xxx.xxx.xxx.xxx:41414. Connect it to channel ch1.
agent1.sources = avro-source1
agent1.sources.avro-source1.channels = ch1
agent1.sources.avro-source1.type = avro
agent1.sources.avro-source1.bind = 192.168.11.25
agent1.sources.avro-source1.port = 41414

agent1.sinks = hdfs-sink1
agent1.sinks.hdfs-sink1.type = hdfs
agent1.sinks.hdfs-sink1.channel = ch1
agent1.sinks.hdfs-sink1.hdfs.path = hdfs://192.168.11.23:9000/home/hadoop/data/flume/%Y%m%d/%H
agent1.sinks.hdfs-sink1.hdfs.filePrefix = ch1
agent1.sinks.hdfs-sink1.hdfs.inUseSuffix = .txt
agent1.sinks.hdfs-sink1.hdfs.fileType = DataStream
agent1.sinks.hdfs-sink1.hdfs.rollCount = 0
agent1.sinks.hdfs-sink1.hdfs.rollInterval = 1200
agent1.sinks.hdfs-sink1.hdfs.writeFormat = text
agent1.sinks.hdfs-sink1.hdfs.rollSize = 0
agent1.sinks.hdfs-sink1.hdfs.rollCount=1000000
agent1.sinks.hdfs-sink1.hdfs.batchSize = 10
agent1.sinks.hdfs-sink1.hdfs.threadsPoolSize=10

■ Execute Flume
$ bin/flume-ng agent -n agent1 -c conf -f conf/flume.conf
or
$ bin/flume-ng agent -n agent1 --conf ./conf/ -f conf/flume.conf -Dflume.root.logger=DEBUG,console

Flume - flume-env.sh

JAVA_HOME=/usr/local/java

FLUME_CLASSPATH="/home/hadoop/flume/lib"

Flume - This is the flume.conf in a service server in Flume NG 1.3.1

# To bring logs and send to other source

agent1.channels = ch1
# Define a memory channel called ch1 on agent1
agent1.channels.ch1.type = memory
agent1.channels.ch1.capacity = 100000
agent1.channels.ch1.transactionCapacity = 1000

# Define an Exec Source called exec1
agent1.sources = exec1
agent1.sources.exec1.type = exec
agent1.sources.exec1.command = tail -F /usr/local/tomcat/logs/api/api.log
agent1.sources.exec1.interceptors = ts
agent1.sources.exec1.interceptors.ts.type = timestamp
agent1.sources.exec1.channels = ch1

# properties of hdfs-Cluster1-sink
agent1.sinks = avro-sink1
agent1.sinks.avro-sink1.type = avro
agent1.sinks.avro-sink1.channel = ch1
agent1.sinks.avro-sink1.hostname = 1xx.xxx.111.01
agent1.sinks.avro-sink1.port = 41414

Flume - This is the flume.conf in Flume NG 1.3.1

# To save logs data to HDFS.

agent1.channels = ch1
# Define a memory channel called ch1 on agent1
agent1.channels.ch1.type = memory
agent1.channels.ch1.capacity = 100000
agent1.channels.ch1.transactionCapacity = 1000

# Define an Avro source called avro-source1 on agent1 and tell it
# to bind to xxx.xxx.xxx.xxx:41414. Connect it to channel ch1.
agent1.sources = avro-source1
agent1.sources.avro-source1.channels = ch1
agent1.sources.avro-source1.type = avro
agent1.sources.avro-source1.bind = 1xx.xxx.111.01
agent1.sources.avro-source1.port = 41414

agent1.sinks = hdfs-sink1
agent1.sinks.hdfs-sink1.type = hdfs
agent1.sinks.hdfs-sink1.channel = ch1
agent1.sinks.hdfs-sink1.hdfs.path = hdfs://xxx.xxx.xxx.xxx:9000/home/hadoop/data/flume/%Y%m%d/%H
agent1.sinks.hdfs-sink1.hdfs.filePrefix = ch1
agent1.sinks.hdfs-sink1.hdfs.inUseSuffix = .txt
agent1.sinks.hdfs-sink1.hdfs.fileType = DataStream
agent1.sinks.hdfs-sink1.hdfs.rollCount = 0
agent1.sinks.hdfs-sink1.hdfs.rollInterval = 1200
agent1.sinks.hdfs-sink1.hdfs.writeFormat = text
agent1.sinks.hdfs-sink1.hdfs.rollSize = 0
agent1.sinks.hdfs-sink1.hdfs.rollCount=1000000
agent1.sinks.hdfs-sink1.hdfs.batchSize = 10
agent1.sinks.hdfs-sink1.hdfs.threadsPoolSize=10



----------------------------------------------------
http://www.nextree.co.kr/p2704/

Flume - Shell script for starting in Flume NG 1.3.1

# If you have faced the error, should install the below.
sudo yum install redhat-lsb.x86_64
 
# This script is permitted for Hadoop user.
$sudo su - hadoop

Flume - Ganglia Install in Flume NG 1.3.1

$ yum install arp apr-devel
$ yum install rrdtool rrdtool-devel
$ yum install libconfuse libconfuse-devel
$ yum install pcre pcre-devel
$ yum install expat expat-devel
$ yum install zlib zlib-devel

@ Install libconfuse
$ ./configure --with-pic
$ make
$ make install

$ mkdir -p /home/hadoop/ganglia/rrd/
$ chown nobody.nobody /home/hadoop/ganglia/rrd/
$ cd ./ganglia-3.6.0
$ ./configure --with-librrd=/home/hadoop/ganglia/rrd/ --with-gmetad --prefix=/usr/local/
$ make
$ make install

@You can confirm
$ ls /usr/local/bin/gstat
$ ls /usr/local/bin/gmetric
$ ls /usr/local/sbin/gmond
$ ls /usr/local/sbin/gmetad

@[.] is Ganglia compiled home
@ Register to service
$ cp ./gmond/gmond.init /etc/rc.d/init.d/gmond
$ chkconfig --add gmond
$ chkconfig --list gmond
$ vi /etc/rc.d/init.d/gmond
--> Edit ->GMOND=/usr/local/sbin/gmond

$ cp ./gmetad/gmetad.init /etc/rc.d/init.d/gmetad
$ chkconfig --add gmetad
$ chkconfig --list gmetad
$ vi /etc/rc.d/init.d/gmetad
--> Edit -> GMOND=/usr/local/sbin/gmetad

@ Copy conf
$ /usr/local/sbin/gmond --default_config > /usr/local/etc/gmond.conf

@ Set rrd tool
$ vi /etc/local/etc/gmetad
 -> rrd_rootdir "/home/hadoop/ganglia/rrd"


@ Start
# /etc/rc.d/init.d/gmond start
# /etc/rc.d/init.d/gmetad start

@ Confirm process
# telnet localhos 8649
--> Output XML

http://blog.daum.net/_blog/BlogTypeView.do?blogid=0N9yp&articleno=25&_bloghome_menu=recenttext#ajax_history_home

http://apexserver.iptime.org/users/yk.choi/weblog/7eca7/

http://ahmadchaudary.wordpress.com/tag/ganglia-monitoring/

Flume - Install Flume NG 1.3.1

■What's Changed?

• There's no more logical or physical nodes. We call all physical nodes agents and agents can run zero or more sources and sinks.
• There's no master and no ZooKeeper dependency anymore. At this time, Flume runs with a simple file-based configuration system.

■Install Flume NG 1.3.1
$ git clone https://git-wip-us.apache.org/repos/asf/flume.git flume
$ cd flume
$ git checkout trunk{code}
OR
$ wget http://ftp.kddilabs.jp/infosystems/apache/flume/1.3.1/apache-flume-1.3.1-bin.tar.gz{code}

■Configuration
$ cp conf/flume-conf.properties.template conf/flume.conf
$ cp conf/flume-env.sh.template conf/flume-env.sh

■Change the file (conf/flume.conf)
#=====================================================
# Define a memory channel called ch1 on agent1
agent1.channels.ch1.type = memory

# Define an Avro source called avro-source1 on agent1 and tell it
# to bind to 0.0.0.0:41414. Connect it to channel ch1.
agent1.sources.avro-source1.channels = ch1
agent1.sources.avro-source1.type = avro
agent1.sources.avro-source1.bind = 0.0.0.0
agent1.sources.avro-source1.port = 41414

# Define a logger sink that simply logs all events it receives
# and connect it to the other end of the same channel.
agent1.sinks.log-sink1.channel = ch1
agent1.sinks.log-sink1.type = logger

# Finally, now that we've defined all of our components, tell
# agent1 which ones we want to activate.
agent1.channels = ch1
agent1.sources = avro-source1
agent1.sinks = log-sink1
#=====================================================

■Execute
$ bin/flume-ng agent --conf ./conf/ -f conf/flume.conf -Dflume.root.logger=DEBUG,console -n agent1

■reference
https://cwiki.apache.org/FLUME/getting-started.html