LINK - java pattern

http://buedt.tistory.com/23

Hadoop - commands

@Find files you want to see.
hadoop dfs -lsr /hadoop/flume/ | grep [search_term].

@hadoop error(release safe mode)
$./bin/hadoop dfsadmin -safemode leave

Hodoop - Get a content of file from Hadoop sample

public class TestMain {
    /**
     * @param args
     * @throws Exception
     */
    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        conf.set("fs.default.name", "hdfs://xxx.28.xxx.51:9000");
        FileSystem dfs = FileSystem.get(conf);

        Path filenamePath = new Path("/home/hadoop/data/flume/20130529/12/ch1.1369796403013");
        FSDataInputStream fsIn = dfs.open(filenamePath);

        // org.apache.commons.io.IOUtils
        byte[] fileBytes = IOUtils.toByteArray(fsIn);
        fsIn.read(fileBytes);

        //create string from byte array
        String strFileContent = new String(fileBytes);
        System.out.println(strFileContent);
    }
}

Hadoop Manger URL

I will make a arrangement about below this later. 

http://confluence.openflamingo.org/pages/viewpage.action?pageId=5537913&focusedCommentId=7209528&#comment-7209528

Flume - Shell script for starting in Flume NG 1.3.1

# If you have faced the error, should install the below.
sudo yum install redhat-lsb.x86_64
 
# This script is permitted for Hadoop user.
$sudo su - hadoop

	#!/bin/bash
	. /lib/lsb/init-functions
	[ -e /etc/sysconfig/flume ] && . /etc/sysconfig/flume
	STATUS_RUNNING=0
	STATUS_DEAD=1
	STATUS_DEAD_AND_LOCK=2
	STATUS_NOT_RUNNING=3
	ERROR_PROGRAM_NOT_INSTALLED=5
	FLUME_LOG_DIR=/home/hadoop/flume/logs
	FLUME_CONF_DIR=/home/hadoop/flume/conf
	FLUME_RUN_DIR=/home/hadoop/var/run/flume
	FLUME_HOME=/home/hadoop/flume
	FLUME_USER=hadoop
	FLUME_LOCK_DIR="/home/hadoop/var/lock/subsys/"
	LOCKFILE="${FLUME_LOCK_DIR}/flume-ng-agent"
	desc="Flume NG agent daemon"
	#FLUME_CONF_FILE=${FLUME_CONF_FILE:-${FLUME_CONF_DIR}/flume.conf}
	FLUME_CONF_FILE=${FLUME_CONF_DIR}/flume.conf
	EXEC_PATH=${FLUME_HOME}/bin/flume-ng
	FLUME_PID_FILE=${FLUME_RUN_DIR}/flume-ng-agent.pid
	# These directories may be tmpfs and may or may not exist
	# depending on the OS (ex: /var/lock/subsys does not exist on debian/ubuntu)
	for dir in "$FLUME_RUN_DIR" "$FLUME_LOCK_DIR"; do
	[ -d "${dir}" ] || install -d -m 0755 -o $FLUME_USER -g $FLUME_USER ${dir}
	done
	#DEFAULT_FLUME_AGENT_NAME="agent1"
	#FLUME_AGENT_NAME=${FLUME_AGENT_NAME:-${DEFAULT_FLUME_AGENT_NAME}}
	#
	FLUME_AGENT_NAME="agent1"
	start() {
	[ -x $exec ] || exit $ERROR_PROGRAM_NOT_INSTALLED
	pidofproc -p $FLUME_PID_FILE java > /dev/null
	status=$?
	if [ "$status" -eq "$STATUS_RUNNING" ]; then
	exit 0
	fi
	log_success_msg "Starting $desc (flume-ng-agent): "
	# /bin/su -s /bin/bash -c "/bin/bash -c 'echo \$\$ >${FLUME_PID_FILE} && exec ${EXEC_PATH} agent --no-reload-conf --conf $FLUME_CONF_DIR --conf-file $FLUME_CONF_FILE --name $FLUME_AGENT_NAME >>${FLUME_LOG_DIR}/flume.init.log 2>&1' &" $FLUME_USER
	/bin/bash -c "/bin/bash -c 'echo \$\$ >${FLUME_PID_FILE} && exec ${EXEC_PATH} agent --no-reload-conf --conf $FLUME_CONF_DIR --conf-file $FLUME_CONF_FILE --name $FLUME_AGENT_NAME >>${FLUME_LOG_DIR}/flume.init.log 2>&1' &"
	RETVAL=$?
	[ $RETVAL -eq 0 ] && touch $LOCKFILE
	return $RETVAL
	}
	stop() {
	if [ ! -e $FLUME_PID_FILE ]; then
	log_failure_msg "Flume agent is not running"
	exit 0
	fi
	FLUME_PID=`cat $FLUME_PID_FILE`
	if [ -n $FLUME_PID ]; then
	kill -TERM ${FLUME_PID} &>/dev/null
	status=0
	while [ $status -eq 0 ]; do
	sleep 1
	ps -p $FLUME_PID &> /dev/null
	status=$?
	done
	fi
	rm -f $LOCKFILE $FLUME_PID_FILE
	log_success_msg "Stopping $desc (flume-ng-agent): "
	return 0
	}
	checkstatus(){
	pidofproc -p $FLUME_PID_FILE java > /dev/null
	status=$?
	case "$status" in
	$STATUS_RUNNING)
	log_success_msg "Flume NG agent is running"
	;;
	$STATUS_DEAD)
	log_failure_msg "Flume NG agent is dead and pid file exists"
	;;
	$STATUS_DEAD_AND_LOCK)
	log_failure_msg "Flume NG agent is dead and lock file exists"
	;;
	$STATUS_NOT_RUNNING)
	log_failure_msg "Flume NG agent is not running"
	;;
	*)
	log_failure_msg "Flume NG agent status is unknown"
	;;
	esac
	return $status
	}
	condrestart(){
	[ -e ${LOCKFILE} ] && restart || :
	}
	case "$1" in
	start)
	start
	;;
	stop)
	stop
	;;
	status)
	checkstatus
	;;
	restart)
	restart
	;;
	condrestart|try-restart)
	condrestart
	;;
	*)
	echo $"Usage: $0 {start|stop|status|restart|try-restart|condrestart}"
	exit 1
	esac
	exit $RETVAL

view raw flume.sh hosted with ❤ by GitHub

Shell - init-functions

@If you want to install this /lib/lsb/init-functions on the linux?
@Just install below this.
$ yum install redhat-lsb

@=======================================
#!/bin/sh

# LSB initscript functions, as defined in the LSB Spec 1.1.0
#
# Lawrence Lim <llim@redhat.com> - Tue, 26 June 2007
# Updated to the latest LSB 3.1 spec
# http://refspecs.freestandards.org/LSB_3.1.0/LSB-Core-generic/LSB-Core-generic_lines.txt

start_daemon () {
        /etc/redhat-lsb/lsb_start_daemon "$@"
}

killproc () {
        /etc/redhat-lsb/lsb_killproc "$@"
}

pidofproc () {
        /etc/redhat-lsb/lsb_pidofproc "$@"
}

log_success_msg () {
        /etc/redhat-lsb/lsb_log_message success "$@"
}

log_failure_msg () {
        /etc/redhat-lsb/lsb_log_message failure "$@"
}

log_warning_msg () {
        /etc/redhat-lsb/lsb_log_message warning "$@"
}

Shell - init.d Script

Let's edit later.

http://werxltd.com/wp/2012/01/05/simple-init-d-script-template/

Flume - Ganglia Install in Flume NG 1.3.1

$ yum install arp apr-devel
$ yum install rrdtool rrdtool-devel
$ yum install libconfuse libconfuse-devel
$ yum install pcre pcre-devel
$ yum install expat expat-devel
$ yum install zlib zlib-devel

@ Install libconfuse
$ ./configure --with-pic
$ make
$ make install

$ mkdir -p /home/hadoop/ganglia/rrd/
$ chown nobody.nobody /home/hadoop/ganglia/rrd/
$ cd ./ganglia-3.6.0
$ ./configure --with-librrd=/home/hadoop/ganglia/rrd/ --with-gmetad --prefix=/usr/local/
$ make
$ make install

@You can confirm
$ ls /usr/local/bin/gstat
$ ls /usr/local/bin/gmetric
$ ls /usr/local/sbin/gmond
$ ls /usr/local/sbin/gmetad

@[.] is Ganglia compiled home
@ Register to service
$ cp ./gmond/gmond.init /etc/rc.d/init.d/gmond
$ chkconfig --add gmond
$ chkconfig --list gmond
$ vi /etc/rc.d/init.d/gmond
--> Edit ->GMOND=/usr/local/sbin/gmond

$ cp ./gmetad/gmetad.init /etc/rc.d/init.d/gmetad
$ chkconfig --add gmetad
$ chkconfig --list gmetad
$ vi /etc/rc.d/init.d/gmetad
--> Edit -> GMOND=/usr/local/sbin/gmetad

@ Copy conf
$ /usr/local/sbin/gmond --default_config > /usr/local/etc/gmond.conf

@ Set rrd tool
$ vi /etc/local/etc/gmetad
 -> rrd_rootdir "/home/hadoop/ganglia/rrd"


@ Start
# /etc/rc.d/init.d/gmond start
# /etc/rc.d/init.d/gmetad start

@ Confirm process
# telnet localhos 8649
--> Output XML

http://blog.daum.net/_blog/BlogTypeView.do?blogid=0N9yp&articleno=25&_bloghome_menu=recenttext#ajax_history_home

http://apexserver.iptime.org/users/yk.choi/weblog/7eca7/

http://ahmadchaudary.wordpress.com/tag/ganglia-monitoring/

Git - Add a tag in order

1.Edit pom.xml
 -Like 1.0-SNAPSHOT to 1.1 2.Index

2.Add Index
 $ git add *

3.Commit
 $ git commit -m "Tag v1.1"

4.Add a tag
 $ git tag v1.1

5.push to remote server
  @ When I did the following command for Github,
       the system didn't ask me to input Id and Password
       (master = tag version)
 $ git push origin v1.1

6.return to the development
 $ git fetch origin
 $ git reset --hard origin/master

How to use Flume ng

bin/flume-ng agent --conf-file conf/flume.conf --name agent1 -Dflume.monitoring.type=GANGLIA -Dflume.monitoring.hosts=172.xx.xxx.xx:5455

Hive - join

@Join

	SELECT a.uu_id, a.time as registerTime, a.activity as register, b.activity as buy
	FROM (select * from tableA where dt="2013-05-17" and activity="register") a
	JOIN (select * from tableB where dt="2013-05-17" and activity="buy") b
	ON(a.uu_id = b.uu_id) limit 10;

view raw JoinHive.sql hosted with ❤ by GitHub

Flume - Install Flume NG 1.3.1

■What's Changed?

• There's no more logical or physical nodes. We call all physical nodes agents and agents can run zero or more sources and sinks.
• There's no master and no ZooKeeper dependency anymore. At this time, Flume runs with a simple file-based configuration system.

■Install Flume NG 1.3.1
$ git clone https://git-wip-us.apache.org/repos/asf/flume.git flume
$ cd flume
$ git checkout trunk{code}
OR
$ wget http://ftp.kddilabs.jp/infosystems/apache/flume/1.3.1/apache-flume-1.3.1-bin.tar.gz{code}

■Configuration
$ cp conf/flume-conf.properties.template conf/flume.conf
$ cp conf/flume-env.sh.template conf/flume-env.sh

■Change the file (conf/flume.conf)
#=====================================================
# Define a memory channel called ch1 on agent1
agent1.channels.ch1.type = memory

# Define an Avro source called avro-source1 on agent1 and tell it
# to bind to 0.0.0.0:41414. Connect it to channel ch1.
agent1.sources.avro-source1.channels = ch1
agent1.sources.avro-source1.type = avro
agent1.sources.avro-source1.bind = 0.0.0.0
agent1.sources.avro-source1.port = 41414

# Define a logger sink that simply logs all events it receives
# and connect it to the other end of the same channel.
agent1.sinks.log-sink1.channel = ch1
agent1.sinks.log-sink1.type = logger

# Finally, now that we've defined all of our components, tell
# agent1 which ones we want to activate.
agent1.channels = ch1
agent1.sources = avro-source1
agent1.sinks = log-sink1
#=====================================================

■Execute
$ bin/flume-ng agent --conf ./conf/ -f conf/flume.conf -Dflume.root.logger=DEBUG,console -n agent1

■reference
https://cwiki.apache.org/FLUME/getting-started.html

Linux - user commond

# Add user

$/usr/sbin/useradd -d /home/njoonk -m njoonk -g njoonk

# Set password

$/usr/bin/passwd njoonk

# Delete user

$userdel testuser #Only a user.
$userdel -r testuser #Only a user with user's directory.

# Create a public in linux

$ ssh-keygen -t rsa

# Input the public-key into the authorized_keys

$ vi authorized_keys
$ chmod 644 .ssh/authorized_keys

Java - GC

JAVA_OPTS="-server"
JAVA_OPTS="${JAVA_OPTS} -Xms1024m -Xmx1024m -Xmn768m -XX:SurvivorRatio=2 -XX:PermSize=64m -XX:MaxPermSize=256m"
JAVA_OPTS="${JAVA_OPTS} -XX:+PrintGCDetails -Xloggc:/usr/local/tomcat/logs/gc.log"

Reference URL
http://fly32.net/438

http://www.javaservice.com/~java/bbs/read.cgi?m=&b=weblogic&c=r_p&n=1221718848&p=6&s=t

About git information

@Add remote git server URL
git fetch origin
git reset --hard origin/master
 
@Download new version from remote(you need to command the fetch)
git checkout HEAD

@Delete tags in remote.
git push origin :tags/{tag name}

@Upload remote
git push origin v1.5
git push origin --tags

@Delete tags in local
git tag -d {tag name}

@Create a tag
git tag v1.0
@Up load a tag to remote
git push --tags

@Show the commit id
git rev-parse [Tag Name]

@Delete Tag
git tag -d [Tag Name]

@Order a commit
git add *
git commit -m "This is the first commit"
git push

@DownLoad all branch from remote
git fetch origin

@
1.$ vim ./.git/config

2.[branch "master"]

        remote = origin

        merge = refs/heads/master

@a good thing to put this bellow into the config file

[alias]

        hist = log --pretty=format:'%h %ad | %s%d [%an]' --graph --date=short

[color]

        ui = true

How to get a thread dump in Linux.

If you have not been ready to get a thread dump in Linux.

./jstack -l -F 22431 > /home/share/kim_joon/thread3.txt

Ruby

http://dimdim.tistory.com/56
http://www.jacopretorius.net/2012/01/ruby-map-collect-and-select.html
http://ruby-doc.org/core-2.0/Array.html

Eclipse - c, c++

http://chanroid.tistory.com/6
http://paralaxer.com/cpp-vs-objective-c/

I will write a post in the blog.