Hadoop Shell Bash Scripts

tae.sh

example.sh

http://stackoverflow.com/questions/15105135/bash-capturing-output-of-awk-into-array

#!/bin/bash

#watch -n1 cat ./example2

#chmod +x ./tae.sh

SPILLED_RECORD=`cat logWordCount | grep 'Spilled Record' | awk -F'=' '{print $2}'`

echo "Spilled Record,"$SPILLED_RECORD

#2

#!/bin/bash

SPILLED_RECORD=`cat logWordCount | grep 'Spilled Record' | awk -F'=' '{print $2}'`

SLOTS_MILLIS_REDUCES=`cat logWordCount | grep 'SLOTS_MILLIS_REDUCES' | awk -F'=' '{print $2}'`

SLOTS_MILLIS_MAPS=`cat logWordCount | grep 'SLOTS_MILLIS_MAPS' | awk -F'=' '{print $2}'`

CPU_TIME_SPENT=`cat logWordCount | grep 'CPU time spent (ms)' | awk -F'=' '{print $2}'`

record=""

for value in $SPILLED_RECORD

do

        record=$record"$value,"

done

echo "Spilled Record,$record"

echo "SLOTS_MILLIS_REDUCE,"$SLOTS_MILLIS_REDUCES

echo "SLOTS_MILLIS_MAPSi,"$SLOTS_MILLIS_MAPS

echo "SCPU_TIME_SPENT,"$CPU_TIME_SPENT

#3

SPILLED_RECORD=`cat logWordCount | grep 'Spilled Record' | awk -F'=' '{print $2}'`

SLOTS_MILLIS_REDUCES=`cat logWordCount | grep 'SLOTS_MILLIS_REDUCES' | awk -F'=' '{print $2}'`

SLOTS_MILLIS_MAPS=`cat logWordCount | grep 'SLOTS_MILLIS_MAPS' | awk -F'=' '{print $2}'`

CPU_TIME_SPENT=`cat logWordCount | grep 'CPU time spent (ms)' | awk -F'=' '{print $2}'`

for value in $SPILLED_RECORD

do      

        echo $value

done

echo "Spilled Record,"$SPILLED_RECORD

echo "SLOTS_MILLIS_REDUCE,"$SLOTS_MILLIS_REDUCES

echo "SLOTS_MILLIS_MAPSi,"$SLOTS_MILLIS_MAPS

echo "SCPU_TIME_SPENT,"$CPU_TIME_SPENT

#!/usr/bin/env bash

/home/jjoon/yarn/bin/hadoop fs -rmr out*;

sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches';

/home/jjoon/yarn/bin/hadoop jar /home/jjoon/yarn/hadoop-jar/PageRank.jar PageRank soc-inputfile.txt output_soc_pagerank1 option1 option2 option3 2> experiment1;

sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches';

/home/jjoon/yarn/bin/hadoop jar /home/jjoon/yarn/hadoop-jar/PageRank.jar PageRank soc-inputfile.txt output_soc_pagerank2 option1 option2 option3 2> experiment2;

sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches';

/home/jjoon/yarn/bin/hadoop jar /home/jjoon/yarn/hadoop-jar/PageRank.jar PageRank soc-inputfile.txt output_soc_pagerank3 option1 option2 option3 2> experiment3;

http://stackoverflow.com/questions/15105135/bash-capturing-output-of-awk-into-array