巧用shell+rsync實作日志自動過濾處理壓縮上傳分類(在斯凱的最終版本下周離職)

上次的版本http://blog.chinaunix.net/uid-21142030-id-5613029.html

點選(此處)折疊或打開

1. [10:58 t ~]$ cat rsync.t
2. #!/usr/bin/env bash
3. #set -x
4. #2015-08-18:    修複直接壓縮201目錄的bug
5. #2015-09-28:    新加項目分類目錄,新加多重判斷
6. #2015-10-19:    修複與glusterfs沖突問題.
7. #2016-04-08:    處理磁盤空緊張時無法處理日志壓縮上傳導緻空間滿問題。
8. #2016-04-11:    解決目錄非單獨挂載時識别為/無限循環問題。
9. #2016-04-12:    優化搜尋邏輯，添加輸出日志。
10. LOGPATH=$1
11. PROJECT=$2
12. SLEEPTIME=1s
13. DATETIME=$(date +%F_%T)
14. DATE=$(date +%F)
15. IP=$(LC_ALL=C /sbin/ifconfig|grep "inet addr:"|grep -v "127.0.0.1"|cut -d: -f2|awk '{print $1}'|sed ':a;$!N;s/\n/_/g;ta;P;D')
16. LINE=`seq -s= 80|sed 's/[0-9]*//g'`
17. USER=admin
18. export RSYNC_PASSWORD=skymobi
19. RSYNCIP=192.168.1.2
20. RSYNCMODULE=chss-t
21. RSYNCCONFMODULE=chss-tc
22. RSYNCDIR=${PROJECT}/"$IP"
23. FINDLOGFILE=/tmp/find_"$IP".log.$DATE
24. RSYNCLOGFILE=/tmp/rsync_"$IP".log.$DATE
25. RSYNCEXELOGFILE=/tmp/rsync.exe.log.$DATE
26. REGEX="-regex .*logs/\(.*log\)?.*201.*\(.*log\)?\(.gz\)?"
27. REGEX2="-regex .*tar.gz"
28. TIME="-mtime +${DAYS:=1}"
29. DEL="-exec rm -rfv {} ;"
30. COMPRESSION="-exec tar -zcvf `basename {}`_${RSYNCMODULE}.tar.gz {} --remove-files --exclude=*tar.gz ;"
31. #建立目錄
32. echo -e "$LINE\n$DATETIME : Create directory for upload."
33. mkdir -p /tmp/$RSYNCDIR
34. echo -e "$DATETIME : Upload create directory.\n$LINE"
35. for rspath in $RSYNCMODULE $RSYNCCONFMODULE
36. do
37.     rsync -atv --log-file=${RSYNCLOGFILE} /tmp/${PROJECT} ${USER}@"$RSYNCIP"::${rspath}
38. done
39. #掃描日志
40. for Path in $LOGPATH
42.   echo -e "$LINE \n$DATETIME : Operation on the [ $Path ] directory "
43.   #1.是否存在目錄
44.   if [ -d /$Path ];then
45. #########################################
46.     RSYNC(){
47.       if test $# -ne 0;then
48.         rsync -aERtvz --no-p --chmod=Do+rX,Fo-X --log-file=${RSYNCLOGFILE} --remove-source-files --skip-compress=*.gz "$@" ${USER}@"$RSYNCIP"::${ttype:=$RSYNCMODULE}/${RSYNCDIR}
49.       else
50.         echo "$DATETIME : no $RSYNCMODULE.tar.gz file found."
51.         return 1
52.       fi
53.     }
54.     FINDER(){
55.       find -L /${Path} $@
57.     SPACECOUNT(){
58.     df -P|grep /${Path} |tail -n 1 |awk '{print $5}'|awk -F% '{print $1}'
60. ##########################################
61.     #2.是否空間緊張
62.     echo "$DATETIME : Start while."
63.     starttime=`date`
64.     SPACE=`SPACECOUNT`
65.     Count=0
66.     while test -n "$SPACE" -a "${SPACE:=0}" -gt 20 -a $Count -lt 2
67.     do
68.       if [ $SPACE -gt 80 ];then
69.     echo "$DATETIME : Start del."
70.         TIME="-mmin +720"
71.         FINDER "$REGEX" "-a" "$TIME" "$DEL" >> $FINDLOGFILE
72.         SPACE=`SPACECOUNT`
73.         continue
74.       elif [ $SPACE -gt 60 ];then
75.     echo "$DATETIME : Start upload file direct."
76.         echo '60%
77.         TIME="-mmin +60"
78.         RSYNC `FINDER "$REGEX" "-a" "$TIME"`
80.         ((Count+=1))
82.       #2.
84.     echo "$DATETIME : Start compression."
85.         echo 'space
86.         TIME="-mmin +0"
87.         FINDER "$REGEX" "-a ! $REGEX2" "-a" "$TIME" "$COMPRESSION" >> $FINDLOGFILE
89.     echo "new space is $SPACE"
90.         break
92.     done
93.     endtime=`date`
94.     haldertime=$((`date -d "$endtime" +%s`-`date -d "$starttime" +%s`))
95.     echo "haldertime:$haldertime"
96.     #3.執行上傳操作
97.     echo "$DATETIME : Start upload tar.gz file."
99. # for ttype in ${RSYNCMODULE} #${RSYNCCONFMODULE}
100. # do
101.           RSYNC $(find /$Path -type f -name "*_${RSYNCMODULE}.tar.gz" -a ! -regex '.*/brick/.*')
102.           #RSYNC $(find /$Path -type f -name "*_$ttype.tar.gz" -a ! -regex '.*/brick/.*')
103. # done
106.     echo "upload_tar_gz_haldertime:$haldertime"
107.   #1.
108.   else
109.     echo /${Path} does not exist...
110.   fi
111.   echo -e "$DATETIME : [ $Path ] To the end of the directory operation. \n$LINE"
113. done >>$RSYNCEXELOGFILE

點選(此處)折疊或打開

19. RSYNCIP=192.168.165.88

1. [11:00 t ~]$ cat while.t
2. #!/bin/bash
3. set -x
4. LOGPATH='opt app data app1'
5. PROJECT=talentest
6. sleeptime=0.5h
7. while :
9. time /bin/bash /bin/rsync.t "$LOGPATH" $PROJECT
10. sleep $sleeptime