eSales Media Git repositories
Browse code

Initial commit

Benjamin Roth authored on16/06/2016 16:11:29
Showing6 changed files
backup.sh
History View file @28840c7
1 1 
new file mode 100644
... ... 
@@ -0,0 +1,103 @@
1 
+#!/bin/bash
2 
+
3 
+#+++++++++++++++++++++++++++++++++++++++++++++++++++
4 
+# eSales Media Backup Script
5 
+#
6 
+# Author: Benjamin Roth
7 
+# Version: 20100615
8 
+#+++++++++++++++++++++++++++++++++++++++++++++++++++
9 
+
10 
+
11 
+
12 
+
13 
+##########################
14 
+#     CONFIG SECTION     #
15 
+##########################
16 
+SCRIPT=${BASH_SOURCE[o]}
17 
+SCRIPTPATH=`dirname $SCRIPT`/
18 
+INCLUDEPATH=${SCRIPTPATH}include/
19 
+CONFIGPATH=${SCRIPTPATH}jobs/
20 
+
21 
+BACKUPPATH=${SCRIPTPATH}../backup/
22 
+
23 
+
24 
+#############################
25 
+#     UTILITY FUNCTIONS     #
26 
+#############################
27 
+function progress() {
28 
+	echo -n " ..."
29 
+	#ps ax | awk '{ print $1 }' | grep -w $$ > /dev/null
30 
+	while true
31 
+	do
32 
+		echo -n "."
33 
+		sleep 3
34 
+	done
35 
+}
36 
+
37 
+function run() {
38 
+	val=$1
39 
+	if [ -f ${CONFIGPATH}${val} -a ${val} != "skeletton" ]
40 
+	then
41 
+		echo "Analyzing job $val..."
42 
+		cat ${CONFIGPATH}${val} |
43 
+		while read LINE
44 
+		do
45 
+			if [ "$LINE" = "" ]
46 
+			then
47 
+				continue
48 
+			else
49 
+				LINE=`echo $LINE | sed 's/^\s*\(.*\) *$/\1/'`
50 
+
51 
+				if [ "`echo $LINE | grep "^\s*[a-zA-Z0-9_]*::.*"`" != "" ]
52 
+				then
53 
+					DRIVER=`echo $LINE | sed 's/^\s*\([a-zA-Z0-9_]*\)::.*$/\1/'`
54 
+					if [ -f ${INCLUDEPATH}driver_${DRIVER} ]
55 
+					then
56 
+						echo -en "\tStarting process $DRIVER"
57 
+						progress &
58 
+						PROGRESSPID=$!
59 
+						trap 'kill $PROGRESSPID &> /dev/null; exit 1' 2 15
60 
+						# Load driver and execute
61 
+						msg=`. ${INCLUDEPATH}driver_${DRIVER} $LINE $BACKUPPATH $val`
62 
+						stat=$?
63 
+						kill $PROGRESSPID &> /dev/null
64 
+						if [ $stat -eq 0 ]
65 
+						then
66 
+							echo -n " OK"
67 
+							echo
68 
+							#echo -e "\t# ${msg}"
69 
+						else
70 
+							echo -n " ERROR"
71 
+							echo
72 
+							echo -e "\t# ${msg}"
73 
+						fi
74 
+					else
75 
+						echo -e "\tDriver $DRIVER not found. Check include dir for driver file."
76 
+					fi
77 
+				fi
78 
+
79 
+			fi
80 
+		done
81 
+		echo "done"
82 
+	else
83 
+		if [ ${val} != "skeletton" ]
84 
+		then
85 
+			echo -e "\tConfig file $val not found."
86 
+		fi
87 
+	fi
88 
+}
89 
+
90 
+
91 
+###############
92 
+#     RUN     #
93 
+###############
94 
+if [ $1 ]
95 
+then
96 
+	run $1
97 
+else
98 
+	ls -1 $CONFIGPATH |
99 
+	while read val
100 
+	do
101 
+		run $val
102 
+	done
103 
+fi
include/driver_clean
History View file @28840c7
0 104 
new file mode 100644
... ... 
@@ -0,0 +1,46 @@
1 
+#!/bin/bash
2 
+if [ ! $1 -a $2 -a $3 ]
3 
+then
4 
+	echo "Wrong parameter count."
5 
+	exit 1
6 
+fi
7 
+
8 
+#####################
9 
+#     PARAMETER     #
10 
+#####################
11 
+TARGETDIR=$2
12 
+DAYOFFSET=`echo $1 | awk -F:: '{ print $2 }'`
13 
+let MINUTEOFFSET=${DAYOFFSET}*1440+30
14 
+JOBNAME=$3
15 
+
16 
+
17 
+
18 
+###################
19 
+#     CLEANUP     #
20 
+###################
21 
+if [ -d $TARGETDIR ]
22 
+then
23 
+	# Check for existing backup files
24 
+	OBSOLETECOUNT=`find $TARGETDIR -maxdepth 1 -mmin +${MINUTEOFFSET} -type f -regex ".*/${JOBNAME//\./\\.}_[0-9]*\(_?[0-9]*\)\..*" | wc -l`
25 
+	VALIDCOUNT=`find $TARGETDIR -maxdepth 1 -mmin -${MINUTEOFFSET} -type f -regex ".*/${JOBNAME//\./\\.}_[0-9]*\(_?[0-9]*\)\..*" | wc -l`
26 
+
27 
+	if [ "$OBSOLETECOUNT" -gt 0 -a "$VALIDCOUNT" -gt 1 ]
28 
+	then
29 
+		find $TARGETDIR -maxdepth 1 -mmin +${MINUTEOFFSET} -type f -regex ".*/${JOBNAME//\./\\.}_[0-9]*\(_?[0-9]*\)\..*" |
30 
+		while read LINE
31 
+		do
32 
+			if [ -f $LINE ]
33 
+			then
34 
+				rm -f $LINE &> /dev/null
35 
+			fi
36 
+		done
37 
+		echo "Cleanup of ${OBSOLETECOUNT} files completed"
38 
+		exit 0
39 
+	else
40 
+		echo "Nothing to cleanup"
41 
+		exit 0
42 
+	fi
43 
+else
44 
+	echo "Check source directory!"
45 
+	exit 1
46 
+fi
include/driver_dir
History View file @28840c7
0 47 
new file mode 100644
... ... 
@@ -0,0 +1,61 @@
1 
+#!/bin/bash
2 
+if [ ! $1 -a $2 -a $3 ]
3 
+then
4 
+	echo "Wrong parameter count."
5 
+	exit 1
6 
+fi
7 
+
8 
+#####################
9 
+#     PARAMETER     #
10 
+#####################
11 
+TARGETDIR=$2
12 
+SRCPATH=`echo $1 | awk -F:: '{ print $2 }'`
13 
+SRCDIR=`basename "${SRCPATH}"`
14 
+SRCPARENT=`dirname "${SRCPATH}"`
15 
+EXCLUDES=`echo $1 | awk -F:: '{ print $3 }'`
16 
+TSTAMP=`date +%Y%m%d`
17 
+JOBNAME=$3
18 
+
19 
+
20 
+###################
21 
+#     BACKUP      #
22 
+###################
23 
+if [ -d $SRCPATH -a -d $TARGETDIR ]
24 
+then
25 
+	# Check for existing backup file
26 
+	if [ -f "${TARGETDIR}${JOBNAME}_${TSTAMP}.tgz" ]
27 
+	then
28 
+		suffix=2
29 
+		while [ -f "${TARGETDIR}${JOBNAME}_${TSTAMP}_${suffix}.tgz" ]
30 
+		do
31 
+			suffix=`expr $suffix + 1`
32 
+		done
33 
+		BACKUPFILE="${TARGETDIR}${JOBNAME}_${TSTAMP}_${suffix}.tgz"
34 
+	else
35 
+		BACKUPFILE="${TARGETDIR}${JOBNAME}_${TSTAMP}.tgz"
36 
+	fi
37 
+	tar -czf ${BACKUPFILE} --ignore-failed-read --directory ${SRCPARENT} ${SRCDIR} &> /dev/null 2> ${BACKUPFILE}.err
38 
+	if [ $? -eq 0 ]
39 
+	then
40 
+		echo Backup $BACKUPFILE created
41 
+		if [ -f ${BACKUPFILE}.err ]
42 
+		then
43 
+			rm ${BACKUPFILE}.err &> /dev/null
44 
+		fi
45 
+		exit 0
46 
+	else
47 
+		cat ${BACKUPFILE}.err
48 
+		if [ -f $BACKUPFILE ]
49 
+		then
50 
+			rm $BACKUPFILE &> /dev/null
51 
+		fi
52 
+		if [ -f ${BACKUPFILE}.err ]
53 
+		then
54 
+			rm ${BACKUPFILE}.err &> /dev/null
55 
+		fi
56 
+		exit 1
57 
+	fi
58 
+else
59 
+	echo "Check target and source directories!"
60 
+	exit 1
61 
+fi
include/driver_ftp
History View file @28840c7
0 62 
new file mode 100644
... ... 
@@ -0,0 +1,149 @@
1 
+#!/bin/bash
2 
+if [ ! $1 -a $2 -a $3 ]
3 
+then
4 
+	echo "Wrong parameter count."
5 
+	exit 1
6 
+fi
7 
+
8 
+#####################
9 
+#     PARAMETER     #
10 
+#####################
11 
+TARGETDIR=$2
12 
+SERVER=`echo $1 | awk -F:: '{ print $2 }'`
13 
+USER=`echo $1 | awk -F:: '{ print $3 }' | awk -F: '{ print $1 }'`
14 
+PASSWD=`echo $1 | awk -F:: '{ print $3 }' | awk -F: '{ print $2 }'`
15 
+WEEKLYDOW=`echo $1 | awk -F:: '{ print $4 }'`
16 
+JOBNAME=$3
17 
+
18 
+##########################
19 
+#     PREREQUISITIES     #
20 
+##########################
21 
+
22 
+# REMOTE DIR STRUCTURE
23 
+REMOTETREE=$(mktemp)
24 
+ftp -pn $SERVER <<END_OF_SESSION
25 
+quote USER $USER
26 
+quote PASS $PASSWD
27 
+ls -R $REMOTETREE
28 
+bye
29 
+END_OF_SESSION
30 
+
31 
+grep ":$" $REMOTETREE | cut -d: -f1 | uniq | sort -o $REMOTETREE
32 
+
33 
+# REQUIRED STRUCTURE
34 
+REQTREE=$(mktemp)
35 
+echo "weekly" >  $REQTREE
36 
+echo "daily" >>  $REQTREE
37 
+
38 
+cat $REQTREE | cut -d: -f1 | uniq | sort -o $REQTREE
39 
+
40 
+# compare local and remote, list missing remote directories
41 
+MISSINGDIR=$(mktemp)
42 
+for ITEM in $(diff $REMOTETREE $REQTREE | grep "^>" | cut -d\> -f2 ) ; do
43 
+	echo "$ITEM" >> $MISSINGDIR
44 
+done
45 
+
46 
+# remove blank lines and lines where mkdir has no argument and clean up manually
47 
+sed -i '/^$/d' $MISSINGDIR
48 
+sed -i '/^mkdir$/d' $MISSINGDIR
49 
+echo "/" >> $MISSINGDIR		#harmless 'directory exists" to avoid empty file
50 
+
51 
+# create missing directories
52 
+for ITEM in $(cat $MISSINGDIR); do
53 
+	ftp -pn $SERVER <<END_OF_SESSION
54 
+	user $USER $PASSWD
55 
+	cd /
56 
+	mkdir $ITEM
57 
+	bye
58 
+END_OF_SESSION
59 
+done
60 
+
61 
+#####################
62 
+#     FUNCTIONS     #
63 
+#####################
64 
+
65 
+function upload() {
66 
+	if [ ! $1 -a $2 ]
67 
+	then
68 
+		echo "Wrong parameter count."
69 
+		exit 1
70 
+	fi
71 
+	files=$1
72 
+	dir=$2
73 
+
74 
+	## upload the files
75 
+	for ITEM in $(cat $files); do
76 
+		FILE=$(basename $ITEM)
77 
+		LOCALDIR=$(dirname $ITEM)
78 
+		REMOTEDIR=/${dir}/
79 
+
80 
+		## ftp starts here
81 
+		ftp -pn $SERVER <<END_OF_SESSION
82 
+		user $USER $PASSWD
83 
+		binary
84 
+		lcd $LOCALDIR
85 
+		cd $REMOTEDIR
86 
+		put $FILE ._$FILE.part
87 
+		rename ._$FILE.part $FILE
88 
+		bye
89 
+END_OF_SESSION
90 
+	done
91 
+}
92 
+
93 
+##############################
94 
+#     FETCH BACKUP FILES     #
95 
+##############################
96 
+if [ -d $TARGETDIR ]
97 
+then
98 
+	# Check for existing backup files
99 
+	COUNT=`find $TARGETDIR -maxdepth 1 -type f -regex ".*/${JOBNAME//\./\\.}_[0-9]*\(_?[0-9]*\)\..*" | wc -l`
100 
+
101 
+	if [ "$COUNT" -gt 0 ]
102 
+	then
103 
+		MD5FILES=$(mktemp)
104 
+
105 
+		UPLOADWEEKLY=$(mktemp)
106 
+		UPLOADDAILY=$(mktemp)
107 
+
108 
+		find $TARGETDIR -maxdepth 1 -type f -regex ".*/${JOBNAME//\./\\.}_[0-9]*\(_?[0-9]*\)\.sql" -printf '%T@ %p\n' | sort -k 1n | sed 's/^[^ ]* //' | tail -n 1 |
109 
+		while read LINE
110 
+		do
111 
+			if [ -f $LINE ]
112 
+			then
113 
+				echo $LINE >> $UPLOADWEEKLY
114 
+				echo $LINE >> $UPLOADDAILY
115 
+				md5sum $LINE | awk '{print $1}' > ${LINE}.md5
116 
+				echo ${LINE}.md5 >> $MD5FILES
117 
+			fi
118 
+		done
119 
+
120 
+
121 
+		find $TARGETDIR -maxdepth 1 -type f -regex ".*/${JOBNAME//\./\\.}_[0-9]*\(_?[0-9]*\)\.tgz" -printf '%T@ %p\n' | sort -k 1n | sed 's/^[^ ]* //' | tail -n 1 |
122 
+		while read LINE
123 
+		do
124 
+			if [ -f $LINE ]
125 
+			then
126 
+				echo $LINE >> $UPLOADWEEKLY
127 
+				echo $LINE >> $UPLOADDAILY
128 
+				md5sum $LINE | awk '{print $1}' > ${LINE}.md5
129 
+				echo ${LINE}.md5 >> $MD5FILES
130 
+			fi
131 
+		done
132 
+
133 
+		upload $UPLOADDAILY daily
134 
+		upload $MD5FILES daily
135 
+
136 
+		if [ `date "+%u"` -eq $WEEKLYDOW ] || [ `date "+%w"` -eq $WEEKLYDOW ]
137 
+		then
138 
+			upload $UPLOADWEEKLY weekly
139 
+			upload $MD5FILES weekly
140 
+		fi
141 
+
142 
+
143 
+		for ITEM in $(cat $MD5FILES); do
144 
+			rm $ITEM
145 
+		done
146 
+	fi
147 
+fi
148 
+
149 
+exit 0
include/driver_mysql
History View file @28840c7
0 150 
new file mode 100644
... ... 
@@ -0,0 +1,78 @@
1 
+#!/bin/bash
2 
+if [ ! $1 -a $2 -a $3 ]
3 
+then
4 
+	echo "Wrong parameter count."
5 
+	exit 1
6 
+fi
7 
+
8 
+#####################
9 
+#     PARAMETER     #
10 
+#####################
11 
+SERVER=`echo $1 | awk -F:: '{ print $2 }' | awk -F: '{ print $1 }'`
12 
+PORT=`echo $1 | awk -F:: '{ print $2 }' | awk -F: '{ print $2 }'`
13 
+DATABASE=`echo $1 | awk -F:: '{ print $3 }'`
14 
+USER=`echo $1 | awk -F:: '{ print $4 }'`
15 
+PASSWORD=`echo $1 | awk -F:: '{ print $5 }'`
16 
+SOCKET=`echo $1 | awk -F:: '{ print $6 }'`
17 
+
18 
+TARGETDIR=$2
19 
+TSTAMP=`date +%Y%m%d`
20 
+JOBNAME=$3
21 
+
22 
+
23 
+###################
24 
+#     BACKUP      #
25 
+###################
26 
+if [ -d $TARGETDIR ]
27 
+then
28 
+	# Set Port
29 
+	if [ "$PORT" = "" -o "$PORT" = "0" ]
30 
+	then
31 
+		PORT=3306
32 
+	fi
33 
+	# Check for existing backup file
34 
+	if [ -f "${TARGETDIR}${JOBNAME}_${TSTAMP}.sql" ]
35 
+	then
36 
+		suffix=2
37 
+		while [ -f "${TARGETDIR}${JOBNAME}_${TSTAMP}_${suffix}.sql" ]
38 
+		do
39 
+			suffix=`expr $suffix + 1`
40 
+		done
41 
+		BACKUPFILE="${TARGETDIR}${JOBNAME}_${TSTAMP}_${suffix}.sql"
42 
+	else
43 
+		BACKUPFILE="${TARGETDIR}${JOBNAME}_${TSTAMP}.sql"
44 
+	fi
45 
+
46 
+	if [ "$SOCKET" != "" ]
47 
+	then
48 
+		#mysqldump -n -B $DATABASE -u $USER -p${PASSWORD} -S $SOCKET > $BACKUPFILE 2> ${BACKUPFILE}.err
49 
+		mysqldump -n -u $USER -p${PASSWORD} -S $SOCKET -h $SERVER $DATABASE > $BACKUPFILE 2> ${BACKUPFILE}.err
50 
+	else
51 
+		#mysqldump -n -B $DATABASE -u $USER -p${PASSWORD} -P $PORT > $BACKUPFILE 2> ${BACKUPFILE}.err
52 
+		mysqldump -n -u $USER -p${PASSWORD} -P $PORT -h $SERVER $DATABASE > $BACKUPFILE 2> ${BACKUPFILE}.err
53 
+	fi
54 
+
55 
+	if [ $? -eq 0 ]
56 
+	then
57 
+		echo Dump $BACKUPFILE created
58 
+		if [ -f ${BACKUPFILE}.err ]
59 
+		then
60 
+			rm ${BACKUPFILE}.err &> /dev/null
61 
+		fi
62 
+		exit 0
63 
+	else
64 
+		cat ${BACKUPFILE}.err
65 
+		if [ -f $BACKUPFILE ]
66 
+		then
67 
+			rm $BACKUPFILE &> /dev/null
68 
+		fi
69 
+		if [ -f ${BACKUPFILE}.err ]
70 
+		then
71 
+			rm ${BACKUPFILE}.err &> /dev/null
72 
+		fi
73 
+		exit 1
74 
+	fi
75 
+else
76 
+	echo "Check target and source directories!"
77 
+	exit 1
78 
+fi
jobs/skeletton
History View file @28840c7
0 79 
new file mode 100644
... ... 
@@ -0,0 +1,23 @@
1 
+#++++++++++++++++++++++++++++++++++++++++++++++++++++++++
2 
+# BACKUP JOB
3 
+#
4 
+# This file ist intended to configure a backup job.
5 
+# Just setup the jobs to do by defining a config line.
6 
+# ----------
7 
+# MYSQL Backup
8 
+# Syntax: mysql::<server:port>::<database>::<user>::<password>[::<socket>]
9 
+#
10 
+# ----------
11 
+# DIR Backup
12 
+# Syntax: dir::<path_to_backup>[::<exclude>]
13 
+#
14 
+# ----------
15 
+# CLEAN Backups
16 
+# Syntax: clean::<number_of_days_to_keep>
17 
+#
18 
+# ----------
19 
+# FTP Backup
20 
+# Syntax: ftp::<server><user:password>::<day_of_week_for_weekly_backup_upload>
21 
+#
22 
+#++++++++++++++++++++++++++++++++++++++++++++++++++++++++
23 
+