aboutsummaryrefslogtreecommitdiff
path: root/handlers/maildir
blob: b678b838797bb4655c613f58850a8e0d87f3f4ab (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
# -*- mode: sh; sh-basic-offset: 8; indent-tabs-mode: nil; -*-

###############################################################
#
#  This handler slowly creates a backup of each user's maildir
#  to a remote server. It is designed to be run with low overhead
#  in terms of cpu and bandwidth so it runs pretty slow.
#
#  if destdir is /backup/maildir/, then it will contain the files
#    daily.1
#    daily.2
#    daily.3
#    weekly.1
#    weekly.2
#    monthly.1
#  if keepdaily is 3, keepweekly is 2, and keepmonthly is 1. 
#
#  The basic algorithm is to rsync each maildir individually,
#  and to use hard links for retaining historical data.
#
#  We rsync each maildir individually because it becomes very
#  unweldy to start a single rsync of many hundreds of thousands
#  of files. 
#
#  For the backup rotation to work, destuser must be able to run 
#  arbitrary bash commands on the desthost.
#
##############################################################

getconf rotate yes
getconf remove yes

getconf loadlimit 5
getconf speedlimit 0
getconf keepdaily 5
getconf keepweekly 3
getconf keepmonthly 1

getconf srcdir /var/maildir
getconf destdir
getconf desthost
getconf destport 22
getconf destuser

failedcount=0

# strip trailing /
destdir=${destdir%/}
srcdir=${srcdir%/}

# used for testing
#getconf letter
#getconf testuser elijah
getconf backup yes
#letters=e
letters="a b c d e f g h i j k l m n o p q r s t u v w x y z"

[ -d $srcdir ] || fatal "source directory $srcdir doesn't exist"

[ ! $test ] || testflags="--dry-run -v"
rsyncflags="$testflags -e 'ssh -p $destport' -r -v --ignore-existing --delete --size-only --bwlimit=$speedlimit"
excludes="--exclude '.Trash/\*' --exclude '.Mistakes/\*' --exclude '.Spam/\*'"

# see if we can login
debug "ssh -o PasswordAuthentication=no $desthost -l $destuser 'echo -n 1'"
if [ ! $test ]; then
	result=`ssh -o PasswordAuthentication=no $desthost -l $destuser 'echo -n 1' 2>&1`
	if [ "$result" != "1" ]; then
		fatal "Can't connect to $desthost as $destuser."
	fi
fi

##################################################################
### FUNCTIONS

function do_user() {
	local user=$1
	local destdir=$2
	local letter=${user:0:1}
	local dir="$srcdir/$letter/$user"
	[ -d $dir ] || fatal "maildir $dir not found".

#	while true; do
#		load=`uptime | sed 's/^.*load average: \\([^,]*\\).*$/\\1/'`
#		over=`expr $load \> $loadlimit`
#		if [ $over == 1 ]; then
#			info "load $load, sleeping..."
#			sleep 600
#		else
#			break
#		fi
#	done
	
  	cmd="$RSYNC $rsyncflags $excludes $dir $destuser@$desthost:$destdir/$letter"
	ret=`rsync -e "ssh -p $destport" -r \
--links --ignore-existing --delete --size-only --bwlimit=$speedlimit \
--exclude '.Trash/*' --exclude '.Mistakes/*' --exclude '.Spam/*' \
$dir $destuser@$desthost:$destdir/$letter \
2>&1`
	ret=$?
	# ignore 0 (success) and 24 (file vanished before it could be copied)
	if [ $ret != 0 -a $ret != 24 ]; then
		warning "rsync $user failed"
		warning "  returned: $ret"
		let "failedcount = failedcount + 1"
		if [ $failedcount -gt 100 ]; then
			fatal "100 rsync errors -- something is not working right. bailing out."
		fi
	fi
}

# remove any maildirs from backup which might have been deleted
# and add new ones which have just been created.

function do_remove() {
	local tmp1=`maketemp maildir-tmp-file`
	local tmp2=`maketemp maildir-tmp-file`
	
	for i in a b c d e f g h i j k l m n o p q r s t u v w x y z; do
		ls -1 "$srcdir/$i" | sort > $tmp1
		ssh -p $destport $desthost ls -1 '$destdir/maildir/$i' | sort > $tmp2
		for deluser in `join -v 2 $tmp1 $tmp2`; do
			cmd="ssh -p $destport $desthost rm -vr '$destdir/maildir/$i/$deluser/'"
			debug $cmd
		done
	done
	rm $tmp1
	rm $tmp2	
}

function do_rotate() {
	backuproot=$destdir

(
	debug Connecting to $desthost
	ssh -T -o PasswordAuthentication=no $desthost -l $destuser <<EOF
##### BEGIN REMOTE SCRIPT #####
	seconds_daily=86400
	seconds_weekly=604800
	seconds_monthly=2628000
	keepdaily=$keepdaily
	keepweekly=$keepweekly
	keepmonthly=$keepmonthly
	now=\`date +%s\`

	for rottype in daily weekly monthly; do
		seconds=\$((seconds_\${rottype}))

		dir="$backuproot/\$rottype"
		if [ ! -d \$dir.1 ]; then
			echo "Info: \$dir.1 does not exist. This backup is missing, so we are skipping the rotation."
			continue 1
		elif [ ! -f \$dir.1/created ]; then
			echo "Warning: \$dir.1/created does not exist. This backup may be only partially completed. Skipping rotation."
			continue 1
		fi
		
		# Rotate the current list of backups, if we can.
		oldest=\`find $backuproot -type d -maxdepth 1 -name \$rottype'.*' | sed 's/^.*\.//' | sort -n | tail -1\`
		echo "Debug: oldest \$oldest"
		[ "\$oldest" == "" ] && oldest=0
		for (( i=\$oldest; i > 0; i-- )); do
			if [ -d \$dir.\$i ]; then
				if [ -f \$dir.\$i/created ]; then
					created=\`tail -1 \$dir.\$i/created\`
				else
					created=0
				fi
				cutoff_time=\$(( now - (seconds*(i-1)) ))
				if [ ! \$created -gt \$cutoff_time ]; then
					next=\$(( i + 1 ))
					if [ ! -d \$dir.\$next ]; then
						echo "Debug: mv \$dir.\$i \$dir.\$next"
						mv \$dir.\$i \$dir.\$next
						date +%c%n%s > \$dir.\$next/rotated
					else
						echo "Info: skipping rotation of \$dir.\$i because \$dir.\$next already exists."
					fi
				else
					echo "Info: skipping rotation of \$dir.\$i because it was created" \$(( (now-created)/86400)) "days ago ("\$(( (now-cutoff_time)/86400))" needed)."
				fi
			fi
		done
	done

	max=\$((keepdaily+1))
	if [ \( \$keepweekly -gt 0 -a -d $backuproot/daily.\$max \) -a ! -d $backuproot/weekly.1 ]; then
		echo mv $backuproot/daily.\$max $backuproot/weekly.1
		mv $backuproot/daily.\$max $backuproot/weekly.1
		date +%c%n%s > $backuproot/weekly.1/rotated
	fi

	max=\$((keepweekly+1))
	if [ \( \$keepmonthly -gt 0 -a -d $backuproot/weekly.\$max \) -a ! -d $backuproot/monthly.1 ]; then
		echo mv $backuproot/weekly.\$max $backuproot/monthly.1
		mv $backuproot/weekly.\$max $backuproot/monthly.1
		date +%c%n%s > $backuproot/monthly.1/rotated
	fi

	for rottype in daily weekly monthly; do
		max=\$((keep\${rottype}+1))
		dir="$backuproot/\$rottype"
		oldest=\`find $backuproot -type d -maxdepth 1 -name \$rottype'.*' | sed 's/^.*\.//' | sort -n | tail -1\`
		[ "\$oldest" == "" ] && oldest=0 
		# if we've rotated the last backup off the stack, remove it.
		for (( i=\$oldest; i >= \$max; i-- )); do
			if [ -d \$dir.\$i ]; then
				if [ -d $backuproot/rotate.tmp ]; then
					echo "Info: removing $backuproot/rotate.tmp"
					rm -rf $backuproot/rotate.tmp
				fi
				echo "Info: moving \$dir.\$i to $backuproot/rotate.tmp"
				mv \$dir.\$i $backuproot/rotate.tmp
			fi
		done
	done
####### END REMOTE SCRIPT #######
EOF
) | (while read a; do passthru $a; done)

}


function setup_remote_dirs() {
	local backuptype=$1
	local dir="$destdir/$backuptype"

(
	ssh -T -o PasswordAuthentication=no $desthost -l $destuser <<EOF
		if [ ! -d $destdir ]; then
			echo "Fatal: Destination directory $destdir does not exist on host $desthost."
			exit 1
		elif [ -d $dir.1 ]; then
			if [ -f $dir.1/created ]; then
				echo "Warning: $dir.1 already exists. Overwriting contents."
			else
				echo "Warning: we seem to be resuming a partially written $dir.1"
			fi
		else
			if [ -d $destdir/rotate.tmp ]; then
				mv $destdir/rotate.tmp $dir.1
				if [ \$? == 1 ]; then
					echo "Fatal: could mv $destdir/rotate.tmp $dir.1 on host $desthost"
					exit 1
				fi
			else
				mkdir $dir.1
				if [ \$? == 1 ]; then
					echo "Fatal: could not create directory $dir.1 on host $desthost"
					exit 1
				fi
				for i in a b c d e f g h i j k l m n o p q r s t u v w y x z; do
					mkdir $dir.1/\$i
				done
			fi
			if [ -d $destdir/$backuptype.2 ]; then
				echo "Info: updating hard links to $dir.1. This may take a while."
				cp -alf $destdir/$backuptype.2/. $dir.1
				#if [ \$? == 1 ]; then
				#	echo "Fatal: could not create hard links to $dir.1 on host $desthost"
				#	exit 1
				#fi
			fi
		fi
		[ -f $dir.1/created ] && rm $dir.1/created
		[ -f $dir.1/rotated ] && rm $dir.1/rotated
		exit 0
EOF
) | (while read a; do passthru $a; done)

	if [ $? == 1 ]; then exit; fi
}

###
##################################################################

### ROTATE BACKUPS ###

if [ "$rotate" == "yes" ]; then
	do_rotate
fi

### REMOVE OLD MAILDIRS ###

if [ "$remove" == "yes" ]; then
	debug remove
fi

### MAKE BACKUPS ###

if [ "$backup" == "yes" ]; then
	if [ $keepdaily -gt 0 ]; then btype=daily
	elif [ $keepweekly -gt 0 ]; then btype=weekly
	elif [ $keepmonthly -gt 0 ]; then btype=monthly
	else fatal "keeping no backups"; fi

	setup_remote_dirs $btype
	
	for i in $letters; do
		[ -d "$srcdir/$i" ] || fatal "directory $srcdir/$i not found."
		cd "$srcdir/$i"
		debug $i
		for user in `ls -1`; do
			if [ "$testuser" != "" -a "$testuser" != "$user" ]; then continue; fi
			do_user $user $destdir/$btype.1
		done
	done

	ssh -o PasswordAuthentication=no $desthost -l $destuser "date +%c%n%s > $destdir/$btype.1/created"
fi