aboutsummaryrefslogtreecommitdiff
path: root/handlers/maildir.in
blob: ab72ce1531b379ef2030fc938bd9ee3f8b1519fd (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
# -*- mode: sh; sh-basic-offset: 3; indent-tabs-mode: nil; -*-
# vim: set filetype=sh sw=3 sts=3 expandtab autoindent:

###############################################################
#
#  This handler slowly creates a backup of each user's maildir
#  to a remote server. It is designed to be run with low overhead
#  in terms of cpu and bandwidth so it runs pretty slow.
#  Hardlinking is used to save storage space.
#
#  This handler expects that your maildir directory structure is
#  either one of the following:
#
#  1. /$srcdir/[a-zA-Z0-9]/$user for example:
#  /var/maildir/a/anarchist
#  /var/maildir/a/arthur
#  ...
#  /var/maildir/Z/Zaphod
#  /var/maildir/Z/Zebra
#
#  2. or the following:
#  /var/maildir/domain.org/user1
#  /var/maildir/domain.org/user2
#  ...
#  /var/maildir/anotherdomain.org/user1
#  /var/maildir/anotherdomain.org/user2
#  ...
#
#  if the configuration is setup to have keepdaily at 3,
#  keepweekly is 2, and keepmonthly is 1, then each user's
#  maildir backup snapshot directory will contain these files:
#    daily.1
#    daily.2
#    daily.3
#    weekly.1
#    weekly.2
#    monthly.1
#
#  The basic algorithm is to rsync each maildir individually,
#  and to use hard links for retaining historical data.
#
#  We handle each maildir individually because it becomes very
#  unweldy to hardlink and rsync many hundreds of thousands
#  of files at once. It is much faster to take on smaller
#  chunks at a time.
#
#  For the backup rotation to work, destuser must be able to run
#  arbitrary bash commands on the desthost.
#
#  Any maildir which is deleted from the source will be moved to
#  "deleted" directory in the destination. It is up to you to
#  periodically remove this directory or old maildirs in it.
#
##############################################################

getconf rotate yes
getconf remove yes
getconf backup yes

getconf loadlimit 5
getconf speedlimit 0
getconf keepdaily 5
getconf keepweekly 3
getconf keepmonthly 1

getconf srcdir /var/maildir
getconf destdir
getconf desthost
getconf destport 22
getconf destuser
getconf destid_file /root/.ssh/id_rsa

getconf multiconnection notset

failedcount=0
# strip trailing /
destdir=${destdir%/}
srcdir=${srcdir%/}

[ -d $srcdir ] || fatal "source directory $srcdir doesn't exist"

[ "$multiconnection" == "notset" ] && fatal "The maildir handler uses a very different destination format. See the example .maildir for more information"

if [ $test ]; then
   testflags="--dry-run -v"
fi

rsyncflags="$testflags -e 'ssh -p $destport -i $destid_file' -r -v --ignore-existing --delete --size-only --bwlimit=$speedlimit"
excludes="--exclude '.Trash/\*' --exclude '.Mistakes/\*' --exclude '.Spam/\*'"

##################################################################
### FUNCTIONS

function do_user() {
   local user=$1
   local btype=$2
   local userdir=${3%/}
   local source="$srcdir/$userdir/$user/"
   local target="$destdir/$userdir/$user/$btype.1"
   if [ ! -d $source ]; then
      warning "maildir $source not found"
      return
   fi

   debug "syncing"
   ret=`$RSYNC -e "ssh -p $destport -i $destid_file" -r \
      --links --ignore-existing --delete --size-only --bwlimit=$speedlimit \
      --exclude '.Trash/*' --exclude '.Mistakes/*' --exclude '.Spam/*' \
      $source $destuser@$desthost:$target \
      2>&1`
   ret=$?
   # ignore 0 (success) and 24 (file vanished before it could be copied)
   if [ $ret != 0 -a $ret != 24 ]; then
      warning "rsync $user failed"
      warning "  returned: $ret"
      let "failedcount = failedcount + 1"
      if [ $failedcount -gt 100 ]; then
         fatal "100 rsync errors -- something is not working right. bailing out."
      fi
   fi
   ssh -o PasswordAuthentication=no $desthost -l $destuser -i $destid_file "date +%c%n%s > $target/created"
}

# remove any maildirs from backup which might have been deleted
# and add new ones which have just been created.
# (actually, it just moved them to the directory "deleted")

function do_remove() {
   local tmp1=`maketemp maildir-tmp-file`
   local tmp2=`maketemp maildir-tmp-file`

   ssh -p $destport -i $destid_file $destuser@$desthost mkdir -p "$destdir/deleted"
      cd "$srcdir"
      for userdir in `ls -d1 */`; do
         ls -1 "$srcdir/$userdir" | sort > $tmp1
         ssh -p $destport -i $destid_file $destuser@$desthost ls -1 "$destdir/$userdir" | sort > $tmp2
      for deluser in `join -v 2 $tmp1 $tmp2`; do
         [ "$deluser" != "" ] || continue
         info "removing $destuser@$desthost:$destdir/$userdir$deluser/"
         ssh -p $destport -i $destid_file $destuser@$desthost mv "$destdir/$userdir$deluser/" "$destdir/deleted"
         ssh -p $destport -i $destid_file $destuser@$desthost "date +%c%n%s > '$destdir/deleted/$deluser/deleted_on'"
      done
   done
   rm $tmp1
   rm $tmp2
}

function do_rotate() {
   [ "$rotate" == "yes" ] || return;
   local user=$1
   local userdir=${2%/}
   local backuproot="$destdir/$userdir/$user"
(
   ssh -T -o PasswordAuthentication=no $desthost -l $destuser -i $destid_file <<EOF
##### BEGIN REMOTE SCRIPT #####
   seconds_daily=86400
   seconds_weekly=604800
   seconds_monthly=2628000
   keepdaily=$keepdaily
   keepweekly=$keepweekly
   keepmonthly=$keepmonthly
   now=\`date +%s\`

   if [ ! -d "$backuproot" ]; then
      echo "Debug: skipping rotate of $user. $backuproot doesn't exist."
      exit
   fi
   for rottype in daily weekly monthly; do
      seconds=\$((seconds_\${rottype}))

      dir="$backuproot/\$rottype"
      if [ ! -d \$dir.1 ]; then
         echo "Debug: \$dir.1 does not exist, skipping."
         continue 1
      elif [ ! -f \$dir.1/created ]; then
         echo "Warning: \$dir.1/created does not exist. This backup may be only partially completed. Skipping rotation."
         continue 1
      fi

      # Rotate the current list of backups, if we can.
      oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\`
      #echo "Debug: oldest \$oldest"
      [ "\$oldest" == "" ] && oldest=0
      for (( i=\$oldest; i > 0; i-- )); do
         if [ -d \$dir.\$i ]; then
            if [ -f \$dir.\$i/created ]; then
               created=\`tail -1 \$dir.\$i/created\`
            else
               created=0
            fi
            cutoff_time=\$(( now - (seconds*(i-1)) ))
            if [ ! \$created -gt \$cutoff_time ]; then
               next=\$(( i + 1 ))
               if [ ! -d \$dir.\$next ]; then
                  echo "Debug: \$rottype.\$i --> \$rottype.\$next"
                  mv \$dir.\$i \$dir.\$next
                  date +%c%n%s > \$dir.\$next/rotated
               else
                  echo "Debug: skipping rotation of \$dir.\$i because \$dir.\$next already exists."
               fi
            else
               echo "Debug: skipping rotation of \$dir.\$i because it was created" \$(( (now-created)/86400)) "days ago ("\$(( (now-cutoff_time)/86400))" needed)."
            fi
         fi
      done
   done

   max=\$((keepdaily+1))
   if [ \( \$keepweekly -gt 0 -a -d $backuproot/daily.\$max \) -a ! -d $backuproot/weekly.1 ]; then
      echo "Debug: daily.\$max --> weekly.1"
      mv $backuproot/daily.\$max $backuproot/weekly.1
      date +%c%n%s > $backuproot/weekly.1/rotated
   fi

   max=\$((keepweekly+1))
   if [ \( \$keepmonthly -gt 0 -a -d $backuproot/weekly.\$max \) -a ! -d $backuproot/monthly.1 ]; then
      echo "Debug: weekly.\$max --> monthly.1"
      mv $backuproot/weekly.\$max $backuproot/monthly.1
      date +%c%n%s > $backuproot/monthly.1/rotated
   fi

   for rottype in daily weekly monthly; do
      max=\$((keep\${rottype}+1))
      dir="$backuproot/\$rottype"
      oldest=\`find $backuproot -maxdepth 1 -type d -name \$rottype'.*' | @SED@ 's/^.*\.//' | sort -n | tail -1\`
      [ "\$oldest" == "" ] && oldest=0
      # if we've rotated the last backup off the stack, remove it.
      for (( i=\$oldest; i >= \$max; i-- )); do
         if [ -d \$dir.\$i ]; then
            if [ -d $backuproot/rotate.tmp ]; then
               echo "Debug: removing rotate.tmp"
               rm -rf $backuproot/rotate.tmp
            fi
            echo "Debug: moving \$rottype.\$i to rotate.tmp"
            mv \$dir.\$i $backuproot/rotate.tmp
         fi
      done
   done
####### END REMOTE SCRIPT #######
EOF
) | (while read a; do passthru $a; done)

}


function setup_remote_dirs() {
   local user=$1
   local backuptype=$2
   local userdir=${3%/}
   local dir="$destdir/$userdir/$user/$backuptype"
   local tmpdir="$destdir/$userdir/$user/rotate.tmp"
(
   ssh -T -o PasswordAuthentication=no $desthost -l $destuser -i $destid_file <<EOF
      if [ ! -d $destdir ]; then
         echo "Fatal: Destination directory $destdir does not exist on host $desthost."
         exit 1
      elif [ -d $dir.1 ]; then
         if [ -f $dir.1/created ]; then
            echo "Warning: $dir.1 already exists. Overwriting contents."
         else
            echo "Warning: we seem to be resuming a partially written $dir.1"
         fi
      else
         if [ -d $tmpdir ]; then
            mv $tmpdir $dir.1
            if [ \$? == 1 ]; then
               echo "Fatal: could mv $destdir/rotate.tmp $dir.1 on host $desthost"
               exit 1
            fi
         else
            mkdir --parents $dir.1
            if [ \$? == 1 ]; then
               echo "Fatal: could not create directory $dir.1 on host $desthost"
               exit 1
            fi
         fi
         if [ -d $dir.2 ]; then
            echo "Debug: update links $backuptype.2 --> $backuptype.1"
            cp -alf $dir.2/. $dir.1
            #if [ \$? == 1 ]; then
            #   echo "Fatal: could not create hard links to $dir.1 on host $desthost"
            #   exit 1
            #fi
         fi
      fi
      [ -f $dir.1/created ] && rm $dir.1/created
      [ -f $dir.1/rotated ] && rm $dir.1/rotated
      exit 0
EOF
) | (while read a; do passthru $a; done)

   if [ $? == 1 ]; then exit; fi
}

function start_mux() {
   if [ "$multiconnection" == "yes" ]; then
      debug "Starting dummy ssh connection"
      ssh -p $destport -i $destid_file $destuser@$desthost sleep 1d &
      sleep 1
   fi
}

function end_mux() {
   if [ "$multiconnection" == "yes" ]; then
      debug "Stopping dummy ssh connection"
      ssh -p $destport -i $destid_file $destuser@$desthost pkill sleep
   fi
}

###
##################################################################

# see if we can login
debug "ssh -o PasswordAuthentication=no $desthost -l $destuser -i $destid_file 'echo -n 1'"
if [ ! $test ]; then
   result=`ssh -o PasswordAuthentication=no $desthost -l $destuser -i $destid_file 'echo -n 1' 2>&1`
   if [ "$result" != "1" ]; then
      fatal "Can't connect to $desthost as $destuser using $destid_file."
   fi
fi

end_mux
start_mux

## SANITY CHECKS ##
status=`ssh -p $destport -i $destid_file $destuser@$desthost "[ -d \"$destdir\" ] && echo 'ok'"`
if [ "$status" != "ok" ]; then
   end_mux
   fatal "Destination directory $destdir doesn't exist!"
   exit
fi

### REMOVE OLD MAILDIRS ###

if [ "$remove" == "yes" ]; then
   do_remove
fi

### MAKE BACKUPS ###

if [ "$backup" == "yes" ]; then
   if [ $keepdaily -gt 0 ]; then btype=daily
   elif [ $keepweekly -gt 0 ]; then btype=weekly
   elif [ $keepmonthly -gt 0 ]; then btype=monthly
   else fatal "keeping no backups"; fi

   if [ "$testuser" != "" ]; then
      cd "$srcdir/${user:0:1}"
      do_rotate $testuser
      setup_remote_dirs $testuser $btype
      do_user $testuser $btype
   else
      [ -d "$srcdir" ] || fatal "directory $srcdir not found."
      cd "$srcdir"
      for userdir in `ls -d1 */`; do
         [ -d "$srcdir/$userdir" ] || fatal "directory $srcdir/$userdir not found."
         cd "$srcdir/$userdir"
         debug $userdir
         for user in `ls -1`; do
            [ "$user" != "" ] || continue
            debug "$user $userdir"
            do_rotate $user $userdir
            setup_remote_dirs $user $btype $userdir
            do_user $user $btype $userdir
         done
      done
   fi
fi

end_mux