diff --git a/ChangeLog b/ChangeLog
index 4d03cff1021519dfb5bbe0cf180f8402705cf7a6..808b1d43efaa285bc40a7fd9b326643f302d756e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,23 @@
2006-10-30 Per Cederqvist <ceder@sedesopim.lysator.liu.se>
+ Progress indication. Lock files. More output. Run ssh in batch
+ mode. Support --failed. Support a stop file.
+
+ * backup-one-task: New argument: msg. New optional argument
+ --failed. Default remoterdiff to /opt/LYSrdiff/bin/rdiff-backup,
+ and add special cases for sedesopim, home and uhumhummy. Remove
+ special case for stalingrad. Handle exclusion files. Create
+ rdiff-backup.out and backup-summary.txt. Lock the result
+ directory. Unset SSH_ASKPASS and SSH_AUTH_SOCK. Run ssh in batch
+ mode, to avoid password prompts. Print status message including
+ ETA while a backup runs. Use the --force argument to rdiff-backup
+ to make it easier to recover from an initial failure.
+
+ * backup-all: New optional argument --failed. Count the number of
+ jobs to run, and pass a running total as the msg argument of
+ backup-one-task, to get a nice progress indication. Stop cleanly
+ if the file /nobackup/backup.lysator/etc/stop is found.
+
Back up sedesopim.
* fetch-backup-work: Back up sedesopim.
diff --git a/backup-all b/backup-all
index 5bd1a0fdcd5e37f6f30102da28a3cd29e7b21e4b..51994f7a34251ddcfa81f478ff656fec41854aad 100755
--- a/backup-all
+++ b/backup-all
@@ -1,5 +1,24 @@
#!/bin/sh
+usage () {
+ echo $0: usage: $0 '[ --failed ] partno...' >&2
+}
+
+failed=
+
+while true
+do
+ case "x$1" in
+ x--failed)
+ shift
+ failed=--failed;;
+ x--*)
+ usage
+ exit 1;;
+ x*) break;;
+ esac
+done
+
PARTS="$@"
if [ "$PARTS" = "" ]
@@ -25,12 +44,27 @@ done
/nobackup/backup.lysator/bin/distribute-tasks
+total=0
+for lysrdiffpart in $PARTS
+do
+ total=`expr $total + \`wc -l </lysrdiff/$lysrdiffpart/perm/lysrdiff/tasks\``
+done
+
+ctr=1
for lysrdiffpart in $PARTS
do
while read category subcategory server serverpath
do
+ if [ -f /nobackup/backup.lysator/etc/stop ]
+ then
+ echo Stop file found. Exiting. >&2
+ exit 1
+ fi
/nobackup/backup.lysator/bin/backup-one-task \
- $lysrdiffpart "$category" "$subcategory" "$server" "$serverpath"
+ $failed \
+ $lysrdiffpart "$category" "$subcategory" "$server" "$serverpath" \
+ "$ctr/$total $category $subcategory"
+ ctr=`expr $ctr + 1`
done < /lysrdiff/$lysrdiffpart/perm/lysrdiff/tasks
done
diff --git a/backup-one-task b/backup-one-task
index cf9805fa7cbd738bd5bf5cc334347090888ba8fd..192588106a0fce2b0af733b817ed348edc4feccc 100755
--- a/backup-one-task
+++ b/backup-one-task
@@ -1,9 +1,30 @@
#!/bin/sh
-if [ $# != 5 ]
+usage () {
+ echo $0: usage: $0 [ options ] lysrdiffpart category subcategory server serverpath msg >&2
+ echo Example: $0 0 home ceder inservitus /export/home/ceder "1/123" >&2
+ echo Recognized options: >&2
+ echo ' --failed Only run failed backups' >&2
+}
+
+failed=0
+
+while [ $# -gt 1 ]
+do
+ case "x$1" in
+ x--failed)
+ shift
+ failed=1;;
+ x--*)
+ usage
+ exit 1;;
+ x*) break;;
+ esac
+done
+
+if [ $# != 6 ]
then
- echo $0: usage: $0 lysrdiffpart category subcategory server serverpath >&2
- echo Example: $0 0 home ceder inservitus /export/home/ceder >&2
+ usage
exit 1
fi
@@ -12,53 +33,120 @@ category="$2"
subcategory="$3"
server="$4"
serverpath="$5"
+msg="$6"
case "$server" in
manhattan) remoterdiff=/usr/bin/rdiff-backup;;
- stalingrad) remoterdiff=/usr/bin/rdiff-backup;;
- *) remoterdiff=/usr/local/bin/rdiff-backup;;
+ sedesopim) remoterdiff=/usr/bin/rdiff-backup;;
+ home) remoterdiff=/usr/local/bin/rdiff-backup;;
+ uhumhummy) remoterdiff=/usr/local/bin/rdiff-backup;;
+ *) remoterdiff=/opt/LYSrdiff/bin/rdiff-backup;;
esac
lysrdiff="/lysrdiff/$lysrdiffpart/perm/lysrdiff"
base="$lysrdiff/backups/$category/$subcategory"
exclude="$base"/exclude
+excl_abs="$base"/exclude.abs
files="$base"/files
+rdifflogfile="$base"/rdiff-backup.out
+summaryfile="$base"/backup-summary.txt
+lockdir="$base"/lock
state="$lysrdiff"/state
mkdir -p "$files"
+mkdir "$lockdir" || exit 1
statebase="$state"/"$category"-"$subcategory"
-#if [ ! -f "$statebase"-fail ]
-#then
-# exit 0
-#fi
+if [ $failed = 1 ] && [ ! -f "$statebase"-fail ]
+then
+ rmdir "$lockdir"
+ exit 0
+fi
if [ "$server" = manhattan ]
then
+ rmdir "$lockdir"
exit 0
fi
+unset SSH_ASKPASS
+unset SSH_AUTH_SOCK
+
+CLR='\e[2K\r'
+
+echo -ne "${msg}: fetching exclude file"
+
# Fetch an up-to-date exclude file.
rm -f "$exclude"
-scp -i /root/.ssh/backupkey "$server":"$serverpath"/.lysbackupexclude \
- "$exclude"
+scp -B -q -i /root/.ssh/backupkey \
+ "$server":"$serverpath"/.lysrdiff-exclude "$exclude" >/dev/null 2>&1
+# Create an emtpy exclude file if non existed on the server.
+touch "$exclude"
+
+sed 's%^\([-+] \|\)\([^/]\)%\1'"$serverpath"'/\2%' \
+ < "$exclude" \
+| tr '\n' '\0' > "$excl_abs"
+
+
+# See how long time the previous backup took.
+if [ -f "$statebase"-start ] && [ -f "$statebase"-end ]
+then
+ startdec="`stat --format '%Y' \"$statebase\"-start`"
+ enddec="`stat --format '%Y' \"$statebase\"-end`"
+ seconds=`expr $enddec - $startdec`
+ ETA=`date +' (ETA: %H:%M:%S)' -d "$seconds seconds"`
+else
+ ETA=""
+fi
touch "$base"/backup-attempt-start
touch "$statebase"-attempt
-if rdiff-backup --exclude-other-filesystems --null-separator \
- --remote-schema \
- 'ssh -a -k -x -i /root/.ssh/backupkey %s '$remoterdiff' --server' \
- "$server"::"$serverpath" "$files"
+echo -ne "${CLR}${msg}: running rdiff-backup$ETA"
+
+schema="ssh -o BatchMode=yes -a -k -x -i /root/.ssh/backupkey"
+schema="$schema %s $remoterdiff --server"
+
+rdiff-backup --exclude-other-filesystems --null-separator \
+ --include-globbing-filelist "$excl_abs" \
+ --remote-schema "$schema" \
+ --force \
+ "$server"::"$serverpath" "$files" > "$rdifflogfile" 2>&1
+exit=$?
+
+echo -ne "${CLR}"
+
+if [ $exit = 0 ]
then
touch "$base"/last-good-backup
mv "$base"/backup-attempt-start "$base"/last-good-start
touch "$statebase"-end
mv "$statebase"-attempt "$statebase"-start
+
+ starttime="`stat --format '%y' \"$statebase\"-start|sed 's/\..*//'`"
+ endtime="`stat --format '%y' \"$statebase\"-end|sed 's/\..*//'`"
+ startdec="`stat --format '%Y' \"$statebase\"-start`"
+ enddec="`stat --format '%Y' \"$statebase\"-end`"
+
+ echo $starttime $endtime time=`expr $enddec - $startdec` >> "$summaryfile"
+
rm -f "$base"/last-failure
rm -f "$statebase"-fail
+ if [ `wc -c < "$rdifflogfile"` != 0 ]
+ then
+ echo ${msg}: WARNING: output from rdiff-backup:
+ echo
+ sed 's/^/ /' "$rdifflogfile"
+ echo
+ fi
else
mv "$base"/backup-attempt-start "$base"/last-failure
mv "$statebase"-attempt "$statebase"-fail
+ echo ${msg}: FAIL:
+ echo
+ sed 's/^/ /' "$rdifflogfile"
+ echo
fi
+
+rmdir "$lockdir"