diff --git a/ChangeLog b/ChangeLog
index 7ead3730466988fa05f9c5f145492dc39aeeaebd..a6089a85d10a0edbef4229a576ab2ec9197ccce2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,27 @@
+2006-10-23 Per Cederqvist <ceder@lysator.liu.se>
+
+ LYSrdiff can now backup up to separate partitions, creates more
+ state timestamp files, randomizes the backup order, and backs up
+ most user data.
+
+ * fetch-backup-work: Fetch all lsvn, lyscvs, lyswww and home
+ tasks, not just a select few. Fetch roxen tasks.
+
+ * backup-all: New arguments: a list of lysrdiffpart partitions to
+ run. Check that the correct filesystems are mounted before doing
+ anything.
+
+ * backup-one-task: New argument: lysrdiffpart. That argument
+ specifies which lysrdiff partition that the backup should be
+ stored on. It is a small integer. Adjusted the paths to handle
+ it. Skip manhattan, since it is (temporarily) dead. Store state
+ information in a separate directory, and touch flag files for when
+ a backup starts and ends, fails and is being attempted.
+
+ * distribute-tasks: New script. This script is responsible for
+ assigning new tasks to lysrdiff partitions. It also randomizes
+ the order of the tasks, to make the backup more fair.
+
2006-10-14 Per Cederqvist <ceder@lysator.liu.se>
Initial checkin of the proof-of-concept scripts used to backup
diff --git a/backup-all b/backup-all
index 94473d0ec358155ed9488bbcc7a4b1693c5bbe00..5bd1a0fdcd5e37f6f30102da28a3cd29e7b21e4b 100755
--- a/backup-all
+++ b/backup-all
@@ -1,11 +1,37 @@
#!/bin/sh
-df -h /nobackup/backup.lysator/backups
+PARTS="$@"
-while read category subcategory server serverpath
+if [ "$PARTS" = "" ]
+then
+ echo $0: you forgot to specify what parts to back up >&2
+ exit 1
+fi
+
+df -h /lysrdiff/0/perm | sed -n 1p
+for lysrdiffpart in $PARTS
+do
+ df -h /lysrdiff/$lysrdiffpart/perm | sed 1d
+done
+
+for lysrdiffpart in $PARTS
+do
+ if [ "`cat /lysrdiff/$lysrdiffpart/perm/lysrdiff.id`" != "$lysrdiffpart perm" ]
+ then
+ echo lysrdiffpart $lysrdiffpart not found >&2
+ exit 1
+ fi
+done
+
+/nobackup/backup.lysator/bin/distribute-tasks
+
+for lysrdiffpart in $PARTS
do
- /nobackup/backup.lysator/bin/backup-one-task \
- "$category" "$subcategory" "$server" "$serverpath"
-done < /nobackup/backup.lysator/var/tasks
+ while read category subcategory server serverpath
+ do
+ /nobackup/backup.lysator/bin/backup-one-task \
+ $lysrdiffpart "$category" "$subcategory" "$server" "$serverpath"
+ done < /lysrdiff/$lysrdiffpart/perm/lysrdiff/tasks
+done
-df -h /nobackup/backup.lysator/backups
+df -h /lysrdiff/0/perm /lysrdiff/1/perm
diff --git a/backup-one-task b/backup-one-task
index f0c918e8413b37605fee51809ccf4920a6ac365d..cf9805fa7cbd738bd5bf5cc334347090888ba8fd 100755
--- a/backup-one-task
+++ b/backup-one-task
@@ -1,16 +1,17 @@
#!/bin/sh
-if [ $# != 4 ]
+if [ $# != 5 ]
then
- echo $0: usage: $0 category subcategory server serverpath >&2
- echo Example: $0 home ceder inservitus /export/home/ceder >&2
+ echo $0: usage: $0 lysrdiffpart category subcategory server serverpath >&2
+ echo Example: $0 0 home ceder inservitus /export/home/ceder >&2
exit 1
fi
-category="$1"
-subcategory="$2"
-server="$3"
-serverpath="$4"
+lysrdiffpart="$1"
+category="$2"
+subcategory="$3"
+server="$4"
+serverpath="$5"
case "$server" in
manhattan) remoterdiff=/usr/bin/rdiff-backup;;
@@ -18,19 +19,46 @@ case "$server" in
*) remoterdiff=/usr/local/bin/rdiff-backup;;
esac
-base="/nobackup/backup.lysator/backups/$category/$subcategory"
+lysrdiff="/lysrdiff/$lysrdiffpart/perm/lysrdiff"
+base="$lysrdiff/backups/$category/$subcategory"
exclude="$base"/exclude
files="$base"/files
+state="$lysrdiff"/state
mkdir -p "$files"
-rm -f "$exclude"
+
+statebase="$state"/"$category"-"$subcategory"
+
+#if [ ! -f "$statebase"-fail ]
+#then
+# exit 0
+#fi
+
+if [ "$server" = manhattan ]
+then
+ exit 0
+fi
# Fetch an up-to-date exclude file.
+rm -f "$exclude"
scp -i /root/.ssh/backupkey "$server":"$serverpath"/.lysbackupexclude \
"$exclude"
-rdiff-backup --exclude-other-filesystems --null-separator \
- --remote-schema \
- 'ssh -a -k -x -i /root/.ssh/backupkey %s '$remoterdiff' --server' \
- "$server"::"$serverpath" "$files" \
- && touch "$base"/last-good-backup
+touch "$base"/backup-attempt-start
+touch "$statebase"-attempt
+
+if rdiff-backup --exclude-other-filesystems --null-separator \
+ --remote-schema \
+ 'ssh -a -k -x -i /root/.ssh/backupkey %s '$remoterdiff' --server' \
+ "$server"::"$serverpath" "$files"
+then
+ touch "$base"/last-good-backup
+ mv "$base"/backup-attempt-start "$base"/last-good-start
+ touch "$statebase"-end
+ mv "$statebase"-attempt "$statebase"-start
+ rm -f "$base"/last-failure
+ rm -f "$statebase"-fail
+else
+ mv "$base"/backup-attempt-start "$base"/last-failure
+ mv "$statebase"-attempt "$statebase"-fail
+fi
diff --git a/distribute-tasks b/distribute-tasks
new file mode 100755
index 0000000000000000000000000000000000000000..067568225444f8d30d5cbb6c236f36e988714b7a
--- /dev/null
+++ b/distribute-tasks
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import random
+
+ROOT = "/lysrdiff"
+
+class JobInfo(object):
+ def __init__(self, category, subcategory, host, directory,
+ lysrdiffpart=None):
+ self.__category = category
+ self.__subcategory = subcategory
+ self.__host = host
+ self.__directory = directory
+ self.__lysrdiffpart = lysrdiffpart
+
+ def set_lysrdiffpart(self, part):
+ self.__lysrdiffpart = part
+
+ def category(self):
+ return self.__category
+
+ def subcategory(self):
+ return self.__subcategory
+
+ def host(self):
+ return self.__host
+
+ def directory(self):
+ return self.__directory
+
+ def lysrdiffpart(self):
+ return self.__lysrdiffpart
+
+ def source(self):
+ return (self.host(), self.directory())
+
+ def rdiff_dest(self):
+ return "/lysrdiff/%d/perm/lysrdiff/backups/%s/%s" % (
+ self.lysrdiffpart(), self.category(), self.subcategory())
+
+ def task_desc(self):
+ return "%s %s %s %s" % (
+ self.category(), self.subcategory(),
+ self.host(), self.directory())
+
+def newtasks():
+ return int(open("/nobackup/backup.lysator/var/newtasks").read())
+
+# key: (host, directory)
+# value: JobInfo
+tasks_per_source = {}
+
+fatal = False
+
+def tasklist_file(lysrdiffpart):
+ return "/lysrdiff/%d/perm/lysrdiff/tasks" % lysrdiffpart
+
+def parse_line(line, lysrdiffpart=None):
+ [category, subcategory, host, directory] = line.split()
+ info = JobInfo(category, subcategory, host, directory, lysrdiffpart)
+ return info
+
+def read_tasks(lysrdiffpart):
+ global fatal
+
+ for line in file(tasklist_file(lysrdiffpart), "r"):
+ info = parse_line(line, lysrdiffpart)
+
+ if (info.host(), info.directory()) in tasks_per_source:
+ sys.stderr.write("Duplicate backup detected!\n %s\n %s\n" % (
+ tasks_per_source[info.source()].task_desc(),
+ info.task_desc()))
+ fatal = True
+
+ tasks_per_source[info.source()] = info
+
+def read_new_tasks():
+ new_found = False
+ for line in file("/nobackup/backup.lysator/var/tasks"):
+ info = parse_line(line)
+ if (info.host(), info.directory()) not in tasks_per_source:
+ info.set_lysrdiffpart(newtasks())
+ tasks_per_source[(info.host(), info.directory())] = info
+ new_found = True
+
+ return new_found
+
+def write_task_lists():
+ jobs = tasks_per_source.values()
+ random.shuffle(jobs)
+ files = {}
+ for job in jobs:
+ if job.lysrdiffpart() not in files:
+ files[job.lysrdiffpart()] = file(
+ tasklist_file(job.lysrdiffpart()) + ".new", "w")
+ files[job.lysrdiffpart()].write(job.task_desc() + "\n")
+ for lysrdiffpart, fp in files.items():
+ fp.close()
+ fn = tasklist_file(lysrdiffpart)
+ os.rename(fn + ".new", fn)
+
+def main():
+ global fatal
+
+ for lysrdiffpart in range(2):
+ read_tasks(lysrdiffpart)
+ read_new_tasks()
+ if not fatal:
+ write_task_lists()
+
+if __name__ == '__main__':
+ main()
diff --git a/fetch-backup-work b/fetch-backup-work
index 9fa70b11184938b5a99b3988f3864bed81467cf6..1f0ab021630d5608ef09c73914dc217fdb79c804 100755
--- a/fetch-backup-work
+++ b/fetch-backup-work
@@ -4,14 +4,14 @@ rm /nobackup/backup.lysator/var/tasks
ssh stalingrad ls -1 /svnroot \
| sort \
- | head -10 \
- | awk '{print "lsvn", $1, "stalingrad /svnroot/" $1}' \
+ | awk '$1 == "lost+found" { next }
+ {print "lsvn", $1, "stalingrad /svnroot/" $1}' \
>> /nobackup/backup.lysator/var/tasks
ssh stalingrad ls -1 /cvsroot \
| sort \
- | head -10 \
- | awk '{print "lyscvs", $1, "stalingrad /cvsroot/" $1}' \
+ | awk '$1 == "lost+found" { next }
+ {print "lyscvs", $1, "stalingrad /cvsroot/" $1}' \
>> /nobackup/backup.lysator/var/tasks
ssh uhumhummy ls -1 /lysator/lyswww/users/common \
@@ -19,17 +19,42 @@ ssh uhumhummy ls -1 /lysator/lyswww/users/common \
| grep -v THIS_IS_USERWEB \
| grep -v '^cap\.fm$' \
| sort \
- | grep -v '^a' \
- | grep -v '^b' \
- | head -50 \
| awk '{print "lyswww", $1, "uhumhummy /lysator/lyswww/users/common/" $1}' \
>> /nobackup/backup.lysator/var/tasks
+ssh uhumhummy ls -1 /lysator/lyswww/users/roxen_only \
+ | sort \
+ | awk '{print "roxen", $1, "uhumhummy /lysator/lyswww/users/roxen_only/" $1}'\
+>> /nobackup/backup.lysator/var/tasks
+
ssh nema.lysator.liu.se ypcat -k auto_home \
| sort \
| grep -v DELETE \
| grep -v '^ *$' \
- | head -75 \
| sed 's/:/ /' \
+ | awk '$1 == "mailman" { next }
+ $1 == "patrick" { next }
+ $1 == "snapshot" { next }
+ $1 == "u1" { next }
+ $1 == "u2" { next }
+ $1 == "u3" { next }
+ $1 == "u4" { next }
+ $1 == "u5" { next }
+ $1 == "u6" { next }
+ $1 == "lysdisk1" { next }
+ $1 == "lysdisk2" { next }
+ $1 == "lysdisk3" { next }
+ $1 == "lysdisk4" { next }
+ $1 == "lysdisk5" { next }
+ $1 == "lysdisk6" { next }
+ $1 == "lysdisk7" { next }
+ $1 == "lysdisk8" { next }
+ $1 == "lysdisk9" { next }
+ $1 == "lysdisk10" { next }
+ $1 == "lysdisk11" { next }
+ $1 == "kheldar" { next }
+ $1 == "quota1" { next }
+ $1 == "sge" { next }
+ { print }' \
| sed 's/^/home /' \
>> /nobackup/backup.lysator/var/tasks