More backups, to separate partitions.

LYSrdiff can now backup up to separate partitions, creates more state
timestamp files, randomizes the backup order, and backs up most user
data.

* fetch-backup-work: Fetch all lsvn, lyscvs, lyswww and home tasks,
  not just a select few.  Fetch roxen tasks.

* backup-all: New arguments: a list of lysrdiffpart partitions to run.
  Check that the correct filesystems are mounted before doing
  anything.

* backup-one-task: New argument: lysrdiffpart.  That argument
  specifies which lysrdiff partition that the backup should be stored
  on.  It is a small integer.  Adjusted the paths to handle it.  Skip
  manhattan, since it is (temporarily) dead.  Store state information
  in a separate directory, and touch flag files for when a backup
  starts and ends, fails and is being attempted.

* distribute-tasks: New script.  This script is responsible for
  assigning new tasks to lysrdiff partitions.  It also randomizes the
  order of the tasks, to make the backup more fair.

ChangeLog

+2006-10-23  Per Cederqvist  <ceder@lysator.liu.se>
+
+	LYSrdiff can now backup up to separate partitions, creates more
+	state timestamp files, randomizes the backup order, and backs up
+	most user data.
+
+	* fetch-backup-work: Fetch all lsvn, lyscvs, lyswww and home
+	tasks, not just a select few.  Fetch roxen tasks.
+
+	* backup-all: New arguments: a  list of lysrdiffpart partitions to
+	run.  Check that the correct filesystems are mounted before doing
+	anything.
+
+	* backup-one-task: New argument: lysrdiffpart.  That argument
+	specifies which lysrdiff partition that the backup should be
+	stored on.  It is a small integer.  Adjusted the paths to handle
+	it.  Skip manhattan, since it is (temporarily) dead.  Store state
+	information in a separate directory, and touch flag files for when
+	a backup starts and ends, fails and is being attempted.
+
+	* distribute-tasks: New script.  This script is responsible for
+	assigning new tasks to lysrdiff partitions.  It also randomizes
+	the order of the tasks, to make the backup more fair.
+
 2006-10-14  Per Cederqvist  <ceder@lysator.liu.se>
 
 	Initial checkin of the proof-of-concept scripts used to backup

backup-all

 #!/bin/sh
 
-df -h /nobackup/backup.lysator/backups
+PARTS="$@"
 
-while read category subcategory server serverpath
+if [ "$PARTS" = "" ]
+then
+  echo $0: you forgot to specify what parts to back up >&2
+  exit 1
+fi
+
+df -h /lysrdiff/0/perm | sed -n 1p
+for lysrdiffpart in $PARTS
+do
+  df -h /lysrdiff/$lysrdiffpart/perm | sed 1d
+done
+
+for lysrdiffpart in $PARTS
+do
+  if [ "`cat /lysrdiff/$lysrdiffpart/perm/lysrdiff.id`" != "$lysrdiffpart perm" ]
+  then
+    echo lysrdiffpart $lysrdiffpart not found >&2
+    exit 1
+  fi
+done
+
+/nobackup/backup.lysator/bin/distribute-tasks
+
+for lysrdiffpart in $PARTS
 do
-  /nobackup/backup.lysator/bin/backup-one-task \
-	 "$category" "$subcategory" "$server" "$serverpath"
-done < /nobackup/backup.lysator/var/tasks
+  while read category subcategory server serverpath
+  do
+    /nobackup/backup.lysator/bin/backup-one-task \
+      $lysrdiffpart "$category" "$subcategory" "$server" "$serverpath"
+  done < /lysrdiff/$lysrdiffpart/perm/lysrdiff/tasks
+done
 
-df -h /nobackup/backup.lysator/backups
+df -h /lysrdiff/0/perm /lysrdiff/1/perm

backup-one-task

 #!/bin/sh
 
-if [ $# != 4 ]
+if [ $# != 5 ]
 then
-    echo $0: usage: $0 category subcategory server serverpath >&2
-    echo Example: $0 home ceder inservitus /export/home/ceder >&2
+    echo $0: usage: $0 lysrdiffpart category subcategory server serverpath >&2
+    echo Example: $0 0 home ceder inservitus /export/home/ceder >&2
     exit 1
 fi
 
-category="$1"
-subcategory="$2"
-server="$3"
-serverpath="$4"
+lysrdiffpart="$1"
+category="$2"
+subcategory="$3"
+server="$4"
+serverpath="$5"
 
 case "$server" in
     manhattan) remoterdiff=/usr/bin/rdiff-backup;;
@@ -18,19 +19,46 @@ case "$server" in
     *) remoterdiff=/usr/local/bin/rdiff-backup;;
 esac
 
-base="/nobackup/backup.lysator/backups/$category/$subcategory"
+lysrdiff="/lysrdiff/$lysrdiffpart/perm/lysrdiff"
+base="$lysrdiff/backups/$category/$subcategory"
 exclude="$base"/exclude
 files="$base"/files
+state="$lysrdiff"/state
 
 mkdir -p "$files"
-rm -f "$exclude"
+
+statebase="$state"/"$category"-"$subcategory"
+
+#if [ ! -f "$statebase"-fail ]
+#then
+#    exit 0
+#fi
+
+if [ "$server" = manhattan ]
+then
+    exit 0
+fi
 
 # Fetch an up-to-date exclude file.
+rm -f "$exclude"
 scp -i /root/.ssh/backupkey "$server":"$serverpath"/.lysbackupexclude \
      "$exclude"
 
-rdiff-backup --exclude-other-filesystems --null-separator \
- --remote-schema \
-    'ssh -a -k -x -i /root/.ssh/backupkey %s '$remoterdiff' --server' \
-    "$server"::"$serverpath" "$files" \
- && touch "$base"/last-good-backup
+touch "$base"/backup-attempt-start
+touch "$statebase"-attempt
+
+if rdiff-backup --exclude-other-filesystems --null-separator \
+    --remote-schema \
+      'ssh -a -k -x -i /root/.ssh/backupkey %s '$remoterdiff' --server' \
+    "$server"::"$serverpath" "$files"
+then
+    touch "$base"/last-good-backup
+    mv "$base"/backup-attempt-start "$base"/last-good-start
+    touch "$statebase"-end
+    mv "$statebase"-attempt "$statebase"-start
+    rm -f "$base"/last-failure
+    rm -f "$statebase"-fail
+else
+    mv "$base"/backup-attempt-start "$base"/last-failure
+    mv "$statebase"-attempt "$statebase"-fail
+fi

distribute-tasks

+#!/usr/bin/env python
+
+import os
+import sys
+import random
+
+ROOT = "/lysrdiff"
+
+class JobInfo(object):
+    def __init__(self, category, subcategory, host, directory,
+                 lysrdiffpart=None):
+        self.__category = category
+        self.__subcategory = subcategory
+        self.__host = host
+        self.__directory = directory
+        self.__lysrdiffpart = lysrdiffpart
+
+    def set_lysrdiffpart(self, part):
+        self.__lysrdiffpart = part
+
+    def category(self):
+        return self.__category
+
+    def subcategory(self):
+        return self.__subcategory
+
+    def host(self):
+        return self.__host
+
+    def directory(self):
+        return self.__directory
+
+    def lysrdiffpart(self):
+        return self.__lysrdiffpart
+
+    def source(self):
+        return (self.host(), self.directory())
+
+    def rdiff_dest(self):
+        return "/lysrdiff/%d/perm/lysrdiff/backups/%s/%s" % (
+            self.lysrdiffpart(), self.category(), self.subcategory())
+
+    def task_desc(self):
+        return "%s %s %s %s" % (
+            self.category(), self.subcategory(),
+            self.host(), self.directory())
+
+def newtasks():
+    return int(open("/nobackup/backup.lysator/var/newtasks").read())
+
+# key: (host, directory)
+# value: JobInfo
+tasks_per_source = {}
+
+fatal = False
+
+def tasklist_file(lysrdiffpart):
+    return "/lysrdiff/%d/perm/lysrdiff/tasks" % lysrdiffpart
+
+def parse_line(line, lysrdiffpart=None):
+    [category, subcategory, host, directory] = line.split()
+    info = JobInfo(category, subcategory, host, directory, lysrdiffpart)
+    return info
+
+def read_tasks(lysrdiffpart):
+    global fatal
+
+    for line in file(tasklist_file(lysrdiffpart), "r"):
+        info = parse_line(line, lysrdiffpart)
+
+        if (info.host(), info.directory()) in tasks_per_source:
+            sys.stderr.write("Duplicate backup detected!\n  %s\n  %s\n" % (
+                tasks_per_source[info.source()].task_desc(),
+                info.task_desc()))
+            fatal = True
+                             
+        tasks_per_source[info.source()] = info
+
+def read_new_tasks():
+    new_found = False
+    for line in file("/nobackup/backup.lysator/var/tasks"):
+        info = parse_line(line)
+        if (info.host(), info.directory()) not in tasks_per_source:
+            info.set_lysrdiffpart(newtasks())
+            tasks_per_source[(info.host(), info.directory())] = info
+            new_found = True
+
+    return new_found
+
+def write_task_lists():
+    jobs = tasks_per_source.values()
+    random.shuffle(jobs)
+    files = {}
+    for job in jobs:
+        if job.lysrdiffpart() not in files:
+            files[job.lysrdiffpart()] = file(
+                tasklist_file(job.lysrdiffpart()) + ".new", "w")
+        files[job.lysrdiffpart()].write(job.task_desc() + "\n")
+    for lysrdiffpart, fp in files.items():
+        fp.close()
+        fn = tasklist_file(lysrdiffpart)
+        os.rename(fn + ".new", fn)
+
+def main():
+    global fatal
+
+    for lysrdiffpart in range(2):
+        read_tasks(lysrdiffpart)
+    read_new_tasks()
+    if not fatal:
+        write_task_lists()
+
+if __name__ == '__main__':
+    main()

fetch-backup-work

@@ -4,14 +4,14 @@ rm /nobackup/backup.lysator/var/tasks
 
 ssh stalingrad ls -1 /svnroot \
  | sort \
- | head -10 \
- | awk '{print "lsvn", $1, "stalingrad /svnroot/" $1}' \
+ | awk '$1 == "lost+found" { next }
+        {print "lsvn", $1, "stalingrad /svnroot/" $1}' \
 >> /nobackup/backup.lysator/var/tasks
 
 ssh stalingrad ls -1 /cvsroot \
  | sort \
- | head -10 \
- | awk '{print "lyscvs", $1, "stalingrad /cvsroot/" $1}' \
+ | awk '$1 == "lost+found" { next }
+        {print "lyscvs", $1, "stalingrad /cvsroot/" $1}' \
 >> /nobackup/backup.lysator/var/tasks
 
 ssh uhumhummy ls -1 /lysator/lyswww/users/common \
@@ -19,17 +19,42 @@ ssh uhumhummy ls -1 /lysator/lyswww/users/common \
  | grep -v THIS_IS_USERWEB \
  | grep -v '^cap\.fm$' \
  | sort \
- | grep -v '^a' \
- | grep -v '^b' \
- | head -50 \
  | awk '{print "lyswww", $1, "uhumhummy /lysator/lyswww/users/common/" $1}' \
 >> /nobackup/backup.lysator/var/tasks
 
+ssh uhumhummy ls -1 /lysator/lyswww/users/roxen_only \
+ | sort \
+ | awk '{print "roxen", $1, "uhumhummy /lysator/lyswww/users/roxen_only/" $1}'\
+>> /nobackup/backup.lysator/var/tasks
+
 ssh nema.lysator.liu.se ypcat -k auto_home \
  | sort \
  | grep -v DELETE \
  | grep -v '^ *$' \
- | head -75 \
  | sed 's/:/ /' \
+ | awk '$1 == "mailman" { next }
+        $1 == "patrick" { next }
+        $1 == "snapshot" { next }
+        $1 == "u1" { next }
+        $1 == "u2" { next }
+        $1 == "u3" { next }
+        $1 == "u4" { next }
+        $1 == "u5" { next }
+        $1 == "u6" { next }
+        $1 == "lysdisk1" { next }
+        $1 == "lysdisk2" { next }
+        $1 == "lysdisk3" { next }
+        $1 == "lysdisk4" { next }
+        $1 == "lysdisk5" { next }
+        $1 == "lysdisk6" { next }
+        $1 == "lysdisk7" { next }
+        $1 == "lysdisk8" { next }
+        $1 == "lysdisk9" { next }
+        $1 == "lysdisk10" { next }
+        $1 == "lysdisk11" { next }
+        $1 == "kheldar" { next }
+        $1 == "quota1" { next }
+        $1 == "sge" { next }
+	{ print }' \
  | sed 's/^/home /' \
 >> /nobackup/backup.lysator/var/tasks