New file.

verify-disk.py

+#!/usr/bin/env python
+# Verify the existing files in the filesystems.
+
+import os
+import sys
+import md5
+import sha
+import time
+import errno
+
+import MySQLdb
+
+class file_hash:
+    def __init__(self, filename):
+        fp = file(filename, "rb")
+
+        md = md5.new()
+        sh = sha.new()
+
+        while 1:
+            chunk = fp.read()
+            if chunk == "":
+                break
+            md.update(chunk)
+            sh.update(chunk)
+
+        fp.close()
+
+        self.md5 = md.hexdigest()
+        self.sha1 = sh.hexdigest()
+
+def verify(DBH, dir_id):
+    cursor = DBH.cursor()
+    update = DBH.cursor()
+
+    cursor.execute("SELECT COUNT(*) FROM file WHERE dir_id = %d" % (dir_id,) )
+    total = cursor.fetchone()[0]
+    done = 0
+    bad = []
+
+    cursor.execute("SELECT dir_name FROM base"
+                   " WHERE dir_id = %d" % (dir_id, ))
+    dir_name = cursor.fetchone()[0]
+
+    cursor.execute("SELECT file_id, filename, size, unix_timestamp(mtime),"
+                   " md5sum, sha1sum"
+                   " FROM file"
+                   " WHERE file.dir_id = %d"
+                   " ORDER BY verified" % (dir_id,))
+    while 1:
+        res = cursor.fetchmany()
+        if len(res) == 0:
+            break
+        for [file_id, filename, size, mtime, md5sum, sha1sum] in res:
+            fn = os.path.join(dir_name, filename)
+            try:
+                st = os.stat(fn)
+            except OSError, e:
+                if e.errno == errno.ENOENT:
+                    print "Nu such file:", fn
+                    bad.append((file_id, fn))
+                    continue
+                raise
+            if st.st_size != size:
+                print "Wrong size:", fn
+                bad.append((file_id, fn))
+                continue
+            h = file_hash(fn)
+            if h.md5 != md5sum or h.sha1 != sha1sum:
+                print "Checksum error:", fn
+                bad.append((file_id, fn))
+                continue
+            if st.st_mtime != mtime:
+                print "Wrong mtime:", fn
+                # print mtime - st.st_mtime, st.st_mtime, mtime, fn
+                # The next line restores the mtime of the file from
+                # the database.
+                # os.utime(fn, (time.time(), mtime))
+                bad.append((file_id, fn))
+                continue
+            update.execute("UPDATE file SET verified = NOW()"
+                           " WHERE file_id = %s", (file_id))
+            done += 1
+            print "\r%s: Checked %s/%s (%s bad)" % (dir_id, done, total,
+                                                    len(bad)),
+            sys.stdout.flush()
+
+    print
+    cursor.close()
+    return bad
+
+def scan_all(DBH):
+    outer = DBH.cursor()
+    outer.execute("SELECT dir_id, first_scanned FROM base"
+                   " WHERE active = 1")
+    for [dir_id, first_scanned] in outer.fetchall():
+        verify(DBH, dir_id)
+    outer.close()
+    
+
+def main():
+    DBH = MySQLdb.connect(db='isoonline', user='ceder')
+    scan_all(DBH)
+
+if __name__ == '__main__':
+    main()