Add job that updates the file database from the main mirror.

diff --git a/www/manager.py b/www/manager.py
index 5c2299a25b82f3658e0d667bbdffbcf76ce0637d..b7aaaf8e63088078cb67691250a2ce6424cfe8ea 100644 (file)
--- a/www/manager.py
+++ b/www/manager.py
@@ -79,8 +79,23 @@ class MirrorManager(Manager):
                self.mirrors.check_all()
 
 
+class ReleaseFilesManager(Manager):
+       @property
+       def releases(self):
+               return backend.Releases()
+
+       @property
+       def timeout(self):
+               return backend.Config().get_int("releasefiles_check_interval")
+
+       def do(self):
+               for release in self.releases.get_all():
+                       release.scan_files()
+
+
 if __name__ == "__main__":
        d = Daemon()
        d.add(MirrorManager)
+       d.add(ReleaseFilesManager)
 
        d.run()

diff --git a/www/webapp/backend/releases.py b/www/webapp/backend/releases.py
index 8dc7e6c3db7a3a4999a9cd1e9edce28fed984708..de969db26414724297b6fc7043e51a54d0de2ca0 100644 (file)
--- a/www/webapp/backend/releases.py
+++ b/www/webapp/backend/releases.py
@@ -1,6 +1,9 @@
 #!/usr/bin/python
 
+import hashlib
 import logging
+import os
+import re
 import urlparse
 
 from databases import Databases
@@ -137,6 +140,10 @@ class Release(object):
        def date(self):
                return self.__data.get("date")
 
+       @property
+       def path(self):
+               return self.__data.get("path")
+
        @property
        def torrent_hash(self):
                h = self.__data.get("torrent_hash")
@@ -148,6 +155,75 @@ class Release(object):
                        if file.type == type:
                                return file
 
+       def __file_hash(self, filename):
+               sha1 = hashlib.sha1()
+
+               with open(filename) as f:
+                       sha1.update(f.read())
+
+               return sha1.hexdigest()
+
+       def __guess_filetype(self, filename):
+               if filename.endswith(".iso"):
+                       return "iso"
+
+               if filename.endswith(".torrent"):
+                       return "torrent"
+
+               if "xen" in filename:
+                       return "xen"
+
+               if "sources" in filename:
+                       return "source"
+
+               if "usb-fdd" in filename:
+                       return "usbfdd"
+
+               if "usb-hdd" in filename:
+                       return "usbhdd"
+
+               if "scon" in filename:
+                       return "alix"
+
+               if filename.endswith(".img.gz"):
+                       return "flash"
+
+               return "unknown"
+
+       def scan_files(self, basepath="/srv/mirror0"):
+               if not self.path:
+                       return
+
+               path = os.path.join(basepath, self.path)
+
+               files = [f.filename for f in self.files]
+
+               # Make files that do not exists not loadable.
+               for filename in files:
+                       _filename = os.path.join(basepath, filename)
+                       if not os.path.exists(_filename):
+                               self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
+
+               for filename in os.listdir(path):
+                       filename = os.path.join(path, filename)
+
+                       if os.path.isdir(filename):
+                               continue
+
+                       _filename = re.match(".*(releases/.*)", filename).group(1)
+                       if _filename in files:
+                               continue
+
+                       if filename.endswith(".md5"):
+                               continue
+
+                       filehash = self.__file_hash(filename)
+                       filesize = os.path.getsize(filename)
+                       filetype = self.__guess_filetype(filename)
+
+                       self.db.execute("""INSERT INTO files(releases, filename, filesize, filetype, sha1)
+                               VALUES(%s, %s, %s, %s, %s)""", self.id, _filename, filesize, filetype, filehash)
+
 
 class Releases(object):
        __metaclass__ = Singleton