From 3c4f2edc598cad8a6549345e0fcb7c2d1c35bf87 Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 14 Jan 2011 14:38:44 +0100
Subject: [PATCH] Add job that updates the file database from the main mirror.

---
 www/manager.py                 | 15 +++++++
 www/webapp/backend/releases.py | 76 ++++++++++++++++++++++++++++++++++
 2 files changed, 91 insertions(+)

diff --git a/www/manager.py b/www/manager.py
index 5c2299a..b7aaaf8 100644
--- a/www/manager.py
+++ b/www/manager.py
@@ -79,8 +79,23 @@ class MirrorManager(Manager):
 		self.mirrors.check_all()
 
 
+class ReleaseFilesManager(Manager):
+	@property
+	def releases(self):
+		return backend.Releases()
+
+	@property
+	def timeout(self):
+		return backend.Config().get_int("releasefiles_check_interval")
+
+	def do(self):
+		for release in self.releases.get_all():
+			release.scan_files()
+
+
 if __name__ == "__main__":
 	d = Daemon()
 	d.add(MirrorManager)
+	d.add(ReleaseFilesManager)
 
 	d.run()
diff --git a/www/webapp/backend/releases.py b/www/webapp/backend/releases.py
index 8dc7e6c..de969db 100644
--- a/www/webapp/backend/releases.py
+++ b/www/webapp/backend/releases.py
@@ -1,6 +1,9 @@
 #!/usr/bin/python
 
+import hashlib
 import logging
+import os
+import re
 import urlparse
 
 from databases import Databases
@@ -137,6 +140,10 @@ class Release(object):
 	def date(self):
 		return self.__data.get("date")
 
+	@property
+	def path(self):
+		return self.__data.get("path")
+
 	@property
 	def torrent_hash(self):
 		h = self.__data.get("torrent_hash")
@@ -148,6 +155,75 @@ class Release(object):
 			if file.type == type:
 				return file
 
+	def __file_hash(self, filename):
+		sha1 = hashlib.sha1()
+
+		with open(filename) as f:
+			sha1.update(f.read())
+
+		return sha1.hexdigest()
+
+	def __guess_filetype(self, filename):
+		if filename.endswith(".iso"):
+			return "iso"
+
+		if filename.endswith(".torrent"):
+			return "torrent"
+
+		if "xen" in filename:
+			return "xen"
+
+		if "sources" in filename:
+			return "source"
+
+		if "usb-fdd" in filename:
+			return "usbfdd"
+
+		if "usb-hdd" in filename:
+			return "usbhdd"
+
+		if "scon" in filename:
+			return "alix"
+
+		if filename.endswith(".img.gz"):
+			return "flash"
+
+		return "unknown"
+
+	def scan_files(self, basepath="/srv/mirror0"):
+		if not self.path:
+			return
+
+		path = os.path.join(basepath, self.path)
+
+		files = [f.filename for f in self.files]
+
+		# Make files that do not exists not loadable.
+		for filename in files:
+			_filename = os.path.join(basepath, filename)
+			if not os.path.exists(_filename):
+				self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
+
+		for filename in os.listdir(path):
+			filename = os.path.join(path, filename)
+
+			if os.path.isdir(filename):
+				continue
+
+			_filename = re.match(".*(releases/.*)", filename).group(1)
+			if _filename in files:
+				continue
+
+			if filename.endswith(".md5"):
+				continue
+
+			filehash = self.__file_hash(filename)
+			filesize = os.path.getsize(filename)
+			filetype = self.__guess_filetype(filename)
+
+			self.db.execute("""INSERT INTO files(releases, filename, filesize, filetype, sha1)
+				VALUES(%s, %s, %s, %s, %s)""", self.id, _filename, filesize, filetype, filehash)
+
 
 class Releases(object):
 	__metaclass__ = Singleton
-- 
2.39.2