[ipfire.org.git] / src / backend / releases.py

#!/usr/bin/python

import hashlib
import logging
import os
import re
import urllib.request, urllib.parse, urllib.error
import urllib.parse

from . import database
from . import tracker
from .misc import Object

TRACKERS = (
    "http://ipv4.tracker.ipfire.org:6969/announce",
    "udp://ipv4.tracker.ipfire.org:6969",
    "http://ipv6.tracker.ipfire.org:6969/announce",
    "udp://ipv6.tracker.ipfire.org:6969",
)

class File(Object):
	def __init__(self, backend, release, id, data=None):
		Object.__init__(self, backend)

		self.id = id
		self._release = release

		# get all data from database
		self.__data = data

	def __eq__(self, other):
		if isinstance(other, self.__class__):
			return self.id == otherid

	def __lt__(self, other):
		if isinstance(other, self.__class__):
			return self.prio < other.prio

	@property
	def data(self):
		if self.__data is None:
			self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
			assert self.__data

		return self.__data

	@property
	def release(self):
		if not self._release:
			release_id = self.data.get("releases")
			self._release = Release(release_id)

		return self._release

	@property
	def type(self):
		filename = self.filename

		if filename.endswith(".iso"):
			return "iso"

		elif filename.endswith(".torrent"):
			return "torrent"

		elif "xen" in filename:
			if "downloader" in filename:
				return "xen-downloader"

			return "xen"

		elif "sources" in filename:
			return "source"

		elif "usb-fdd" in filename:
			return "usbfdd"

		elif "usb-hdd" in filename:
			return "usbhdd"

		elif "armv5tel" in filename and "scon" in filename:
			return "armv5tel-scon"

		elif "armv5tel" in filename:
			return "armv5tel"

		elif "scon" in filename:
			return "alix"

		elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
			return "flash"

		else:
			return "unknown"

	@property
	def url(self):
                return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)

	@property
	def desc(self):
		_ = lambda x: x

		descriptions = {
			"armv5tel"	: _("Flash Image"),
			"armv5tel-scon"	: _("Flash Image with serial console"),
			"iso"		: _("CD Image"),
			"torrent"	: _("Torrent File"),
			"flash"		: _("Flash Image"),
			"alix"		: _("Flash Image with serial console"),
			"usbfdd"	: _("USB FDD Image"),
			"usbhdd"	: _("USB HDD Image"),
			"xen"		: _("Pre-generated Xen Image"),
			"xen-downloader": _("Xen-Image Generator"),
		}

		try:
			return descriptions[self.type]
		except KeyError:
			return _("Unknown image type")

	@property
	def prio(self):
		priorities = {
			"iso"		: 10,
			"torrent"	: 20,
			"flash"		: 40,
			"alix"		: 41,
			"usbfdd"	: 31,
			"usbhdd"	: 30,
			"armv5tel"	: 40,
			"armv5tel-scon"	: 41,
			"xen"		: 50,
			"xen-downloader": 51,
		}
		
		try:
			return priorities[self.type]
		except KeyError:
			return 999

	@property
	def rem(self):
		_ = lambda x: x
	
		remarks = {
			"armv5tel"	: _("This image runs on many ARM-based boards"),
			"armv5tel-scon"	: _("This image runs on ARM boards with a serial console"),
			"iso"		: _("Use this image to burn a CD and install IPFire from it."),
			"torrent"	: _("Download the CD image from the torrent network."),
			"flash"		: _("An image that is meant to run on embedded devices."),
			"alix"		: _("Flash image where a serial console is enabled by default."),
			"usbfdd"	: _("Install IPFire from a floppy-formated USB key."),
			"usbhdd"	: _("If the floppy image doesn't work, use this image instead."),
			"xen"		: _("A ready-to-run image for Xen."),
			"xen-downloader": _("Generator for creating a Xen image."),
		}

		try:
			return remarks[self.type]
		except KeyError:
			return _("Unknown image type")

	@property
	def sha1(self):
		return self.data.get("sha1")

	@property
	def filename(self):
		return self.data.get("filename")

	@property
	def basename(self):
		return os.path.basename(self.filename)

	@property
	def size(self):
		return self.data.get("filesize")

	@property
	def arch(self):
		known_arches = ("x86_64", "i586", "arm")

		for arch in known_arches:
			if arch in self.basename:
				return arch

		return "N/A"

	@property
	def torrent_hash(self):
		return self.data.get("torrent_hash", None)

	@property
	def torrent_url(self):
		if self.torrent_hash:
			return "%s.torrent" % self.url

	@property
	def magnet_link(self):
		# Don't return anything if we have no torrent hash.
		if self.torrent_hash is None:
			return

		s = "magnet:?xt=urn:btih:%s" % self.torrent_hash

		#s += "&xl=%d" % self.size
		s += "&dn=%s" % urllib.parse.quote(self.basename)

		# Add our tracker.
		for tracker in TRACKERS:
			s += "&tr=%s" % tracker

		# Add web download URL
		s += "&as=%s" % urllib.parse.quote(self.url)

		return s


class Release(Object):
	def __init__(self, backend, id, data=None):
		Object.__init__(self, backend)
		self.id = id

		# get all data from database
		self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
		assert self.__data

		self.__files = []

	def __str__(self):
		return self.name

	def __repr__(self):
		return "<%s %s>" % (self.__class__.__name__, self.name)

	def __cmp__(self, other):
		return cmp(self.id, other.id)

	@property
	def arches(self):
		for arch in ("x86_64", "aarch64", "i586", "arm"):
			if arch in (f.arch for f in self.files):
				yield arch

	@property
	def files(self):
		if not self.__files:
			files = self.db.query("SELECT * FROM files WHERE releases = %s \
				AND NOT filename LIKE '%%.torrent'", self.id)

			self.__files = [File(self.backend, self, f.id, f) for f in files]
			self.__files.sort()

		return self.__files

	def get_files_by_arch(self, arch):
		for f in self.files:
			if f.arch == arch:
				yield f

	@property
	def torrents(self):
		torrents = []

		for file in self.files:
			if not file.torrent_hash:
				continue

			torrents.append(file)

		return torrents

	@property
	def name(self):
		return self.__data.name

	@property
	def slug(self):
		return self.__data.sname

	# XXX compat
	sname = slug

	# XXX cache this
	@property
	def blog(self):
		if self.__data.blog_id:
			return self.backend.blog.get_by_id(self.__data.blog_id)

	@property
	def fireinfo_id(self):
		name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")

		res = self.db.get("SELECT id FROM fireinfo_releases \
			WHERE name = %s", name)

		if res:
			return res.id

	@property
	def stable(self):
		return self.__data.stable

	@property
	def published(self):
		return self.__data.published

	date = published

	@property
	def path(self):
		return self.__data.path

	def get_file(self, type):
		for file in self.files:
			if file.type == type:
				return file

	def __file_hash(self, filename):
		sha1 = hashlib.sha1()

		with open(filename) as f:
			buf_size = 1024
			buf = f.read(buf_size)
			while buf:
				sha1.update(buf)
				buf = f.read(buf_size)

		return sha1.hexdigest()

	def scan_files(self, basepath="/srv/mirror0"):
		if not self.path:
			return

		path = os.path.join(basepath, self.path)
		if not os.path.exists(path):
			return

		files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
		files = [f.filename for f in files]

		# Make files that do not exists not loadable.
		for filename in files:
			_filename = os.path.join(basepath, filename)
			if not os.path.exists(_filename):
				self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)

		for filename in os.listdir(path):
			filename = os.path.join(path, filename)

			if os.path.isdir(filename):
				continue

			_filename = re.match(".*(releases/.*)", filename).group(1)
			if _filename in files:
				continue

			if filename.endswith(".md5"):
				continue

			logging.info("Hashing %s..." % filename)
			filehash = self.__file_hash(filename)
			filesize = os.path.getsize(filename)

			# Check if there is a torrent download available for this file:
			torrent_hash = ""
			torrent_file = "%s.torrent" % filename
			if os.path.exists(torrent_file):
				torrent_hash = self.torrent_read_hash(torrent_file)

			self.db.execute("INSERT INTO files(releases, filename, filesize, \
				sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s)",
				self.id, _filename, filesize, filehash, torrent_hash)

		# Search for all files that miss a torrent hash.
		files = self.db.query("SELECT id, filename FROM files \
			WHERE releases = %s AND torrent_hash IS NULL", self.id)

		for file in files:
			path = os.path.join(basepath, file.filename)

			torrent_file = "%s.torrent" % path
			if os.path.exists(torrent_file):
				torrent_hash = self.torrent_read_hash(torrent_file)

				self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
					torrent_hash, file.id)

	def torrent_read_hash(self, filename):
		f = None
		try:
			f = open(filename, "rb")

			metainfo = tracker.bdecode(f.read())
			metainfo = tracker.bencode(metainfo["info"])

			hash = hashlib.sha1()
			hash.update(metainfo)

			return hash.hexdigest()

		finally:
			if f:
				f.close()

	def supports_arch(self, arch):
		return arch in ("x86_64", "i586")

	def supports_platform(self, platform):
		# Currently there is nothing else than pcbios supported
		if platform == "pcbios":
			return True

		return False

	def is_netboot_capable(self):
		return self.path and "ipfire-2.x" in self.path

	def netboot_kernel_url(self, arch, platform):
		assert self.supports_arch(arch)
		assert self.supports_platform(platform)

		if self.sname >= "ipfire-2.19-core100":
			return "https://downloads.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)

		return "https://downloads.ipfire.org/%s/images/vmlinuz" % self.path

	def netboot_initrd_url(self, arch, platform):
		assert self.supports_arch(arch)
		assert self.supports_platform(platform)

		if self.sname >= "ipfire-2.19-core100":
			return "https://downloads.ipfire.org/%s/images/%s/instroot" % (self.path, arch)

		return "https://downloads.ipfire.org/%s/images/instroot" % self.path

	def netboot_args(self, arch, platform):
		return ""

	@property
	def post(self):
		if self.__data.blog_id:
			return self.backend.blog.get_by_id(self.__data.blog_id)

	# Fireinfo Stuff

	@property
	def penetration(self):
		penetration = self.memcache.get("%s-penetration" % self.sname)

		# Cache HIT
		if penetration:
			return penetration

		# Get penetration from fireinfo
		penetration = self.backend.fireinfo.get_release_penetration(self)

		# Cache for 1 hour
		self.memcache.set("%s-penetration" % self.sname, penetration, 3600)

		return penetration


class Releases(Object):
	def _get_release(self, query, *args):
		res = self.db.get(query, *args)

		if res:
			return Release(self.backend, res.id, data=res)

	def _get_releases(self, query, *args):
		res = self.db.query(query, *args)

		for row in res:
			yield Release(self.backend, row.id, data=row)

	def get_by_id(self, id):
		ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_by_sname(self, sname):
		ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_by_news_id(self, news_id):
		ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_latest(self, stable=True):
		ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
			AND stable = %s ORDER BY published DESC LIMIT 1", stable)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_releases_older_than(self, release, limit=None):
		return self._get_releases("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published < %s \
				ORDER BY published DESC LIMIT %s", release.published, limit)

	def get_latest_unstable(self):
		ret = self.db.get("SELECT * FROM releases r1 \
			WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
			AND stable = %s AND NOT EXISTS ( \
				SELECT * FROM releases r2 WHERE r2.stable = %s AND \
					r2.published IS NOT NULL AND r2.published >= r1.published \
				) ORDER BY r1.published DESC LIMIT 1", False, True)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_stable(self):
		query = self.db.query("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
			ORDER BY published DESC")

		releases = []
		for row in query:
			release = Release(self.backend, row.id, data=row)
			releases.append(release)

		return releases

	def get_unstable(self):
		query = self.db.query("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
			ORDER BY published DESC")

		releases = []
		for row in query:
			release = Release(self.backend, row.id, data=row)
			releases.append(release)

		return releases

	def get_all(self):
		query = self.db.query("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published <= NOW() \
			ORDER BY published DESC")

		releases = []
		for row in query:
			release = Release(self.backend, row.id, data=row)
			releases.append(release)

		return releases

	def _get_all(self):
		query = self.db.query("SELECT * FROM releases ORDER BY published DESC")

		releases = []
		for row in query:
			release = Release(self.backend, row.id, data=row)
			releases.append(release)

		return releases

	def get_file_for_torrent_hash(self, torrent_hash):
		file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
			torrent_hash)

		if not file:
			return

		release = Release(self.backend, file.releases)
		file = File(self.backend, release, file.id)

		return file
Commit	Line	Data
940227cb MT	1	#!/usr/bin/python
940227cb MT	2
3c4f2edc	3	import hashlib
940227cb	4	import logging
3c4f2edc MT	5	import os
3c4f2edc MT	6	import re
11347e46 MT	7	import urllib.request, urllib.parse, urllib.error
11347e46 MT	8	import urllib.parse
940227cb	9
11347e46 MT	10	from . import database
	11	from . import tracker
	12	from .misc import Object
fadcfd00	13
f7635119 MT	14	TRACKERS = (
	15	"http://ipv4.tracker.ipfire.org:6969/announce",
	16	"udp://ipv4.tracker.ipfire.org:6969",
	17	"http://ipv6.tracker.ipfire.org:6969/announce",
	18	"udp://ipv6.tracker.ipfire.org:6969",
	19	)
	20
9068dba1 MT	21	class File(Object):
	22	def __init__(self, backend, release, id, data=None):
	23	Object.__init__(self, backend)
940227cb	24
938d083d MT	25	self.id = id
938d083d MT	26	self._release = release
940227cb MT	27
940227cb MT	28	# get all data from database
9068dba1	29	self.__data = data
940227cb	30
5ecd3be7 MT	31	def __eq__(self, other):
	32	if isinstance(other, self.__class__):
	33	return self.id == otherid
	34
	35	def __lt__(self, other):
	36	if isinstance(other, self.__class__):
	37	return self.prio < other.prio
938d083d MT	38
	39	@property
	40	def data(self):
	41	if self.__data is None:
	42	self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
	43	assert self.__data
	44
	45	return self.__data
	46
	47	@property
	48	def release(self):
	49	if not self._release:
	50	release_id = self.data.get("releases")
	51	self._release = Release(release_id)
	52
	53	return self._release
940227cb MT	54
	55	@property
	56	def type(self):
fadcfd00 MT	57	filename = self.filename
	58
	59	if filename.endswith(".iso"):
	60	return "iso"
	61
	62	elif filename.endswith(".torrent"):
	63	return "torrent"
	64
	65	elif "xen" in filename:
80594ae3 MT	66	if "downloader" in filename:
	67	return "xen-downloader"
	68
fadcfd00 MT	69	return "xen"
	70
	71	elif "sources" in filename:
	72	return "source"
	73
	74	elif "usb-fdd" in filename:
	75	return "usbfdd"
	76
	77	elif "usb-hdd" in filename:
	78	return "usbhdd"
	79
3230a47f MT	80	elif "armv5tel" in filename and "scon" in filename:
	81	return "armv5tel-scon"
	82
fadcfd00 MT	83	elif "armv5tel" in filename:
	84	return "armv5tel"
	85
	86	elif "scon" in filename:
	87	return "alix"
	88
116312c5	89	elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
fadcfd00 MT	90	return "flash"
	91
	92	else:
	93	return "unknown"
940227cb MT	94
	95	@property
	96	def url(self):
11347e46	97	return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
940227cb MT	98
	99	@property
	100	def desc(self):
	101	_ = lambda x: x
	102
	103	descriptions = {
e77cd04c MT	104	"armv5tel" : _("Flash Image"),
	105	"armv5tel-scon" : _("Flash Image with serial console"),
	106	"iso" : _("CD Image"),
	107	"torrent" : _("Torrent File"),
	108	"flash" : _("Flash Image"),
	109	"alix" : _("Flash Image with serial console"),
940227cb MT	110	"usbfdd" : _("USB FDD Image"),
940227cb MT	111	"usbhdd" : _("USB HDD Image"),
e77cd04c	112	"xen" : _("Pre-generated Xen Image"),
80594ae3	113	"xen-downloader": _("Xen-Image Generator"),
940227cb MT	114	}
	115
	116	try:
	117	return descriptions[self.type]
	118	except KeyError:
	119	return _("Unknown image type")
	120
	121	@property
	122	def prio(self):
	123	priorities = {
	124	"iso" : 10,
	125	"torrent" : 20,
	126	"flash" : 40,
	127	"alix" : 41,
	128	"usbfdd" : 31,
	129	"usbhdd" : 30,
3230a47f MT	130	"armv5tel" : 40,
3230a47f MT	131	"armv5tel-scon" : 41,
940227cb	132	"xen" : 50,
80594ae3	133	"xen-downloader": 51,
940227cb MT	134	}
	135
	136	try:
	137	return priorities[self.type]
	138	except KeyError:
	139	return 999
	140
	141	@property
	142	def rem(self):
	143	_ = lambda x: x
	144
	145	remarks = {
fadcfd00	146	"armv5tel" : _("This image runs on many ARM-based boards"),
3230a47f	147	"armv5tel-scon" : _("This image runs on ARM boards with a serial console"),
940227cb MT	148	"iso" : _("Use this image to burn a CD and install IPFire from it."),
	149	"torrent" : _("Download the CD image from the torrent network."),
	150	"flash" : _("An image that is meant to run on embedded devices."),
	151	"alix" : _("Flash image where a serial console is enabled by default."),
	152	"usbfdd" : _("Install IPFire from a floppy-formated USB key."),
	153	"usbhdd" : _("If the floppy image doesn't work, use this image instead."),
	154	"xen" : _("A ready-to-run image for Xen."),
80594ae3	155	"xen-downloader": _("Generator for creating a Xen image."),
940227cb MT	156	}
	157
	158	try:
	159	return remarks[self.type]
	160	except KeyError:
	161	return _("Unknown image type")
	162
	163	@property
	164	def sha1(self):
938d083d	165	return self.data.get("sha1")
940227cb MT	166
	167	@property
	168	def filename(self):
938d083d	169	return self.data.get("filename")
940227cb	170
c9698a0b MT	171	@property
	172	def basename(self):
	173	return os.path.basename(self.filename)
	174
60024cc8 MT	175	@property
60024cc8 MT	176	def size(self):
938d083d	177	return self.data.get("filesize")
60024cc8 MT	178
	179	@property
	180	def arch(self):
6371bfda	181	known_arches = ("x86_64", "i586", "arm")
60024cc8 MT	182
	183	for arch in known_arches:
	184	if arch in self.basename:
	185	return arch
	186
fadcfd00 MT	187	return "N/A"
	188
	189	@property
	190	def torrent_hash(self):
938d083d	191	return self.data.get("torrent_hash", None)
fadcfd00	192
a69e87a1 MT	193	@property
	194	def torrent_url(self):
	195	if self.torrent_hash:
	196	return "%s.torrent" % self.url
5276cd41	197
fadcfd00 MT	198	@property
	199	def magnet_link(self):
	200	# Don't return anything if we have no torrent hash.
	201	if self.torrent_hash is None:
	202	return
	203
	204	s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
	205
	206	#s += "&xl=%d" % self.size
11347e46	207	s += "&dn=%s" % urllib.parse.quote(self.basename)
fadcfd00 MT	208
fadcfd00 MT	209	# Add our tracker.
a69e87a1 MT	210	for tracker in TRACKERS:
a69e87a1 MT	211	s += "&tr=%s" % tracker
fadcfd00	212
a69e87a1 MT	213	# Add web download URL
a69e87a1 MT	214	s += "&as=%s" % urllib.parse.quote(self.url)
dc7b8879	215
fadcfd00	216	return s
60024cc8	217
940227cb	218
9068dba1 MT	219	class Release(Object):
	220	def __init__(self, backend, id, data=None):
	221	Object.__init__(self, backend)
940227cb MT	222	self.id = id
	223
	224	# get all data from database
9068dba1	225	self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
940227cb MT	226	assert self.__data
	227
	228	self.__files = []
	229
e77cd04c MT	230	def __str__(self):
	231	return self.name
	232
940227cb MT	233	def __repr__(self):
	234	return "<%s %s>" % (self.__class__.__name__, self.name)
	235
9068dba1 MT	236	def __cmp__(self, other):
	237	return cmp(self.id, other.id)
	238
e77cd04c MT	239	@property
	240	def arches(self):
	241	for arch in ("x86_64", "aarch64", "i586", "arm"):
	242	if arch in (f.arch for f in self.files):
	243	yield arch
	244
940227cb MT	245	@property
	246	def files(self):
	247	if not self.__files:
9068dba1 MT	248	files = self.db.query("SELECT * FROM files WHERE releases = %s \
9068dba1 MT	249	AND NOT filename LIKE '%%.torrent'", self.id)
940227cb	250
9068dba1 MT	251	self.__files = [File(self.backend, self, f.id, f) for f in files]
9068dba1 MT	252	self.__files.sort()
940227cb MT	253
	254	return self.__files
	255
e77cd04c MT	256	def get_files_by_arch(self, arch):
	257	for f in self.files:
	258	if f.arch == arch:
	259	yield f
	260
938d083d MT	261	@property
	262	def torrents(self):
	263	torrents = []
	264
	265	for file in self.files:
	266	if not file.torrent_hash:
	267	continue
	268
	269	torrents.append(file)
	270
	271	return torrents
	272
940227cb MT	273	@property
940227cb MT	274	def name(self):
9068dba1 MT	275	return self.__data.name
	276
	277	@property
e77cd04c	278	def slug(self):
9068dba1	279	return self.__data.sname
940227cb	280
e77cd04c MT	281	# XXX compat
	282	sname = slug
	283
	284	# XXX cache this
	285	@property
	286	def blog(self):
	287	if self.__data.blog_id:
	288	return self.backend.blog.get_by_id(self.__data.blog_id)
	289
dd3a5446 MT	290	@property
	291	def fireinfo_id(self):
	292	name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
	293
	294	res = self.db.get("SELECT id FROM fireinfo_releases \
	295	WHERE name = %s", name)
	296
	297	if res:
	298	return res.id
	299
940227cb MT	300	@property
940227cb MT	301	def stable(self):
9068dba1	302	return self.__data.stable
940227cb MT	303
	304	@property
	305	def published(self):
9068dba1	306	return self.__data.published
940227cb	307
9068dba1	308	date = published
940227cb	309
3c4f2edc MT	310	@property
3c4f2edc MT	311	def path(self):
9068dba1	312	return self.__data.path
3c4f2edc	313
940227cb MT	314	def get_file(self, type):
	315	for file in self.files:
	316	if file.type == type:
	317	return file
	318
3c4f2edc MT	319	def __file_hash(self, filename):
	320	sha1 = hashlib.sha1()
	321
	322	with open(filename) as f:
b5f4eef0 MT	323	buf_size = 1024
	324	buf = f.read(buf_size)
	325	while buf:
	326	sha1.update(buf)
	327	buf = f.read(buf_size)
3c4f2edc MT	328
	329	return sha1.hexdigest()
	330
3c4f2edc MT	331	def scan_files(self, basepath="/srv/mirror0"):
	332	if not self.path:
	333	return
	334
	335	path = os.path.join(basepath, self.path)
921d98cc MT	336	if not os.path.exists(path):
	337	return
	338
fadcfd00 MT	339	files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
fadcfd00 MT	340	files = [f.filename for f in files]
3c4f2edc MT	341
	342	# Make files that do not exists not loadable.
	343	for filename in files:
	344	_filename = os.path.join(basepath, filename)
	345	if not os.path.exists(_filename):
	346	self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
	347
	348	for filename in os.listdir(path):
	349	filename = os.path.join(path, filename)
	350
	351	if os.path.isdir(filename):
	352	continue
	353
	354	_filename = re.match(".(releases/.)", filename).group(1)
	355	if _filename in files:
	356	continue
	357
	358	if filename.endswith(".md5"):
	359	continue
	360
9068dba1	361	logging.info("Hashing %s..." % filename)
3c4f2edc MT	362	filehash = self.__file_hash(filename)
3c4f2edc MT	363	filesize = os.path.getsize(filename)
3c4f2edc	364
fadcfd00 MT	365	# Check if there is a torrent download available for this file:
	366	torrent_hash = ""
	367	torrent_file = "%s.torrent" % filename
	368	if os.path.exists(torrent_file):
	369	torrent_hash = self.torrent_read_hash(torrent_file)
	370
	371	self.db.execute("INSERT INTO files(releases, filename, filesize, \
	372	sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s)",
	373	self.id, _filename, filesize, filehash, torrent_hash)
	374
	375	# Search for all files that miss a torrent hash.
	376	files = self.db.query("SELECT id, filename FROM files \
	377	WHERE releases = %s AND torrent_hash IS NULL", self.id)
	378
	379	for file in files:
	380	path = os.path.join(basepath, file.filename)
	381
	382	torrent_file = "%s.torrent" % path
	383	if os.path.exists(torrent_file):
	384	torrent_hash = self.torrent_read_hash(torrent_file)
	385
	386	self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
	387	torrent_hash, file.id)
	388
	389	def torrent_read_hash(self, filename):
	390	f = None
	391	try:
	392	f = open(filename, "rb")
	393
	394	metainfo = tracker.bdecode(f.read())
	395	metainfo = tracker.bencode(metainfo["info"])
	396
	397	hash = hashlib.sha1()
	398	hash.update(metainfo)
	399
	400	return hash.hexdigest()
	401
	402	finally:
	403	if f:
	404	f.close()
3c4f2edc	405
37b5c0cf	406	def supports_arch(self, arch):
6371bfda	407	return arch in ("x86_64", "i586")
37b5c0cf MT	408
	409	def supports_platform(self, platform):
	410	# Currently there is nothing else than pcbios supported
	411	if platform == "pcbios":
	412	return True
	413
	414	return False
	415
9068dba1 MT	416	def is_netboot_capable(self):
	417	return self.path and "ipfire-2.x" in self.path
	418
37b5c0cf MT	419	def netboot_kernel_url(self, arch, platform):
	420	assert self.supports_arch(arch)
	421	assert self.supports_platform(platform)
	422
6371bfda	423	if self.sname >= "ipfire-2.19-core100":
73493ea6	424	return "https://downloads.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
6371bfda	425
73493ea6	426	return "https://downloads.ipfire.org/%s/images/vmlinuz" % self.path
940227cb	427
37b5c0cf MT	428	def netboot_initrd_url(self, arch, platform):
	429	assert self.supports_arch(arch)
	430	assert self.supports_platform(platform)
	431
6371bfda	432	if self.sname >= "ipfire-2.19-core100":
73493ea6	433	return "https://downloads.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
6371bfda	434
73493ea6	435	return "https://downloads.ipfire.org/%s/images/instroot" % self.path
940227cb	436
37b5c0cf MT	437	def netboot_args(self, arch, platform):
	438	return ""
	439
2fed2438	440	@property
5baa3aef	441	def post(self):
487417ad MT	442	if self.__data.blog_id:
487417ad MT	443	return self.backend.blog.get_by_id(self.__data.blog_id)
2fed2438	444
dd3a5446 MT	445	# Fireinfo Stuff
	446
	447	@property
	448	def penetration(self):
	449	penetration = self.memcache.get("%s-penetration" % self.sname)
	450
	451	# Cache HIT
	452	if penetration:
	453	return penetration
	454
	455	# Get penetration from fireinfo
	456	penetration = self.backend.fireinfo.get_release_penetration(self)
	457
	458	# Cache for 1 hour
	459	self.memcache.set("%s-penetration" % self.sname, penetration, 3600)
	460
	461	return penetration
	462
940227cb	463
9068dba1	464	class Releases(Object):
984e4e7b MT	465	def _get_release(self, query, *args):
	466	res = self.db.get(query, *args)
	467
	468	if res:
	469	return Release(self.backend, res.id, data=res)
	470
e77cd04c MT	471	def _get_releases(self, query, *args):
	472	res = self.db.query(query, *args)
	473
	474	for row in res:
	475	yield Release(self.backend, row.id, data=row)
	476
940227cb	477	def get_by_id(self, id):
9068dba1 MT	478	ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
	479
	480	if ret:
	481	return Release(self.backend, ret.id, data=ret)
	482
	483	def get_by_sname(self, sname):
	484	ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
	485
	486	if ret:
	487	return Release(self.backend, ret.id, data=ret)
940227cb	488
9de13943 MT	489	def get_by_news_id(self, news_id):
	490	ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
	491
	492	if ret:
	493	return Release(self.backend, ret.id, data=ret)
	494
9068dba1 MT	495	def get_latest(self, stable=True):
	496	ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
	497	AND stable = %s ORDER BY published DESC LIMIT 1", stable)
940227cb	498
9068dba1 MT	499	if ret:
9068dba1 MT	500	return Release(self.backend, ret.id, data=ret)
940227cb	501
e77cd04c MT	502	def get_releases_older_than(self, release, limit=None):
	503	return self._get_releases("SELECT * FROM releases \
	504	WHERE published IS NOT NULL AND published < %s \
	505	ORDER BY published DESC LIMIT %s", release.published, limit)
	506
0aab96e0 MT	507	def get_latest_unstable(self):
	508	ret = self.db.get("SELECT * FROM releases r1 \
	509	WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
	510	AND stable = %s AND NOT EXISTS ( \
	511	SELECT * FROM releases r2 WHERE r2.stable = %s AND \
	512	r2.published IS NOT NULL AND r2.published >= r1.published \
	513	) ORDER BY r1.published DESC LIMIT 1", False, True)
	514
	515	if ret:
	516	return Release(self.backend, ret.id, data=ret)
	517
940227cb	518	def get_stable(self):
9068dba1 MT	519	query = self.db.query("SELECT * FROM releases \
	520	WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
	521	ORDER BY published DESC")
940227cb	522
9068dba1 MT	523	releases = []
	524	for row in query:
	525	release = Release(self.backend, row.id, data=row)
	526	releases.append(release)
	527
	528	return releases
940227cb MT	529
940227cb MT	530	def get_unstable(self):
9068dba1 MT	531	query = self.db.query("SELECT * FROM releases \
	532	WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
	533	ORDER BY published DESC")
	534
	535	releases = []
	536	for row in query:
	537	release = Release(self.backend, row.id, data=row)
	538	releases.append(release)
940227cb	539
9068dba1	540	return releases
940227cb MT	541
940227cb MT	542	def get_all(self):
9068dba1 MT	543	query = self.db.query("SELECT * FROM releases \
	544	WHERE published IS NOT NULL AND published <= NOW() \
	545	ORDER BY published DESC")
940227cb	546
9068dba1 MT	547	releases = []
	548	for row in query:
	549	release = Release(self.backend, row.id, data=row)
	550	releases.append(release)
	551
	552	return releases
940227cb	553
d0eee7f8 MT	554	def _get_all(self):
	555	query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
	556
	557	releases = []
	558	for row in query:
	559	release = Release(self.backend, row.id, data=row)
	560	releases.append(release)
	561
	562	return releases
	563
938d083d MT	564	def get_file_for_torrent_hash(self, torrent_hash):
938d083d MT	565	file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
fadcfd00 MT	566	torrent_hash)
	567
	568	if not file:
	569	return
	570
ea324f48 MT	571	release = Release(self.backend, file.releases)
ea324f48 MT	572	file = File(self.backend, release, file.id)
938d083d MT	573
938d083d MT	574	return file