[ipfire.org.git] / src / backend / releases.py

#!/usr/bin/python

import hashlib
import logging
import os
import re
import urllib.parse
import yabencode

from . import database
from .misc import Object
from .decorators import *

TRACKERS = (
    "http://ipv4.tracker.ipfire.org:6969/announce",
    "udp://ipv4.tracker.ipfire.org:6969",
    "http://ipv6.tracker.ipfire.org:6969/announce",
    "udp://ipv6.tracker.ipfire.org:6969",
)

class File(Object):
	def __init__(self, backend, release, id, data=None):
		Object.__init__(self, backend)

		self.id = id
		self._release = release

		# get all data from database
		self.__data = data

	def __eq__(self, other):
		if isinstance(other, self.__class__):
			return self.id == otherid

	def __lt__(self, other):
		if isinstance(other, self.__class__):
			return self.prio < other.prio

	@property
	def data(self):
		if self.__data is None:
			self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
			assert self.__data

		return self.__data

	@property
	def release(self):
		if not self._release:
			release_id = self.data.get("releases")
			self._release = Release(release_id)

		return self._release

	@property
	def type(self):
		filename = self.filename

		if filename.endswith(".iso"):
			return "iso"

		elif filename.endswith(".torrent"):
			return "torrent"

		elif "xen" in filename:
			if "downloader" in filename:
				return "xen-downloader"

			return "xen"

		elif "sources" in filename:
			return "source"

		elif "usb-fdd" in filename:
			return "usbfdd"

		elif "usb-hdd" in filename:
			return "usbhdd"

		elif "armv5tel" in filename and "scon" in filename:
			return "armv5tel-scon"

		elif "armv5tel" in filename:
			return "armv5tel"

		elif "scon" in filename:
			return "alix"

		elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
			return "flash"

		else:
			return "unknown"

	@property
	def url(self):
		return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)

	@property
	def desc(self):
		_ = lambda x: x

		descriptions = {
			"armv5tel"	: _("Flash Image"),
			"armv5tel-scon"	: _("Flash Image with serial console"),
			"iso"		: _("ISO Image"),
			"torrent"	: _("Torrent File"),
			"flash"		: _("Flash Image"),
			"alix"		: _("Flash Image with serial console"),
			"usbfdd"	: _("USB FDD Image"),
			"usbhdd"	: _("USB HDD Image"),
			"xen"		: _("Pre-generated Xen Image"),
			"xen-downloader": _("Xen-Image Generator"),
		}

		try:
			return descriptions[self.type]
		except KeyError:
			return _("Unknown image type")

	@property
	def prio(self):
		priorities = {
			"iso"		: 10,
			"torrent"	: 20,
			"flash"		: 40,
			"alix"		: 41,
			"usbfdd"	: 31,
			"usbhdd"	: 30,
			"armv5tel"	: 40,
			"armv5tel-scon"	: 41,
			"xen"		: 50,
			"xen-downloader": 51,
		}
		
		try:
			return priorities[self.type]
		except KeyError:
			return 999

	@property
	def sha256(self):
		return self.data.get("sha256")

	@property
	def sha1(self):
		return self.data.get("sha1")

	@property
	def filename(self):
		return self.data.get("filename")

	@property
	def basename(self):
		return os.path.basename(self.filename)

	@property
	def size(self):
		return self.data.get("filesize")

	@property
	def arch(self):
		known_arches = ("x86_64", "aarch64", "arm", "i586")

		for arch in known_arches:
			if arch in self.basename:
				return arch

		return "N/A"

	@property
	def torrent_hash(self):
		return self.data.get("torrent_hash", None)

	@property
	def torrent_url(self):
		if self.torrent_hash:
			return "%s.torrent" % self.url

	@property
	def magnet_link(self):
		# Don't return anything if we have no torrent hash.
		if self.torrent_hash is None:
			return

		s = "magnet:?xt=urn:btih:%s" % self.torrent_hash

		#s += "&xl=%d" % self.size
		s += "&dn=%s" % urllib.parse.quote(self.basename)

		# Add our tracker.
		for tracker in TRACKERS:
			s += "&tr=%s" % tracker

		# Add web download URL
		s += "&as=%s" % urllib.parse.quote(self.url)

		return s


class Release(Object):
	def __init__(self, backend, id, data=None):
		Object.__init__(self, backend)
		self.id = id

		# get all data from database
		self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
		assert self.__data

		self.__files = []

	def __str__(self):
		return self.name

	def __repr__(self):
		return "<%s %s>" % (self.__class__.__name__, self.name)

	def __cmp__(self, other):
		return cmp(self.id, other.id)

	@property
	def arches(self):
		for arch in ("x86_64", "aarch64", "i586", "arm"):
			if arch in (f.arch for f in self.files):
				yield arch

	@property
	def primary_arches(self):
		arches = []

		# Add x86_64 when available, otherwise add i586
		for arch in ("x86_64", "i586"):
			if arch in self.arches:
				arches.append(arch)
				break

		# Add aarch64 if available
		if "aarch64" in self.arches:
			arches.append("aarch64")

		# Add ARM before 2.27 if available
		if "arm" in self.arches and self.sname < "ipfire-2.27-core159":
			arches.append("arm")

		return arches

	@property
	def secondary_arches(self):
		arches = []

		for arch in self.arches:
			if arch in self.primary_arches:
				continue

			arches.append(arch)

		return arches

	@property
	def experimental_arches(self):
		return []

	@property
	def files(self):
		if not self.__files:
			files = self.db.query("SELECT * FROM files WHERE releases = %s \
				AND NOT filename LIKE '%%.torrent'", self.id)

			self.__files = [File(self.backend, self, f.id, f) for f in files]
			self.__files.sort()

		return self.__files

	def get_files_by_arch(self, arch):
		for f in self.files:
			if f.arch == arch:
				yield f

	@property
	def torrents(self):
		torrents = []

		for file in self.files:
			if not file.torrent_hash:
				continue

			torrents.append(file)

		return torrents

	@property
	def name(self):
		return self.__data.name

	@property
	def slug(self):
		return self.__data.sname

	# XXX compat
	sname = slug

	@lazy_property
	def blog(self):
		if self.__data.blog_id:
			return self.backend.blog.get_by_id(self.__data.blog_id)

	@property
	def fireinfo_id(self):
		name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")

		res = self.db.get("SELECT id FROM fireinfo_releases \
			WHERE name = %s", name)

		if res:
			return res.id

	@property
	def stable(self):
		return self.__data.stable

	@property
	def published(self):
		return self.__data.published

	date = published

	@property
	def path(self):
		return self.__data.path

	def get_file(self, type):
		for file in self.files:
			if file.type == type:
				return file

	def __file_hash(self, filename, algo="sha256"):
		h = hashlib.new(algo)

		with open(filename, "rb") as f:
			buf_size = 1024
			buf = f.read(buf_size)
			while buf:
				h.update(buf)
				buf = f.read(buf_size)

		return h.hexdigest()

	def scan_files(self, basepath="/pub/mirror"):
		if not self.path:
			return

		path = os.path.join(basepath, self.path)
		if not os.path.exists(path):
			return

		files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
		files = [f.filename for f in files]

		# Make files that do not exists not loadable.
		for filename in files:
			_filename = os.path.join(basepath, filename)
			if not os.path.exists(_filename):
				self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)

		for filename in os.listdir(path):
			filename = os.path.join(path, filename)

			if os.path.isdir(filename):
				continue

			_filename = re.match(".*(releases/.*)", filename).group(1)
			if _filename in files:
				continue

			if filename.endswith(".md5"):
				continue

			logging.info("Hashing %s..." % filename)
			hash_sha256 = self.__file_hash(filename, "sha256")
			hash_sha1   = self.__file_hash(filename, "sha1")
			filesize = os.path.getsize(filename)

			# Check if there is a torrent download available for this file:
			torrent_hash = ""
			torrent_file = "%s.torrent" % filename
			if os.path.exists(torrent_file):
				torrent_hash = self.torrent_read_hash(torrent_file)

			self.db.execute("INSERT INTO files(releases, filename, filesize, \
				sha256, sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s, %s)",
				self.id, _filename, filesize, hash_sha256, hash_sha1, torrent_hash)

		# Search for all files that miss a torrent hash.
		files = self.db.query("SELECT id, filename FROM files \
			WHERE releases = %s AND torrent_hash IS NULL", self.id)

		for file in files:
			path = os.path.join(basepath, file.filename)

			torrent_file = "%s.torrent" % path
			if os.path.exists(torrent_file):
				torrent_hash = self.torrent_read_hash(torrent_file)

				self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
					torrent_hash, file.id)

	def torrent_read_hash(self, filename):
		with open(filename, "rb") as f:
			metainfo = yabencode.decode(f)
			metainfo = yabencode.encode(metainfo["info"])

			h = hashlib.new("sha1")
			h.update(metainfo)

			return h.hexdigest()

	def supports_arch(self, arch):
		return arch in ("x86_64", "i586")

	def supports_platform(self, platform):
		# Currently there is nothing else than pcbios supported
		if platform == "pcbios":
			return True

		return False

	def is_netboot_capable(self):
		return self.path and "ipfire-2.x" in self.path

	def netboot_kernel_url(self, arch, platform):
		assert self.supports_arch(arch)
		assert self.supports_platform(platform)

		if self.sname >= "ipfire-2.19-core100":
			return "http://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)

		return "http://boot.ipfire.org/%s/images/vmlinuz" % self.path

	def netboot_initrd_url(self, arch, platform):
		assert self.supports_arch(arch)
		assert self.supports_platform(platform)

		if self.sname >= "ipfire-2.19-core100":
			return "http://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch)

		return "http://boot.ipfire.org/%s/images/instroot" % self.path

	def netboot_args(self, arch, platform):
		return ""

	@property
	def post(self):
		if self.__data.blog_id:
			return self.backend.blog.get_by_id(self.__data.blog_id)

	# Fireinfo Stuff

	@property
	def penetration(self):
		# Get penetration from fireinfo
		return self.backend.fireinfo.get_release_penetration(self)


class Releases(Object):
	def _get_release(self, query, *args):
		res = self.db.get(query, *args)

		if res:
			return Release(self.backend, res.id, data=res)

	def _get_releases(self, query, *args):
		res = self.db.query(query, *args)

		for row in res:
			yield Release(self.backend, row.id, data=row)

	def __iter__(self):
		releases = self._get_releases("SELECT * FROM releases \
			ORDER BY published DESC NULLS FIRST")

		return iter(releases)

	def get_by_id(self, id):
		ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_by_sname(self, sname):
		ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_by_news_id(self, news_id):
		ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_latest(self, stable=True):
		ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
			AND stable = %s ORDER BY published DESC LIMIT 1", stable)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_releases_older_than(self, release, limit=None):
		return self._get_releases("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published < %s \
				ORDER BY published DESC LIMIT %s", release.published, limit)

	def get_latest_unstable(self):
		ret = self.db.get("SELECT * FROM releases r1 \
			WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
			AND stable = %s AND NOT EXISTS ( \
				SELECT * FROM releases r2 WHERE r2.stable = %s AND \
					r2.published IS NOT NULL AND r2.published >= r1.published \
				) ORDER BY r1.published DESC LIMIT 1", False, True)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_stable(self):
		query = self.db.query("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
			ORDER BY published DESC")

		releases = []
		for row in query:
			release = Release(self.backend, row.id, data=row)
			releases.append(release)

		return releases

	def get_unstable(self):
		query = self.db.query("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
			ORDER BY published DESC")

		releases = []
		for row in query:
			release = Release(self.backend, row.id, data=row)
			releases.append(release)

		return releases

	def get_all(self):
		query = self.db.query("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published <= NOW() \
			ORDER BY published DESC")

		releases = []
		for row in query:
			release = Release(self.backend, row.id, data=row)
			releases.append(release)

		return releases

	def _get_all(self):
		query = self.db.query("SELECT * FROM releases ORDER BY published DESC")

		releases = []
		for row in query:
			release = Release(self.backend, row.id, data=row)
			releases.append(release)

		return releases

	def get_file_for_torrent_hash(self, torrent_hash):
		file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
			torrent_hash)

		if not file:
			return

		release = Release(self.backend, file.releases)
		file = File(self.backend, release, file.id)

		return file

	async def scan_files(self, basepath="/pub/mirror"):
		for release in self:
			logging.debug("Scanning %s..." % release)

			with self.db.transaction():
				release.scan_files(basepath=basepath)
Commit	Line	Data
940227cb MT	1	#!/usr/bin/python
940227cb MT	2
3c4f2edc	3	import hashlib
940227cb	4	import logging
3c4f2edc MT	5	import os
3c4f2edc MT	6	import re
11347e46	7	import urllib.parse
3ec6114d	8	import yabencode
940227cb	9
11347e46	10	from . import database
11347e46	11	from .misc import Object
45c7049d	12	from .decorators import *
fadcfd00	13
f7635119 MT	14	TRACKERS = (
	15	"http://ipv4.tracker.ipfire.org:6969/announce",
	16	"udp://ipv4.tracker.ipfire.org:6969",
	17	"http://ipv6.tracker.ipfire.org:6969/announce",
	18	"udp://ipv6.tracker.ipfire.org:6969",
	19	)
	20
9068dba1 MT	21	class File(Object):
	22	def __init__(self, backend, release, id, data=None):
	23	Object.__init__(self, backend)
940227cb	24
938d083d MT	25	self.id = id
938d083d MT	26	self._release = release
940227cb MT	27
940227cb MT	28	# get all data from database
9068dba1	29	self.__data = data
940227cb	30
5ecd3be7 MT	31	def __eq__(self, other):
	32	if isinstance(other, self.__class__):
	33	return self.id == otherid
	34
	35	def __lt__(self, other):
	36	if isinstance(other, self.__class__):
	37	return self.prio < other.prio
938d083d MT	38
	39	@property
	40	def data(self):
	41	if self.__data is None:
	42	self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
	43	assert self.__data
	44
	45	return self.__data
	46
	47	@property
	48	def release(self):
	49	if not self._release:
	50	release_id = self.data.get("releases")
	51	self._release = Release(release_id)
	52
	53	return self._release
940227cb MT	54
	55	@property
	56	def type(self):
fadcfd00 MT	57	filename = self.filename
	58
	59	if filename.endswith(".iso"):
	60	return "iso"
	61
	62	elif filename.endswith(".torrent"):
	63	return "torrent"
	64
	65	elif "xen" in filename:
80594ae3 MT	66	if "downloader" in filename:
	67	return "xen-downloader"
	68
fadcfd00 MT	69	return "xen"
	70
	71	elif "sources" in filename:
	72	return "source"
	73
	74	elif "usb-fdd" in filename:
	75	return "usbfdd"
	76
	77	elif "usb-hdd" in filename:
	78	return "usbhdd"
	79
3230a47f MT	80	elif "armv5tel" in filename and "scon" in filename:
	81	return "armv5tel-scon"
	82
fadcfd00 MT	83	elif "armv5tel" in filename:
	84	return "armv5tel"
	85
	86	elif "scon" in filename:
	87	return "alix"
	88
116312c5	89	elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
fadcfd00 MT	90	return "flash"
	91
	92	else:
	93	return "unknown"
940227cb MT	94
	95	@property
	96	def url(self):
eea71144	97	return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
940227cb MT	98
	99	@property
	100	def desc(self):
	101	_ = lambda x: x
	102
	103	descriptions = {
e77cd04c MT	104	"armv5tel" : _("Flash Image"),
e77cd04c MT	105	"armv5tel-scon" : _("Flash Image with serial console"),
a47f3c78	106	"iso" : _("ISO Image"),
e77cd04c MT	107	"torrent" : _("Torrent File"),
	108	"flash" : _("Flash Image"),
	109	"alix" : _("Flash Image with serial console"),
940227cb MT	110	"usbfdd" : _("USB FDD Image"),
940227cb MT	111	"usbhdd" : _("USB HDD Image"),
e77cd04c	112	"xen" : _("Pre-generated Xen Image"),
80594ae3	113	"xen-downloader": _("Xen-Image Generator"),
940227cb MT	114	}
	115
	116	try:
	117	return descriptions[self.type]
	118	except KeyError:
	119	return _("Unknown image type")
	120
	121	@property
	122	def prio(self):
	123	priorities = {
	124	"iso" : 10,
	125	"torrent" : 20,
	126	"flash" : 40,
	127	"alix" : 41,
	128	"usbfdd" : 31,
	129	"usbhdd" : 30,
3230a47f MT	130	"armv5tel" : 40,
3230a47f MT	131	"armv5tel-scon" : 41,
940227cb	132	"xen" : 50,
80594ae3	133	"xen-downloader": 51,
940227cb MT	134	}
	135
	136	try:
	137	return priorities[self.type]
	138	except KeyError:
	139	return 999
	140
752c8888 MT	141	@property
	142	def sha256(self):
	143	return self.data.get("sha256")
	144
940227cb MT	145	@property
940227cb MT	146	def sha1(self):
938d083d	147	return self.data.get("sha1")
940227cb MT	148
	149	@property
	150	def filename(self):
938d083d	151	return self.data.get("filename")
940227cb	152
c9698a0b MT	153	@property
	154	def basename(self):
	155	return os.path.basename(self.filename)
	156
60024cc8 MT	157	@property
60024cc8 MT	158	def size(self):
938d083d	159	return self.data.get("filesize")
60024cc8 MT	160
	161	@property
	162	def arch(self):
6c62d21d	163	known_arches = ("x86_64", "aarch64", "arm", "i586")
60024cc8 MT	164
	165	for arch in known_arches:
	166	if arch in self.basename:
	167	return arch
	168
fadcfd00 MT	169	return "N/A"
	170
	171	@property
	172	def torrent_hash(self):
938d083d	173	return self.data.get("torrent_hash", None)
fadcfd00	174
a69e87a1 MT	175	@property
	176	def torrent_url(self):
	177	if self.torrent_hash:
	178	return "%s.torrent" % self.url
5276cd41	179
fadcfd00 MT	180	@property
	181	def magnet_link(self):
	182	# Don't return anything if we have no torrent hash.
	183	if self.torrent_hash is None:
	184	return
	185
	186	s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
	187
	188	#s += "&xl=%d" % self.size
11347e46	189	s += "&dn=%s" % urllib.parse.quote(self.basename)
fadcfd00 MT	190
fadcfd00 MT	191	# Add our tracker.
a69e87a1 MT	192	for tracker in TRACKERS:
a69e87a1 MT	193	s += "&tr=%s" % tracker
fadcfd00	194
a69e87a1 MT	195	# Add web download URL
a69e87a1 MT	196	s += "&as=%s" % urllib.parse.quote(self.url)
dc7b8879	197
fadcfd00	198	return s
60024cc8	199
940227cb	200
9068dba1 MT	201	class Release(Object):
	202	def __init__(self, backend, id, data=None):
	203	Object.__init__(self, backend)
940227cb MT	204	self.id = id
	205
	206	# get all data from database
9068dba1	207	self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
940227cb MT	208	assert self.__data
	209
	210	self.__files = []
	211
e77cd04c MT	212	def __str__(self):
	213	return self.name
	214
940227cb MT	215	def __repr__(self):
	216	return "<%s %s>" % (self.__class__.__name__, self.name)
	217
9068dba1 MT	218	def __cmp__(self, other):
	219	return cmp(self.id, other.id)
	220
e77cd04c MT	221	@property
	222	def arches(self):
	223	for arch in ("x86_64", "aarch64", "i586", "arm"):
	224	if arch in (f.arch for f in self.files):
	225	yield arch
	226
6c62d21d MT	227	@property
	228	def primary_arches(self):
	229	arches = []
	230
	231	# Add x86_64 when available, otherwise add i586
	232	for arch in ("x86_64", "i586"):
	233	if arch in self.arches:
	234	arches.append(arch)
	235	break
	236
4bf117d0 MT	237	# Add aarch64 if available
	238	if "aarch64" in self.arches:
	239	arches.append("aarch64")
	240
	241	# Add ARM before 2.27 if available
	242	if "arm" in self.arches and self.sname < "ipfire-2.27-core159":
6c62d21d MT	243	arches.append("arm")
	244
	245	return arches
	246
	247	@property
	248	def secondary_arches(self):
	249	arches = []
	250
	251	for arch in self.arches:
	252	if arch in self.primary_arches:
	253	continue
	254
	255	arches.append(arch)
	256
	257	return arches
	258
	259	@property
	260	def experimental_arches(self):
4bf117d0	261	return []
6c62d21d	262
940227cb MT	263	@property
	264	def files(self):
	265	if not self.__files:
9068dba1 MT	266	files = self.db.query("SELECT * FROM files WHERE releases = %s \
9068dba1 MT	267	AND NOT filename LIKE '%%.torrent'", self.id)
940227cb	268
9068dba1 MT	269	self.__files = [File(self.backend, self, f.id, f) for f in files]
9068dba1 MT	270	self.__files.sort()
940227cb MT	271
	272	return self.__files
	273
e77cd04c MT	274	def get_files_by_arch(self, arch):
	275	for f in self.files:
	276	if f.arch == arch:
	277	yield f
	278
938d083d MT	279	@property
	280	def torrents(self):
	281	torrents = []
	282
	283	for file in self.files:
	284	if not file.torrent_hash:
	285	continue
	286
	287	torrents.append(file)
	288
	289	return torrents
	290
940227cb MT	291	@property
940227cb MT	292	def name(self):
9068dba1 MT	293	return self.__data.name
	294
	295	@property
e77cd04c	296	def slug(self):
9068dba1	297	return self.__data.sname
940227cb	298
e77cd04c MT	299	# XXX compat
	300	sname = slug
	301
45c7049d	302	@lazy_property
e77cd04c MT	303	def blog(self):
	304	if self.__data.blog_id:
	305	return self.backend.blog.get_by_id(self.__data.blog_id)
	306
dd3a5446 MT	307	@property
	308	def fireinfo_id(self):
	309	name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
	310
	311	res = self.db.get("SELECT id FROM fireinfo_releases \
	312	WHERE name = %s", name)
	313
	314	if res:
	315	return res.id
	316
940227cb MT	317	@property
940227cb MT	318	def stable(self):
9068dba1	319	return self.__data.stable
940227cb MT	320
	321	@property
	322	def published(self):
9068dba1	323	return self.__data.published
940227cb	324
9068dba1	325	date = published
940227cb	326
3c4f2edc MT	327	@property
3c4f2edc MT	328	def path(self):
9068dba1	329	return self.__data.path
3c4f2edc	330
940227cb MT	331	def get_file(self, type):
	332	for file in self.files:
	333	if file.type == type:
	334	return file
	335
8b99a680 MT	336	def __file_hash(self, filename, algo="sha256"):
8b99a680 MT	337	h = hashlib.new(algo)
3c4f2edc	338
3ca4157e	339	with open(filename, "rb") as f:
b5f4eef0 MT	340	buf_size = 1024
	341	buf = f.read(buf_size)
	342	while buf:
8b99a680	343	h.update(buf)
b5f4eef0	344	buf = f.read(buf_size)
3c4f2edc	345
8b99a680	346	return h.hexdigest()
3c4f2edc	347
e136d583	348	def scan_files(self, basepath="/pub/mirror"):
3c4f2edc MT	349	if not self.path:
	350	return
	351
	352	path = os.path.join(basepath, self.path)
921d98cc MT	353	if not os.path.exists(path):
	354	return
	355
fadcfd00 MT	356	files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
fadcfd00 MT	357	files = [f.filename for f in files]
3c4f2edc MT	358
	359	# Make files that do not exists not loadable.
	360	for filename in files:
	361	_filename = os.path.join(basepath, filename)
	362	if not os.path.exists(_filename):
	363	self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
	364
	365	for filename in os.listdir(path):
	366	filename = os.path.join(path, filename)
	367
	368	if os.path.isdir(filename):
	369	continue
	370
	371	_filename = re.match(".(releases/.)", filename).group(1)
	372	if _filename in files:
	373	continue
	374
	375	if filename.endswith(".md5"):
	376	continue
	377
9068dba1	378	logging.info("Hashing %s..." % filename)
8b99a680 MT	379	hash_sha256 = self.__file_hash(filename, "sha256")
8b99a680 MT	380	hash_sha1 = self.__file_hash(filename, "sha1")
3c4f2edc	381	filesize = os.path.getsize(filename)
3c4f2edc	382
fadcfd00 MT	383	# Check if there is a torrent download available for this file:
	384	torrent_hash = ""
	385	torrent_file = "%s.torrent" % filename
	386	if os.path.exists(torrent_file):
	387	torrent_hash = self.torrent_read_hash(torrent_file)
	388
	389	self.db.execute("INSERT INTO files(releases, filename, filesize, \
8b99a680 MT	390	sha256, sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s, %s)",
8b99a680 MT	391	self.id, _filename, filesize, hash_sha256, hash_sha1, torrent_hash)
fadcfd00 MT	392
	393	# Search for all files that miss a torrent hash.
	394	files = self.db.query("SELECT id, filename FROM files \
	395	WHERE releases = %s AND torrent_hash IS NULL", self.id)
	396
	397	for file in files:
	398	path = os.path.join(basepath, file.filename)
	399
	400	torrent_file = "%s.torrent" % path
	401	if os.path.exists(torrent_file):
	402	torrent_hash = self.torrent_read_hash(torrent_file)
	403
	404	self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
	405	torrent_hash, file.id)
	406
	407	def torrent_read_hash(self, filename):
3ec6114d MT	408	with open(filename, "rb") as f:
	409	metainfo = yabencode.decode(f)
	410	metainfo = yabencode.encode(metainfo["info"])
fadcfd00	411
3ec6114d MT	412	h = hashlib.new("sha1")
3ec6114d MT	413	h.update(metainfo)
fadcfd00	414
3ec6114d	415	return h.hexdigest()
3c4f2edc	416
37b5c0cf	417	def supports_arch(self, arch):
6371bfda	418	return arch in ("x86_64", "i586")
37b5c0cf MT	419
	420	def supports_platform(self, platform):
	421	# Currently there is nothing else than pcbios supported
	422	if platform == "pcbios":
	423	return True
	424
	425	return False
	426
9068dba1 MT	427	def is_netboot_capable(self):
	428	return self.path and "ipfire-2.x" in self.path
	429
37b5c0cf MT	430	def netboot_kernel_url(self, arch, platform):
	431	assert self.supports_arch(arch)
	432	assert self.supports_platform(platform)
	433
6371bfda	434	if self.sname >= "ipfire-2.19-core100":
453c817c	435	return "http://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
6371bfda	436
453c817c	437	return "http://boot.ipfire.org/%s/images/vmlinuz" % self.path
940227cb	438
37b5c0cf MT	439	def netboot_initrd_url(self, arch, platform):
	440	assert self.supports_arch(arch)
	441	assert self.supports_platform(platform)
	442
6371bfda	443	if self.sname >= "ipfire-2.19-core100":
453c817c	444	return "http://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
6371bfda	445
453c817c	446	return "http://boot.ipfire.org/%s/images/instroot" % self.path
940227cb	447
37b5c0cf MT	448	def netboot_args(self, arch, platform):
	449	return ""
	450
2fed2438	451	@property
5baa3aef	452	def post(self):
487417ad MT	453	if self.__data.blog_id:
487417ad MT	454	return self.backend.blog.get_by_id(self.__data.blog_id)
2fed2438	455
dd3a5446 MT	456	# Fireinfo Stuff
	457
	458	@property
	459	def penetration(self):
dd3a5446	460	# Get penetration from fireinfo
b8a4acfb	461	return self.backend.fireinfo.get_release_penetration(self)
dd3a5446	462
940227cb	463
9068dba1	464	class Releases(Object):
984e4e7b MT	465	def _get_release(self, query, *args):
	466	res = self.db.get(query, *args)
	467
	468	if res:
	469	return Release(self.backend, res.id, data=res)
	470
e77cd04c MT	471	def _get_releases(self, query, *args):
	472	res = self.db.query(query, *args)
	473
	474	for row in res:
	475	yield Release(self.backend, row.id, data=row)
	476
856748b0 MT	477	def __iter__(self):
	478	releases = self._get_releases("SELECT * FROM releases \
	479	ORDER BY published DESC NULLS FIRST")
	480
	481	return iter(releases)
	482
940227cb	483	def get_by_id(self, id):
9068dba1 MT	484	ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
	485
	486	if ret:
	487	return Release(self.backend, ret.id, data=ret)
	488
	489	def get_by_sname(self, sname):
	490	ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
	491
	492	if ret:
	493	return Release(self.backend, ret.id, data=ret)
940227cb	494
9de13943 MT	495	def get_by_news_id(self, news_id):
	496	ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
	497
	498	if ret:
	499	return Release(self.backend, ret.id, data=ret)
	500
9068dba1 MT	501	def get_latest(self, stable=True):
	502	ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
	503	AND stable = %s ORDER BY published DESC LIMIT 1", stable)
940227cb	504
9068dba1 MT	505	if ret:
9068dba1 MT	506	return Release(self.backend, ret.id, data=ret)
940227cb	507
e77cd04c MT	508	def get_releases_older_than(self, release, limit=None):
	509	return self._get_releases("SELECT * FROM releases \
	510	WHERE published IS NOT NULL AND published < %s \
	511	ORDER BY published DESC LIMIT %s", release.published, limit)
	512
0aab96e0 MT	513	def get_latest_unstable(self):
	514	ret = self.db.get("SELECT * FROM releases r1 \
	515	WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
	516	AND stable = %s AND NOT EXISTS ( \
	517	SELECT * FROM releases r2 WHERE r2.stable = %s AND \
	518	r2.published IS NOT NULL AND r2.published >= r1.published \
	519	) ORDER BY r1.published DESC LIMIT 1", False, True)
	520
	521	if ret:
	522	return Release(self.backend, ret.id, data=ret)
	523
940227cb	524	def get_stable(self):
9068dba1 MT	525	query = self.db.query("SELECT * FROM releases \
	526	WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
	527	ORDER BY published DESC")
940227cb	528
9068dba1 MT	529	releases = []
	530	for row in query:
	531	release = Release(self.backend, row.id, data=row)
	532	releases.append(release)
	533
	534	return releases
940227cb MT	535
940227cb MT	536	def get_unstable(self):
9068dba1 MT	537	query = self.db.query("SELECT * FROM releases \
	538	WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
	539	ORDER BY published DESC")
	540
	541	releases = []
	542	for row in query:
	543	release = Release(self.backend, row.id, data=row)
	544	releases.append(release)
940227cb	545
9068dba1	546	return releases
940227cb MT	547
940227cb MT	548	def get_all(self):
9068dba1 MT	549	query = self.db.query("SELECT * FROM releases \
	550	WHERE published IS NOT NULL AND published <= NOW() \
	551	ORDER BY published DESC")
940227cb	552
9068dba1 MT	553	releases = []
	554	for row in query:
	555	release = Release(self.backend, row.id, data=row)
	556	releases.append(release)
	557
	558	return releases
940227cb	559
d0eee7f8 MT	560	def _get_all(self):
	561	query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
	562
	563	releases = []
	564	for row in query:
	565	release = Release(self.backend, row.id, data=row)
	566	releases.append(release)
	567
	568	return releases
	569
938d083d MT	570	def get_file_for_torrent_hash(self, torrent_hash):
938d083d MT	571	file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
fadcfd00 MT	572	torrent_hash)
	573
	574	if not file:
	575	return
	576
ea324f48 MT	577	release = Release(self.backend, file.releases)
ea324f48 MT	578	file = File(self.backend, release, file.id)
938d083d MT	579
938d083d MT	580	return file
856748b0	581
9fdf4fb7	582	async def scan_files(self, basepath="/pub/mirror"):
856748b0	583	for release in self:
fb51c9c7	584	logging.debug("Scanning %s..." % release)
c49610bb MT	585
	586	with self.db.transaction():
	587	release.scan_files(basepath=basepath)