[ipfire.org.git] / src / backend / releases.py

#!/usr/bin/python

import hashlib
import logging
import os
import re
import urllib.parse
import yabencode

from . import database
from .misc import Object
from .decorators import *

TRACKERS = (
    "http://ipv4.tracker.ipfire.org:6969/announce",
    "udp://ipv4.tracker.ipfire.org:6969",
    "http://ipv6.tracker.ipfire.org:6969/announce",
    "udp://ipv6.tracker.ipfire.org:6969",
)

class File(Object):
	def __init__(self, backend, release, id, data=None):
		Object.__init__(self, backend)

		self.id = id
		self._release = release

		# get all data from database
		self.__data = data

	def __eq__(self, other):
		if isinstance(other, self.__class__):
			return self.id == otherid

	def __lt__(self, other):
		if isinstance(other, self.__class__):
			return self.prio < other.prio

	@property
	def data(self):
		if self.__data is None:
			self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
			assert self.__data

		return self.__data

	@property
	def release(self):
		if not self._release:
			release_id = self.data.get("releases")
			self._release = Release(release_id)

		return self._release

	@property
	def type(self):
		filename = self.filename

		if filename.endswith(".iso"):
			return "iso"

		elif filename.endswith(".torrent"):
			return "torrent"

		elif "xen" in filename:
			if "downloader" in filename:
				return "xen-downloader"

			return "xen"

		elif "sources" in filename:
			return "source"

		elif "usb-fdd" in filename:
			return "usbfdd"

		elif "usb-hdd" in filename:
			return "usbhdd"

		elif "armv5tel" in filename and "scon" in filename:
			return "armv5tel-scon"

		elif "armv5tel" in filename:
			return "armv5tel"

		elif "scon" in filename:
			return "alix"

		elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
			return "flash"

		else:
			return "unknown"

	@property
	def url(self):
		return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)

	@property
	def desc(self):
		_ = lambda x: x

		descriptions = {
			"armv5tel"	: _("Flash Image"),
			"armv5tel-scon"	: _("Flash Image with serial console"),
			"iso"		: _("ISO Image"),
			"torrent"	: _("Torrent File"),
			"flash"		: _("Flash Image"),
			"alix"		: _("Flash Image with serial console"),
			"usbfdd"	: _("USB FDD Image"),
			"usbhdd"	: _("USB HDD Image"),
			"xen"		: _("Pre-generated Xen Image"),
			"xen-downloader": _("Xen-Image Generator"),
		}

		try:
			return descriptions[self.type]
		except KeyError:
			return _("Unknown image type")

	@property
	def prio(self):
		priorities = {
			"iso"		: 10,
			"torrent"	: 20,
			"flash"		: 40,
			"alix"		: 41,
			"usbfdd"	: 31,
			"usbhdd"	: 30,
			"armv5tel"	: 40,
			"armv5tel-scon"	: 41,
			"xen"		: 50,
			"xen-downloader": 51,
		}
		
		try:
			return priorities[self.type]
		except KeyError:
			return 999

	@property
	def sha256(self):
		return self.data.get("sha256")

	@property
	def sha1(self):
		return self.data.get("sha1")

	@property
	def filename(self):
		return self.data.get("filename")

	@property
	def basename(self):
		return os.path.basename(self.filename)

	@property
	def size(self):
		return self.data.get("filesize")

	@property
	def arch(self):
		known_arches = ("x86_64", "aarch64", "arm", "i586")

		for arch in known_arches:
			if arch in self.basename:
				return arch

		return "N/A"

	@property
	def torrent_hash(self):
		return self.data.get("torrent_hash", None)

	@property
	def torrent_url(self):
		if self.torrent_hash:
			return "%s.torrent" % self.url

	@property
	def magnet_link(self):
		# Don't return anything if we have no torrent hash.
		if self.torrent_hash is None:
			return

		s = "magnet:?xt=urn:btih:%s" % self.torrent_hash

		#s += "&xl=%d" % self.size
		s += "&dn=%s" % urllib.parse.quote(self.basename)

		# Add our tracker.
		for tracker in TRACKERS:
			s += "&tr=%s" % tracker

		# Add web download URL
		s += "&as=%s" % urllib.parse.quote(self.url)

		return s


class Release(Object):
	def __init__(self, backend, id, data=None):
		Object.__init__(self, backend)
		self.id = id

		# get all data from database
		self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
		assert self.__data

		self.__files = []

	def __str__(self):
		return self.name

	def __repr__(self):
		return "<%s %s>" % (self.__class__.__name__, self.name)

	def __cmp__(self, other):
		return cmp(self.id, other.id)

	@property
	def arches(self):
		for arch in ("x86_64", "aarch64", "i586", "arm"):
			if arch in (f.arch for f in self.files):
				yield arch

	@property
	def primary_arches(self):
		arches = []

		# Add x86_64 when available, otherwise add i586
		for arch in ("x86_64", "i586"):
			if arch in self.arches:
				arches.append(arch)
				break

		# Add ARM if available
		if "arm" in self.arches:
			arches.append("arm")

		return arches

	@property
	def secondary_arches(self):
		arches = []

		for arch in self.arches:
			if arch in self.primary_arches:
				continue

			arches.append(arch)

		return arches

	@property
	def experimental_arches(self):
		return ("aarch64",)

	@property
	def files(self):
		if not self.__files:
			files = self.db.query("SELECT * FROM files WHERE releases = %s \
				AND NOT filename LIKE '%%.torrent'", self.id)

			self.__files = [File(self.backend, self, f.id, f) for f in files]
			self.__files.sort()

		return self.__files

	def get_files_by_arch(self, arch):
		for f in self.files:
			if f.arch == arch:
				yield f

	@property
	def torrents(self):
		torrents = []

		for file in self.files:
			if not file.torrent_hash:
				continue

			torrents.append(file)

		return torrents

	@property
	def name(self):
		return self.__data.name

	@property
	def slug(self):
		return self.__data.sname

	# XXX compat
	sname = slug

	@lazy_property
	def blog(self):
		if self.__data.blog_id:
			return self.backend.blog.get_by_id(self.__data.blog_id)

	@property
	def fireinfo_id(self):
		name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")

		res = self.db.get("SELECT id FROM fireinfo_releases \
			WHERE name = %s", name)

		if res:
			return res.id

	@property
	def stable(self):
		return self.__data.stable

	@property
	def published(self):
		return self.__data.published

	date = published

	@property
	def path(self):
		return self.__data.path

	def get_file(self, type):
		for file in self.files:
			if file.type == type:
				return file

	def __file_hash(self, filename, algo="sha256"):
		h = hashlib.new(algo)

		with open(filename, "rb") as f:
			buf_size = 1024
			buf = f.read(buf_size)
			while buf:
				h.update(buf)
				buf = f.read(buf_size)

		return h.hexdigest()

	def scan_files(self, basepath="/pub/mirror"):
		if not self.path:
			return

		path = os.path.join(basepath, self.path)
		if not os.path.exists(path):
			return

		files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
		files = [f.filename for f in files]

		# Make files that do not exists not loadable.
		for filename in files:
			_filename = os.path.join(basepath, filename)
			if not os.path.exists(_filename):
				self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)

		for filename in os.listdir(path):
			filename = os.path.join(path, filename)

			if os.path.isdir(filename):
				continue

			_filename = re.match(".*(releases/.*)", filename).group(1)
			if _filename in files:
				continue

			if filename.endswith(".md5"):
				continue

			logging.info("Hashing %s..." % filename)
			hash_sha256 = self.__file_hash(filename, "sha256")
			hash_sha1   = self.__file_hash(filename, "sha1")
			filesize = os.path.getsize(filename)

			# Check if there is a torrent download available for this file:
			torrent_hash = ""
			torrent_file = "%s.torrent" % filename
			if os.path.exists(torrent_file):
				torrent_hash = self.torrent_read_hash(torrent_file)

			self.db.execute("INSERT INTO files(releases, filename, filesize, \
				sha256, sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s, %s)",
				self.id, _filename, filesize, hash_sha256, hash_sha1, torrent_hash)

		# Search for all files that miss a torrent hash.
		files = self.db.query("SELECT id, filename FROM files \
			WHERE releases = %s AND torrent_hash IS NULL", self.id)

		for file in files:
			path = os.path.join(basepath, file.filename)

			torrent_file = "%s.torrent" % path
			if os.path.exists(torrent_file):
				torrent_hash = self.torrent_read_hash(torrent_file)

				self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
					torrent_hash, file.id)

	def torrent_read_hash(self, filename):
		with open(filename, "rb") as f:
			metainfo = yabencode.decode(f)
			metainfo = yabencode.encode(metainfo["info"])

			h = hashlib.new("sha1")
			h.update(metainfo)

			return h.hexdigest()

	def supports_arch(self, arch):
		return arch in ("x86_64", "i586")

	def supports_platform(self, platform):
		# Currently there is nothing else than pcbios supported
		if platform == "pcbios":
			return True

		return False

	def is_netboot_capable(self):
		return self.path and "ipfire-2.x" in self.path

	def netboot_kernel_url(self, arch, platform):
		assert self.supports_arch(arch)
		assert self.supports_platform(platform)

		if self.sname >= "ipfire-2.19-core100":
			return "http://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)

		return "http://boot.ipfire.org/%s/images/vmlinuz" % self.path

	def netboot_initrd_url(self, arch, platform):
		assert self.supports_arch(arch)
		assert self.supports_platform(platform)

		if self.sname >= "ipfire-2.19-core100":
			return "http://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch)

		return "http://boot.ipfire.org/%s/images/instroot" % self.path

	def netboot_args(self, arch, platform):
		return ""

	@property
	def post(self):
		if self.__data.blog_id:
			return self.backend.blog.get_by_id(self.__data.blog_id)

	# Fireinfo Stuff

	@property
	def penetration(self):
		# Get penetration from fireinfo
		return self.backend.fireinfo.get_release_penetration(self)


class Releases(Object):
	def _get_release(self, query, *args):
		res = self.db.get(query, *args)

		if res:
			return Release(self.backend, res.id, data=res)

	def _get_releases(self, query, *args):
		res = self.db.query(query, *args)

		for row in res:
			yield Release(self.backend, row.id, data=row)

	def __iter__(self):
		releases = self._get_releases("SELECT * FROM releases \
			ORDER BY published DESC NULLS FIRST")

		return iter(releases)

	def get_by_id(self, id):
		ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_by_sname(self, sname):
		ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_by_news_id(self, news_id):
		ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_latest(self, stable=True):
		ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
			AND stable = %s ORDER BY published DESC LIMIT 1", stable)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_releases_older_than(self, release, limit=None):
		return self._get_releases("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published < %s \
				ORDER BY published DESC LIMIT %s", release.published, limit)

	def get_latest_unstable(self):
		ret = self.db.get("SELECT * FROM releases r1 \
			WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
			AND stable = %s AND NOT EXISTS ( \
				SELECT * FROM releases r2 WHERE r2.stable = %s AND \
					r2.published IS NOT NULL AND r2.published >= r1.published \
				) ORDER BY r1.published DESC LIMIT 1", False, True)

		if ret:
			return Release(self.backend, ret.id, data=ret)

	def get_stable(self):
		query = self.db.query("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
			ORDER BY published DESC")

		releases = []
		for row in query:
			release = Release(self.backend, row.id, data=row)
			releases.append(release)

		return releases

	def get_unstable(self):
		query = self.db.query("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
			ORDER BY published DESC")

		releases = []
		for row in query:
			release = Release(self.backend, row.id, data=row)
			releases.append(release)

		return releases

	def get_all(self):
		query = self.db.query("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published <= NOW() \
			ORDER BY published DESC")

		releases = []
		for row in query:
			release = Release(self.backend, row.id, data=row)
			releases.append(release)

		return releases

	def _get_all(self):
		query = self.db.query("SELECT * FROM releases ORDER BY published DESC")

		releases = []
		for row in query:
			release = Release(self.backend, row.id, data=row)
			releases.append(release)

		return releases

	def get_file_for_torrent_hash(self, torrent_hash):
		file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
			torrent_hash)

		if not file:
			return

		release = Release(self.backend, file.releases)
		file = File(self.backend, release, file.id)

		return file

	async def scan_files(self, basepath="/pub/mirror"):
		for release in self:
			logging.debug("Scanning %s..." % release)

			with self.db.transaction():
				release.scan_files(basepath=basepath)
Commit	Line	Data
940227cb MT	1	#!/usr/bin/python
940227cb MT	2
3c4f2edc	3	import hashlib
940227cb	4	import logging
3c4f2edc MT	5	import os
3c4f2edc MT	6	import re
11347e46	7	import urllib.parse
3ec6114d	8	import yabencode
940227cb	9
11347e46	10	from . import database
11347e46	11	from .misc import Object
45c7049d	12	from .decorators import *
fadcfd00	13
f7635119 MT	14	TRACKERS = (
	15	"http://ipv4.tracker.ipfire.org:6969/announce",
	16	"udp://ipv4.tracker.ipfire.org:6969",
	17	"http://ipv6.tracker.ipfire.org:6969/announce",
	18	"udp://ipv6.tracker.ipfire.org:6969",
	19	)
	20
9068dba1 MT	21	class File(Object):
	22	def __init__(self, backend, release, id, data=None):
	23	Object.__init__(self, backend)
940227cb	24
938d083d MT	25	self.id = id
938d083d MT	26	self._release = release
940227cb MT	27
940227cb MT	28	# get all data from database
9068dba1	29	self.__data = data
940227cb	30
5ecd3be7 MT	31	def __eq__(self, other):
	32	if isinstance(other, self.__class__):
	33	return self.id == otherid
	34
	35	def __lt__(self, other):
	36	if isinstance(other, self.__class__):
	37	return self.prio < other.prio
938d083d MT	38
	39	@property
	40	def data(self):
	41	if self.__data is None:
	42	self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
	43	assert self.__data
	44
	45	return self.__data
	46
	47	@property
	48	def release(self):
	49	if not self._release:
	50	release_id = self.data.get("releases")
	51	self._release = Release(release_id)
	52
	53	return self._release
940227cb MT	54
	55	@property
	56	def type(self):
fadcfd00 MT	57	filename = self.filename
	58
	59	if filename.endswith(".iso"):
	60	return "iso"
	61
	62	elif filename.endswith(".torrent"):
	63	return "torrent"
	64
	65	elif "xen" in filename:
80594ae3 MT	66	if "downloader" in filename:
	67	return "xen-downloader"
	68
fadcfd00 MT	69	return "xen"
	70
	71	elif "sources" in filename:
	72	return "source"
	73
	74	elif "usb-fdd" in filename:
	75	return "usbfdd"
	76
	77	elif "usb-hdd" in filename:
	78	return "usbhdd"
	79
3230a47f MT	80	elif "armv5tel" in filename and "scon" in filename:
	81	return "armv5tel-scon"
	82
fadcfd00 MT	83	elif "armv5tel" in filename:
	84	return "armv5tel"
	85
	86	elif "scon" in filename:
	87	return "alix"
	88
116312c5	89	elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
fadcfd00 MT	90	return "flash"
	91
	92	else:
	93	return "unknown"
940227cb MT	94
	95	@property
	96	def url(self):
eea71144	97	return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
940227cb MT	98
	99	@property
	100	def desc(self):
	101	_ = lambda x: x
	102
	103	descriptions = {
e77cd04c MT	104	"armv5tel" : _("Flash Image"),
e77cd04c MT	105	"armv5tel-scon" : _("Flash Image with serial console"),
a47f3c78	106	"iso" : _("ISO Image"),
e77cd04c MT	107	"torrent" : _("Torrent File"),
	108	"flash" : _("Flash Image"),
	109	"alix" : _("Flash Image with serial console"),
940227cb MT	110	"usbfdd" : _("USB FDD Image"),
940227cb MT	111	"usbhdd" : _("USB HDD Image"),
e77cd04c	112	"xen" : _("Pre-generated Xen Image"),
80594ae3	113	"xen-downloader": _("Xen-Image Generator"),
940227cb MT	114	}
	115
	116	try:
	117	return descriptions[self.type]
	118	except KeyError:
	119	return _("Unknown image type")
	120
	121	@property
	122	def prio(self):
	123	priorities = {
	124	"iso" : 10,
	125	"torrent" : 20,
	126	"flash" : 40,
	127	"alix" : 41,
	128	"usbfdd" : 31,
	129	"usbhdd" : 30,
3230a47f MT	130	"armv5tel" : 40,
3230a47f MT	131	"armv5tel-scon" : 41,
940227cb	132	"xen" : 50,
80594ae3	133	"xen-downloader": 51,
940227cb MT	134	}
	135
	136	try:
	137	return priorities[self.type]
	138	except KeyError:
	139	return 999
	140
752c8888 MT	141	@property
	142	def sha256(self):
	143	return self.data.get("sha256")
	144
940227cb MT	145	@property
940227cb MT	146	def sha1(self):
938d083d	147	return self.data.get("sha1")
940227cb MT	148
	149	@property
	150	def filename(self):
938d083d	151	return self.data.get("filename")
940227cb	152
c9698a0b MT	153	@property
	154	def basename(self):
	155	return os.path.basename(self.filename)
	156
60024cc8 MT	157	@property
60024cc8 MT	158	def size(self):
938d083d	159	return self.data.get("filesize")
60024cc8 MT	160
	161	@property
	162	def arch(self):
6c62d21d	163	known_arches = ("x86_64", "aarch64", "arm", "i586")
60024cc8 MT	164
	165	for arch in known_arches:
	166	if arch in self.basename:
	167	return arch
	168
fadcfd00 MT	169	return "N/A"
	170
	171	@property
	172	def torrent_hash(self):
938d083d	173	return self.data.get("torrent_hash", None)
fadcfd00	174
a69e87a1 MT	175	@property
	176	def torrent_url(self):
	177	if self.torrent_hash:
	178	return "%s.torrent" % self.url
5276cd41	179
fadcfd00 MT	180	@property
	181	def magnet_link(self):
	182	# Don't return anything if we have no torrent hash.
	183	if self.torrent_hash is None:
	184	return
	185
	186	s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
	187
	188	#s += "&xl=%d" % self.size
11347e46	189	s += "&dn=%s" % urllib.parse.quote(self.basename)
fadcfd00 MT	190
fadcfd00 MT	191	# Add our tracker.
a69e87a1 MT	192	for tracker in TRACKERS:
a69e87a1 MT	193	s += "&tr=%s" % tracker
fadcfd00	194
a69e87a1 MT	195	# Add web download URL
a69e87a1 MT	196	s += "&as=%s" % urllib.parse.quote(self.url)
dc7b8879	197
fadcfd00	198	return s
60024cc8	199
940227cb	200
9068dba1 MT	201	class Release(Object):
	202	def __init__(self, backend, id, data=None):
	203	Object.__init__(self, backend)
940227cb MT	204	self.id = id
	205
	206	# get all data from database
9068dba1	207	self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
940227cb MT	208	assert self.__data
	209
	210	self.__files = []
	211
e77cd04c MT	212	def __str__(self):
	213	return self.name
	214
940227cb MT	215	def __repr__(self):
	216	return "<%s %s>" % (self.__class__.__name__, self.name)
	217
9068dba1 MT	218	def __cmp__(self, other):
	219	return cmp(self.id, other.id)
	220
e77cd04c MT	221	@property
	222	def arches(self):
	223	for arch in ("x86_64", "aarch64", "i586", "arm"):
	224	if arch in (f.arch for f in self.files):
	225	yield arch
	226
6c62d21d MT	227	@property
	228	def primary_arches(self):
	229	arches = []
	230
	231	# Add x86_64 when available, otherwise add i586
	232	for arch in ("x86_64", "i586"):
	233	if arch in self.arches:
	234	arches.append(arch)
	235	break
	236
	237	# Add ARM if available
	238	if "arm" in self.arches:
	239	arches.append("arm")
	240
	241	return arches
	242
	243	@property
	244	def secondary_arches(self):
	245	arches = []
	246
	247	for arch in self.arches:
	248	if arch in self.primary_arches:
	249	continue
	250
	251	arches.append(arch)
	252
	253	return arches
	254
	255	@property
	256	def experimental_arches(self):
	257	return ("aarch64",)
	258
940227cb MT	259	@property
	260	def files(self):
	261	if not self.__files:
9068dba1 MT	262	files = self.db.query("SELECT * FROM files WHERE releases = %s \
9068dba1 MT	263	AND NOT filename LIKE '%%.torrent'", self.id)
940227cb	264
9068dba1 MT	265	self.__files = [File(self.backend, self, f.id, f) for f in files]
9068dba1 MT	266	self.__files.sort()
940227cb MT	267
	268	return self.__files
	269
e77cd04c MT	270	def get_files_by_arch(self, arch):
	271	for f in self.files:
	272	if f.arch == arch:
	273	yield f
	274
938d083d MT	275	@property
	276	def torrents(self):
	277	torrents = []
	278
	279	for file in self.files:
	280	if not file.torrent_hash:
	281	continue
	282
	283	torrents.append(file)
	284
	285	return torrents
	286
940227cb MT	287	@property
940227cb MT	288	def name(self):
9068dba1 MT	289	return self.__data.name
	290
	291	@property
e77cd04c	292	def slug(self):
9068dba1	293	return self.__data.sname
940227cb	294
e77cd04c MT	295	# XXX compat
	296	sname = slug
	297
45c7049d	298	@lazy_property
e77cd04c MT	299	def blog(self):
	300	if self.__data.blog_id:
	301	return self.backend.blog.get_by_id(self.__data.blog_id)
	302
dd3a5446 MT	303	@property
	304	def fireinfo_id(self):
	305	name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
	306
	307	res = self.db.get("SELECT id FROM fireinfo_releases \
	308	WHERE name = %s", name)
	309
	310	if res:
	311	return res.id
	312
940227cb MT	313	@property
940227cb MT	314	def stable(self):
9068dba1	315	return self.__data.stable
940227cb MT	316
	317	@property
	318	def published(self):
9068dba1	319	return self.__data.published
940227cb	320
9068dba1	321	date = published
940227cb	322
3c4f2edc MT	323	@property
3c4f2edc MT	324	def path(self):
9068dba1	325	return self.__data.path
3c4f2edc	326
940227cb MT	327	def get_file(self, type):
	328	for file in self.files:
	329	if file.type == type:
	330	return file
	331
8b99a680 MT	332	def __file_hash(self, filename, algo="sha256"):
8b99a680 MT	333	h = hashlib.new(algo)
3c4f2edc	334
3ca4157e	335	with open(filename, "rb") as f:
b5f4eef0 MT	336	buf_size = 1024
	337	buf = f.read(buf_size)
	338	while buf:
8b99a680	339	h.update(buf)
b5f4eef0	340	buf = f.read(buf_size)
3c4f2edc	341
8b99a680	342	return h.hexdigest()
3c4f2edc	343
e136d583	344	def scan_files(self, basepath="/pub/mirror"):
3c4f2edc MT	345	if not self.path:
	346	return
	347
	348	path = os.path.join(basepath, self.path)
921d98cc MT	349	if not os.path.exists(path):
	350	return
	351
fadcfd00 MT	352	files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
fadcfd00 MT	353	files = [f.filename for f in files]
3c4f2edc MT	354
	355	# Make files that do not exists not loadable.
	356	for filename in files:
	357	_filename = os.path.join(basepath, filename)
	358	if not os.path.exists(_filename):
	359	self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
	360
	361	for filename in os.listdir(path):
	362	filename = os.path.join(path, filename)
	363
	364	if os.path.isdir(filename):
	365	continue
	366
	367	_filename = re.match(".(releases/.)", filename).group(1)
	368	if _filename in files:
	369	continue
	370
	371	if filename.endswith(".md5"):
	372	continue
	373
9068dba1	374	logging.info("Hashing %s..." % filename)
8b99a680 MT	375	hash_sha256 = self.__file_hash(filename, "sha256")
8b99a680 MT	376	hash_sha1 = self.__file_hash(filename, "sha1")
3c4f2edc	377	filesize = os.path.getsize(filename)
3c4f2edc	378
fadcfd00 MT	379	# Check if there is a torrent download available for this file:
	380	torrent_hash = ""
	381	torrent_file = "%s.torrent" % filename
	382	if os.path.exists(torrent_file):
	383	torrent_hash = self.torrent_read_hash(torrent_file)
	384
	385	self.db.execute("INSERT INTO files(releases, filename, filesize, \
8b99a680 MT	386	sha256, sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s, %s)",
8b99a680 MT	387	self.id, _filename, filesize, hash_sha256, hash_sha1, torrent_hash)
fadcfd00 MT	388
	389	# Search for all files that miss a torrent hash.
	390	files = self.db.query("SELECT id, filename FROM files \
	391	WHERE releases = %s AND torrent_hash IS NULL", self.id)
	392
	393	for file in files:
	394	path = os.path.join(basepath, file.filename)
	395
	396	torrent_file = "%s.torrent" % path
	397	if os.path.exists(torrent_file):
	398	torrent_hash = self.torrent_read_hash(torrent_file)
	399
	400	self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
	401	torrent_hash, file.id)
	402
	403	def torrent_read_hash(self, filename):
3ec6114d MT	404	with open(filename, "rb") as f:
	405	metainfo = yabencode.decode(f)
	406	metainfo = yabencode.encode(metainfo["info"])
fadcfd00	407
3ec6114d MT	408	h = hashlib.new("sha1")
3ec6114d MT	409	h.update(metainfo)
fadcfd00	410
3ec6114d	411	return h.hexdigest()
3c4f2edc	412
37b5c0cf	413	def supports_arch(self, arch):
6371bfda	414	return arch in ("x86_64", "i586")
37b5c0cf MT	415
	416	def supports_platform(self, platform):
	417	# Currently there is nothing else than pcbios supported
	418	if platform == "pcbios":
	419	return True
	420
	421	return False
	422
9068dba1 MT	423	def is_netboot_capable(self):
	424	return self.path and "ipfire-2.x" in self.path
	425
37b5c0cf MT	426	def netboot_kernel_url(self, arch, platform):
	427	assert self.supports_arch(arch)
	428	assert self.supports_platform(platform)
	429
6371bfda	430	if self.sname >= "ipfire-2.19-core100":
453c817c	431	return "http://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
6371bfda	432
453c817c	433	return "http://boot.ipfire.org/%s/images/vmlinuz" % self.path
940227cb	434
37b5c0cf MT	435	def netboot_initrd_url(self, arch, platform):
	436	assert self.supports_arch(arch)
	437	assert self.supports_platform(platform)
	438
6371bfda	439	if self.sname >= "ipfire-2.19-core100":
453c817c	440	return "http://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
6371bfda	441
453c817c	442	return "http://boot.ipfire.org/%s/images/instroot" % self.path
940227cb	443
37b5c0cf MT	444	def netboot_args(self, arch, platform):
	445	return ""
	446
2fed2438	447	@property
5baa3aef	448	def post(self):
487417ad MT	449	if self.__data.blog_id:
487417ad MT	450	return self.backend.blog.get_by_id(self.__data.blog_id)
2fed2438	451
dd3a5446 MT	452	# Fireinfo Stuff
	453
	454	@property
	455	def penetration(self):
dd3a5446	456	# Get penetration from fireinfo
b8a4acfb	457	return self.backend.fireinfo.get_release_penetration(self)
dd3a5446	458
940227cb	459
9068dba1	460	class Releases(Object):
984e4e7b MT	461	def _get_release(self, query, *args):
	462	res = self.db.get(query, *args)
	463
	464	if res:
	465	return Release(self.backend, res.id, data=res)
	466
e77cd04c MT	467	def _get_releases(self, query, *args):
	468	res = self.db.query(query, *args)
	469
	470	for row in res:
	471	yield Release(self.backend, row.id, data=row)
	472
856748b0 MT	473	def __iter__(self):
	474	releases = self._get_releases("SELECT * FROM releases \
	475	ORDER BY published DESC NULLS FIRST")
	476
	477	return iter(releases)
	478
940227cb	479	def get_by_id(self, id):
9068dba1 MT	480	ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
	481
	482	if ret:
	483	return Release(self.backend, ret.id, data=ret)
	484
	485	def get_by_sname(self, sname):
	486	ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
	487
	488	if ret:
	489	return Release(self.backend, ret.id, data=ret)
940227cb	490
9de13943 MT	491	def get_by_news_id(self, news_id):
	492	ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
	493
	494	if ret:
	495	return Release(self.backend, ret.id, data=ret)
	496
9068dba1 MT	497	def get_latest(self, stable=True):
	498	ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
	499	AND stable = %s ORDER BY published DESC LIMIT 1", stable)
940227cb	500
9068dba1 MT	501	if ret:
9068dba1 MT	502	return Release(self.backend, ret.id, data=ret)
940227cb	503
e77cd04c MT	504	def get_releases_older_than(self, release, limit=None):
	505	return self._get_releases("SELECT * FROM releases \
	506	WHERE published IS NOT NULL AND published < %s \
	507	ORDER BY published DESC LIMIT %s", release.published, limit)
	508
0aab96e0 MT	509	def get_latest_unstable(self):
	510	ret = self.db.get("SELECT * FROM releases r1 \
	511	WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
	512	AND stable = %s AND NOT EXISTS ( \
	513	SELECT * FROM releases r2 WHERE r2.stable = %s AND \
	514	r2.published IS NOT NULL AND r2.published >= r1.published \
	515	) ORDER BY r1.published DESC LIMIT 1", False, True)
	516
	517	if ret:
	518	return Release(self.backend, ret.id, data=ret)
	519
940227cb	520	def get_stable(self):
9068dba1 MT	521	query = self.db.query("SELECT * FROM releases \
	522	WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
	523	ORDER BY published DESC")
940227cb	524
9068dba1 MT	525	releases = []
	526	for row in query:
	527	release = Release(self.backend, row.id, data=row)
	528	releases.append(release)
	529
	530	return releases
940227cb MT	531
940227cb MT	532	def get_unstable(self):
9068dba1 MT	533	query = self.db.query("SELECT * FROM releases \
	534	WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
	535	ORDER BY published DESC")
	536
	537	releases = []
	538	for row in query:
	539	release = Release(self.backend, row.id, data=row)
	540	releases.append(release)
940227cb	541
9068dba1	542	return releases
940227cb MT	543
940227cb MT	544	def get_all(self):
9068dba1 MT	545	query = self.db.query("SELECT * FROM releases \
	546	WHERE published IS NOT NULL AND published <= NOW() \
	547	ORDER BY published DESC")
940227cb	548
9068dba1 MT	549	releases = []
	550	for row in query:
	551	release = Release(self.backend, row.id, data=row)
	552	releases.append(release)
	553
	554	return releases
940227cb	555
d0eee7f8 MT	556	def _get_all(self):
	557	query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
	558
	559	releases = []
	560	for row in query:
	561	release = Release(self.backend, row.id, data=row)
	562	releases.append(release)
	563
	564	return releases
	565
938d083d MT	566	def get_file_for_torrent_hash(self, torrent_hash):
938d083d MT	567	file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
fadcfd00 MT	568	torrent_hash)
	569
	570	if not file:
	571	return
	572
ea324f48 MT	573	release = Release(self.backend, file.releases)
ea324f48 MT	574	file = File(self.backend, release, file.id)
938d083d MT	575
938d083d MT	576	return file
856748b0	577
9fdf4fb7	578	async def scan_files(self, basepath="/pub/mirror"):
856748b0	579	for release in self:
fb51c9c7	580	logging.debug("Scanning %s..." % release)
c49610bb MT	581
	582	with self.db.transaction():
	583	release.scan_files(basepath=basepath)