[people/shoehn/ipfire.org.git] / webapp / backend / mirrors.py

#!/usr/bin/python

from __future__ import division

import datetime
import logging
import math
import os.path
import random
import socket
import time
import tornado.httpclient
import tornado.netutil
import urlparse

from misc import Object

class Downloads(Object):
	@property
	def total(self):
		ret = self.db.get("SELECT COUNT(*) AS total FROM log_download")

		return ret.total

	@property
	def today(self):
		ret = self.db.get("SELECT COUNT(*) AS today FROM log_download WHERE date::date = NOW()::date")

		return ret.today

	@property
	def yesterday(self):
		ret = self.db.get("SELECT COUNT(*) AS yesterday FROM log_download WHERE date::date = (NOW() - INTERVAL '1 day')::date")

		return ret.yesterday

	@property
	def daily_map(self):
		ret = self.db.query("SELECT date::date AS date, COUNT(*) AS downloads FROM log_download"
			" WHERE date::date BETWEEN (NOW() - INTERVAL '30 days')::date AND NOW()::date GROUP BY date::date")

		return ret

	def get_countries(self, duration="all"):
		query = "SELECT country_code, count(country_code) AS count FROM log_download"

		if duration == "today":
			query += " WHERE date::date = NOW()::date"

		query += " GROUP BY country_code ORDER BY count DESC"

		results = self.db.query(query)
		ret = {}

		count = sum([o.count for o in results])
		if count:
			for res in results:
				ret[res.country_code] = res.count / count

		return ret

	def get_mirror_load(self, duration="all"):
		query = "SELECT mirror, COUNT(mirror) AS count FROM log_download"

		if duration == "today":
			query += " WHERE date::date = NOW()::date"

		query += " GROUP BY mirror ORDER BY count DESC"

		results = self.db.query(query)
		ret = {}

		count = sum([o.count for o in results])
		if count:
			for res in results:
				mirror = self.mirrors.get(res.mirror)
				ret[mirror.hostname] = res.count / count

		return ret


class Mirrors(Object):
	def check_all(self):
		for mirror in self.get_all():
			mirror.check()

	def get(self, id):
		return Mirror(self.backend, id)

	def get_all(self):
		res = self.db.query("SELECT * FROM mirrors WHERE enabled = %s", True)

		mirrors = []
		for row in res:
			mirror = Mirror(self.backend, row.id, row)
			mirrors.append(mirror)

		return MirrorSet(self.backend, sorted(mirrors))

	def get_all_up(self):
		res = self.db.query("SELECT * FROM mirrors WHERE enabled = %s AND state = %s \
			ORDER BY hostname", True, "UP")

		mirrors = []
		for row in res:
			m = Mirror(self.backend, row.id, row)
			mirrors.append(m)

		return MirrorSet(self.backend, mirrors)

	def get_by_hostname(self, hostname):
		mirror = self.db.get("SELECT id FROM mirrors WHERE hostname=%s", hostname)

		return Mirror(self.backend, mirror.id)

	def get_with_file(self, filename, country=None):
		# XXX quick and dirty solution - needs a performance boost
		mirror_ids = [m.mirror for m in self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)]

		#if country:
		#	# Sort out all mirrors that are not preferred to the given country
		#	for mirror in self.get_for_country(country):
		#		if not mirror.id in mirror_ids:
		#			mirror_ids.remove(mirror.id)

		mirrors = []
		for mirror_id in mirror_ids:
			mirror = self.get(mirror_id)
			if not mirror.state == "UP":
				continue
			mirrors.append(mirror)

		return mirrors

	def get_for_country(self, country):
		# XXX need option for random order
		mirrors = self.db.query("SELECT id FROM mirrors WHERE prefer_for_countries LIKE %s", country)

		for mirror in mirrors:
			yield self.get(mirror.id)

	def get_for_location(self, location):
		if not location:
			return None

		distance = 2500

		mirrors = []
		all_mirrors = self.get_all()

		while all_mirrors and len(mirrors) <= 3 and distance <= 8000:
			for mirror in all_mirrors:
				mirror_distance = mirror.distance_to(location)
				if mirror_distance is None:
					continue

				if mirror_distance<= distance:
					mirrors.append(mirror)
					all_mirrors.remove(mirror)

			distance *= 1.2

		return mirrors

	def get_all_files(self):
		files = []

		for mirror in self.get_all():
			if not mirror.state == "UP":
				continue

			for file in mirror.filelist:
				if not file in files:
					files.append(file)

		return files


class MirrorSet(Object):
	def __init__(self, backend, mirrors):
		Object.__init__(self, backend)

		self._mirrors = mirrors

	def __add__(self, other):
		mirrors = []

		for mirror in self._mirrors + other._mirrors:
			if mirror in mirrors:
				continue

			mirrors.append(mirror)

		return MirrorSet(self.backend, mirrors)

	def __sub__(self, other):
		mirrors = self._mirrors[:]

		for mirror in other._mirrors:
			if mirror in mirrors:
				mirrors.remove(mirror)

		return MirrorSet(self.backend, mirrors)

	def __iter__(self):
		return iter(self._mirrors)

	def __len__(self):
		return len(self._mirrors)

	def __str__(self):
		return "<MirrorSet %s>" % ", ".join([m.hostname for m in self._mirrors])

	def get_with_file(self, filename):
		with_file = [m.mirror for m in self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)]

		mirrors = []
		for mirror in self._mirrors:
			if mirror.id in with_file:
				mirrors.append(mirror)

		return MirrorSet(self.backend, mirrors)

	def get_random(self):
		mirrors = []
		for mirror in self._mirrors:
			for i in range(0, mirror.priority):
				mirrors.append(mirror)

		return random.choice(mirrors)

	def get_for_country(self, country):
		mirrors = []

		for mirror in self._mirrors:
			if country in mirror.prefer_for_countries:
				mirrors.append(mirror)

		return MirrorSet(self.backend, mirrors)

	def get_for_location(self, location):
		distance = 2500
		mirrors = []

		if location:
			while len(mirrors) <= 3 and distance <= 8000:
				for mirror in self._mirrors:
					if mirror in mirrors:
						continue

					mirror_distance = mirror.distance_to(location)
					if mirror_distance is None:
						continue

					if mirror_distance <= distance:
						mirrors.append(mirror)

				distance *= 1.2

		return MirrorSet(self.backend, mirrors)

	def get_with_state(self, state):
		mirrors = []

		for mirror in self._mirrors:
			if mirror.state == state:
				mirrors.append(mirror)

		return MirrorSet(self.backend, mirrors)


class Mirror(Object):
	def __init__(self, backend, id, data=None):
		Object.__init__(self, backend)

		self.id = id

		if data:
			self._info = data
		else:
			self._info = self.db.get("SELECT * FROM mirrors WHERE id = %s", self.id)
		self._info["url"] = self.generate_url()

		self.__location = None
		self.__country_name = None

	def __repr__(self):
		return "<%s %s>" % (self.__class__.__name__, self.url)

	def __cmp__(self, other):
		ret = cmp(self.country_code, other.country_code)

		if not ret:
			ret = cmp(self.hostname, other.hostname)

		return ret

	def generate_url(self):
		url = "http://%s" % self.hostname
		if not self.path.startswith("/"):
			url += "/"
		url += "%s" % self.path
		if not self.path.endswith("/"):
			url += "/"
		return url

	@property
	def hostname(self):
		return self._info.hostname

	@property
	def path(self):
		return self._info.path

	@property
	def address(self):
		for addr in self.addresses4:
			return addr

		for addr in self.addresses6:
			return addr

	@property
	def owner(self):
		return self._info.owner

	@property
	def location(self):
		if self.__location is None:
			self.__location = self.geoip.get_location(self.address)

		return self.__location

	@property
	def latitude(self):
		if self.location:
			return self.location.latitude

	@property
	def longitude(self):
		if self.location:
			return self.location.longitude

	@property
	def coordinates(self):
		return (self.latitude, self.longitude)

	@property
	def coordiante_str(self):
		coordinates = []

		for i in self.coordinates:
			coordinates.append("%s" % i)

		return ",".join(coordinates)

	@property
	def country_code(self):
		if self.location:
			return self.location.country

	@property
	def country_name(self):
		if self.__country_name is None:
			self.__country_name = self.geoip.get_country_name(self.country_code)

		return self.__country_name

	@property
	def location_str(self):
		location = []

		if self._info.location:
			location.append(self._info.location)

		elif self.location:
			location.append(self.location.city)
			location.append(self.country_name)

		return ", ".join([s for s in location if s])

	@property
	def asn(self):
		if not hasattr(self, "__asn"):
			self.__asn = self.geoip.get_asn(self.address)

		return self.__asn

	@property
	def filelist(self):
		filelist = self.db.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self.id)
		return [f.filename for f in filelist]

	@property
	def prefix(self):
		return ""

	@property
	def url(self):
		return self._info.url

	def build_url(self, filename):
		return urlparse.urljoin(self.url, filename)

	@property
	def last_update(self):
		return self._info.last_update

	@property
	def state(self):
		return self._info.state

	def set_state(self, state):
		logging.info("Setting state of %s to %s" % (self.hostname, state))

		if self.state == state:
			return

		self.db.execute("UPDATE mirrors SET state = %s WHERE id = %s", state, self.id)

		# Reload changed settings
		if hasattr(self, "_info"):
			self._info["state"] = state

	@property
	def enabled(self):
		return self._info.enabled

	@property
	def disabled(self):
		return not self.enabled

	def check(self):
		logging.info("Running check for mirror %s" % self.hostname)

		self.check_timestamp()
		self.check_filelist()

	def check_state(self):
		logging.debug("Checking state of mirror %s" % self.id)

		if not self.enabled:
			self.set_state("DOWN")
			return

		now = datetime.datetime.utcnow()

		time_delta = now - self.last_update
		time_diff = time_delta.total_seconds()

		time_down = self.settings.get_int("mirrors_time_down", 3*24*60*60)
		if time_diff >= time_down:
			self.set_state("DOWN")
			return

		time_outofsync = self.settings.get_int("mirrors_time_outofsync", 6*60*60)
		if time_diff >= time_outofsync:
			self.set_state("OUTOFSYNC")
			return

		self.set_state("UP")

	def check_timestamp(self):
		http = tornado.httpclient.AsyncHTTPClient()

		http.fetch(self.url + ".timestamp",
			headers={ "Pragma" : "no-cache" },
			callback=self.__check_timestamp_response)

	def __check_timestamp_response(self, response):
		if response.error:
			logging.debug("Error getting timestamp from %s" % self.hostname)
			self.set_state("DOWN")
			return

		try:
			timestamp = int(response.body.strip())
		except ValueError:
			timestamp = 0

		timestamp = datetime.datetime.utcfromtimestamp(timestamp)

		self.db.execute("UPDATE mirrors SET last_update = %s WHERE id = %s",
			timestamp, self.id)

		# Reload changed settings
		if hasattr(self, "_info"):
			self._info["timestamp"] = timestamp

		self.check_state()

		logging.info("Successfully updated timestamp from %s" % self.hostname)

	def check_filelist(self):
		# XXX need to remove data from disabled mirrors
		if not self.enabled:
			return

		http = tornado.httpclient.AsyncHTTPClient()

		http.fetch(self.url + ".filelist",
			headers={ "Pragma" : "no-cache" },
			callback=self.__check_filelist_response)

	def __check_filelist_response(self, response):
		if response.error:
			logging.debug("Error getting timestamp from %s" % self.hostname)
			return

		files = self.filelist

		for file in response.body.splitlines():
			file = os.path.join(self.prefix, file)

			if file in files:
				files.remove(file)
				continue

			self.db.execute("INSERT INTO mirror_files(mirror, filename) VALUES(%s, %s)",
				self.id, file)

		for file in files:
			self.db.execute("DELETE FROM mirror_files WHERE mirror=%s AND filename=%s",
				self.id, file)

		logging.info("Successfully updated mirror filelist from %s" % self.hostname)

	@property
	def prefer_for_countries(self):
		countries = self._info.get("prefer_for_countries", "")
		if countries:
			return sorted(countries.split(", "))

		return []

	@property
	def prefer_for_countries_names(self):
		countries = [self.geoip.get_country_name(c.upper()) for c in self.prefer_for_countries]

		return sorted(countries)

	def distance_to(self, location, ignore_preference=False):
		if not location:
			return None

		country_code = None
		if location.country:
			country_code = location.country.lower()

		if not ignore_preference and country_code in self.prefer_for_countries:
			return 0

		# http://www.movable-type.co.uk/scripts/latlong.html

		if self.latitude is None:
			return None

		if self.longitude is None:
			return None

		earth = 6371 # km
		delta_lat = math.radians(self.latitude - location.latitude)
		delta_lon = math.radians(self.longitude - location.longitude)

		lat1 = math.radians(self.latitude)
		lat2 = math.radians(location.latitude)

		a = math.sin(delta_lat / 2) ** 2
		a += math.cos(lat1) * math.cos(lat2) * (math.sin(delta_lon / 2) ** 2)

		b1 = math.sqrt(a)
		b2 = math.sqrt(1 - a)

		c = 2 * math.atan2(b1, b2)

		return c * earth

	def traffic(self, since):
		# XXX needs to be done better

		files = {}
		for entry in self.db.query("SELECT filename, filesize FROM files"):
			files[entry.filename] = entry.filesize

		query = "SELECT COUNT(filename) as count, filename FROM log_download WHERE mirror = %s"
		query += " AND date >= %s GROUP BY filename"

		traffic = 0
		for entry in self.db.query(query, self.id, since):
			if files.has_key(entry.filename):
				traffic += entry.count * files[entry.filename]

		return traffic

	@property
	def priority(self):
		return self._info.get("priority", 10)

	@property
	def development(self):
		return self._info.get("mirrorlist_devel", False)

	@property
	def mirrorlist(self):
		return self._info.get("mirrorlist", False)

	@property
	def addresses(self):
		if not hasattr(self, "__addresses"):
			try:
				addrinfo = socket.getaddrinfo(self.hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM)
			except:
				raise Exception("Could not resolve %s" % self.hostname)

			ret = []
			for family, socktype, proto, canonname, address in addrinfo:
				if family == socket.AF_INET:
					address, port = address
				elif family == socket.AF_INET6:
					address, port, flowid, scopeid = address
				ret.append((family, address))

			self.__addresses = ret

		return self.__addresses

	@property
	def addresses6(self):
		return [address for family, address in self.addresses if family == socket.AF_INET6]

	@property
	def addresses4(self):
		return [address for family, address in self.addresses if family == socket.AF_INET]
Commit	Line	Data
940227cb MT	1	#!/usr/bin/python
940227cb MT	2
9068dba1 MT	3	from __future__ import division
	4
	5	import datetime
940227cb	6	import logging
0673d1b0	7	import math
54af860e	8	import os.path
0673d1b0	9	import random
940227cb MT	10	import socket
	11	import time
	12	import tornado.httpclient
9068dba1 MT	13	import tornado.netutil
9068dba1 MT	14	import urlparse
940227cb	15
9068dba1	16	from misc import Object
60024cc8	17
9068dba1	18	class Downloads(Object):
60024cc8 MT	19	@property
	20	def total(self):
	21	ret = self.db.get("SELECT COUNT(*) AS total FROM log_download")
	22
	23	return ret.total
	24
	25	@property
	26	def today(self):
9068dba1	27	ret = self.db.get("SELECT COUNT(*) AS today FROM log_download WHERE date::date = NOW()::date")
60024cc8 MT	28
	29	return ret.today
	30
	31	@property
	32	def yesterday(self):
9068dba1	33	ret = self.db.get("SELECT COUNT(*) AS yesterday FROM log_download WHERE date::date = (NOW() - INTERVAL '1 day')::date")
60024cc8 MT	34
	35	return ret.yesterday
	36
	37	@property
	38	def daily_map(self):
9068dba1 MT	39	ret = self.db.query("SELECT date::date AS date, COUNT(*) AS downloads FROM log_download"
9068dba1 MT	40	" WHERE date::date BETWEEN (NOW() - INTERVAL '30 days')::date AND NOW()::date GROUP BY date::date")
60024cc8 MT	41
	42	return ret
	43
	44	def get_countries(self, duration="all"):
	45	query = "SELECT country_code, count(country_code) AS count FROM log_download"
	46
	47	if duration == "today":
9068dba1	48	query += " WHERE date::date = NOW()::date"
60024cc8 MT	49
	50	query += " GROUP BY country_code ORDER BY count DESC"
	51
	52	results = self.db.query(query)
	53	ret = {}
	54
	55	count = sum([o.count for o in results])
9068dba1 MT	56	if count:
	57	for res in results:
	58	ret[res.country_code] = res.count / count
60024cc8 MT	59
	60	return ret
	61
	62	def get_mirror_load(self, duration="all"):
	63	query = "SELECT mirror, COUNT(mirror) AS count FROM log_download"
	64
	65	if duration == "today":
9068dba1	66	query += " WHERE date::date = NOW()::date"
60024cc8 MT	67
	68	query += " GROUP BY mirror ORDER BY count DESC"
	69
	70	results = self.db.query(query)
	71	ret = {}
	72
	73	count = sum([o.count for o in results])
9068dba1 MT	74	if count:
	75	for res in results:
	76	mirror = self.mirrors.get(res.mirror)
	77	ret[mirror.hostname] = res.count / count
60024cc8 MT	78
	79	return ret
	80
	81
9068dba1	82	class Mirrors(Object):
940227cb	83	def check_all(self):
9068dba1	84	for mirror in self.get_all():
940227cb MT	85	mirror.check()
	86
	87	def get(self, id):
9068dba1	88	return Mirror(self.backend, id)
940227cb	89
0673d1b0	90	def get_all(self):
9068dba1 MT	91	res = self.db.query("SELECT * FROM mirrors WHERE enabled = %s", True)
	92
	93	mirrors = []
	94	for row in res:
	95	mirror = Mirror(self.backend, row.id, row)
	96	mirrors.append(mirror)
	97
	98	return MirrorSet(self.backend, sorted(mirrors))
0673d1b0	99
bd17b7d1	100	def get_all_up(self):
9068dba1 MT	101	res = self.db.query("SELECT * FROM mirrors WHERE enabled = %s AND state = %s \
9068dba1 MT	102	ORDER BY hostname", True, "UP")
bd17b7d1 MT	103
	104	mirrors = []
	105	for row in res:
9068dba1	106	m = Mirror(self.backend, row.id, row)
bd17b7d1 MT	107	mirrors.append(m)
bd17b7d1 MT	108
9068dba1	109	return MirrorSet(self.backend, mirrors)
bd17b7d1	110
940227cb MT	111	def get_by_hostname(self, hostname):
	112	mirror = self.db.get("SELECT id FROM mirrors WHERE hostname=%s", hostname)
	113
9068dba1	114	return Mirror(self.backend, mirror.id)
940227cb	115
54af860e MT	116	def get_with_file(self, filename, country=None):
	117	# XXX quick and dirty solution - needs a performance boost
	118	mirror_ids = [m.mirror for m in self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)]
	119
	120	#if country:
	121	# # Sort out all mirrors that are not preferred to the given country
	122	# for mirror in self.get_for_country(country):
	123	# if not mirror.id in mirror_ids:
	124	# mirror_ids.remove(mirror.id)
	125
	126	mirrors = []
	127	for mirror_id in mirror_ids:
	128	mirror = self.get(mirror_id)
	129	if not mirror.state == "UP":
	130	continue
	131	mirrors.append(mirror)
	132
54af860e MT	133	return mirrors
	134
	135	def get_for_country(self, country):
	136	# XXX need option for random order
	137	mirrors = self.db.query("SELECT id FROM mirrors WHERE prefer_for_countries LIKE %s", country)
	138
	139	for mirror in mirrors:
	140	yield self.get(mirror.id)
940227cb	141
9068dba1	142	def get_for_location(self, location):
119f55d7 MT	143	if not location:
	144	return None
	145
9068dba1 MT	146	distance = 2500
	147
	148	mirrors = []
	149	all_mirrors = self.get_all()
	150
	151	while all_mirrors and len(mirrors) <= 3 and distance <= 8000:
0673d1b0	152	for mirror in all_mirrors:
9068dba1 MT	153	mirror_distance = mirror.distance_to(location)
	154	if mirror_distance is None:
	155	continue
	156
	157	if mirror_distance<= distance:
0673d1b0 MT	158	mirrors.append(mirror)
	159	all_mirrors.remove(mirror)
	160
	161	distance *= 1.2
	162
	163	return mirrors
	164
edd297c4 MT	165	def get_all_files(self):
	166	files = []
	167
9068dba1	168	for mirror in self.get_all():
edd297c4 MT	169	if not mirror.state == "UP":
	170	continue
	171
	172	for file in mirror.filelist:
	173	if not file in files:
	174	files.append(file)
	175
	176	return files
	177
940227cb	178
9068dba1 MT	179	class MirrorSet(Object):
	180	def __init__(self, backend, mirrors):
	181	Object.__init__(self, backend)
	182
0673d1b0 MT	183	self._mirrors = mirrors
	184
	185	def __add__(self, other):
	186	mirrors = []
	187
	188	for mirror in self._mirrors + other._mirrors:
	189	if mirror in mirrors:
	190	continue
	191
	192	mirrors.append(mirror)
	193
9068dba1	194	return MirrorSet(self.backend, mirrors)
0673d1b0 MT	195
	196	def __sub__(self, other):
	197	mirrors = self._mirrors[:]
	198
	199	for mirror in other._mirrors:
	200	if mirror in mirrors:
	201	mirrors.remove(mirror)
	202
9068dba1	203	return MirrorSet(self.backend, mirrors)
0673d1b0 MT	204
	205	def __iter__(self):
	206	return iter(self._mirrors)
	207
	208	def __len__(self):
	209	return len(self._mirrors)
	210
	211	def __str__(self):
	212	return "<MirrorSet %s>" % ", ".join([m.hostname for m in self._mirrors])
	213
0673d1b0 MT	214	def get_with_file(self, filename):
	215	with_file = [m.mirror for m in self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)]
	216
	217	mirrors = []
	218	for mirror in self._mirrors:
	219	if mirror.id in with_file:
	220	mirrors.append(mirror)
	221
9068dba1	222	return MirrorSet(self.backend, mirrors)
0673d1b0 MT	223
	224	def get_random(self):
	225	mirrors = []
	226	for mirror in self._mirrors:
f1f7eb7e	227	for i in range(0, mirror.priority):
0673d1b0 MT	228	mirrors.append(mirror)
	229
	230	return random.choice(mirrors)
	231
	232	def get_for_country(self, country):
	233	mirrors = []
	234
	235	for mirror in self._mirrors:
	236	if country in mirror.prefer_for_countries:
	237	mirrors.append(mirror)
	238
9068dba1	239	return MirrorSet(self.backend, mirrors)
0673d1b0	240
9068dba1 MT	241	def get_for_location(self, location):
9068dba1 MT	242	distance = 2500
0673d1b0 MT	243	mirrors = []
0673d1b0 MT	244
119f55d7	245	if location:
9068dba1	246	while len(mirrors) <= 3 and distance <= 8000:
119f55d7 MT	247	for mirror in self._mirrors:
	248	if mirror in mirrors:
	249	continue
0673d1b0	250
9068dba1 MT	251	mirror_distance = mirror.distance_to(location)
	252	if mirror_distance is None:
	253	continue
	254
	255	if mirror_distance <= distance:
119f55d7	256	mirrors.append(mirror)
0673d1b0	257
119f55d7	258	distance *= 1.2
0673d1b0	259
9068dba1	260	return MirrorSet(self.backend, mirrors)
0673d1b0 MT	261
	262	def get_with_state(self, state):
	263	mirrors = []
	264
	265	for mirror in self._mirrors:
	266	if mirror.state == state:
	267	mirrors.append(mirror)
	268
9068dba1 MT	269	return MirrorSet(self.backend, mirrors)
9068dba1 MT	270
0673d1b0	271
9068dba1 MT	272	class Mirror(Object):
	273	def __init__(self, backend, id, data=None):
	274	Object.__init__(self, backend)
0673d1b0	275
940227cb MT	276	self.id = id
940227cb MT	277
bd17b7d1 MT	278	if data:
	279	self._info = data
	280	else:
	281	self._info = self.db.get("SELECT * FROM mirrors WHERE id = %s", self.id)
	282	self._info["url"] = self.generate_url()
940227cb	283
119f55d7 MT	284	self.__location = None
	285	self.__country_name = None
	286
54af860e MT	287	def __repr__(self):
	288	return "<%s %s>" % (self.__class__.__name__, self.url)
	289
	290	def __cmp__(self, other):
9068dba1	291	ret = cmp(self.country_code, other.country_code)
54af860e	292
9068dba1 MT	293	if not ret:
	294	ret = cmp(self.hostname, other.hostname)
	295
	296	return ret
940227cb	297
940227cb MT	298	def generate_url(self):
	299	url = "http://%s" % self.hostname
	300	if not self.path.startswith("/"):
	301	url += "/"
	302	url += "%s" % self.path
	303	if not self.path.endswith("/"):
	304	url += "/"
	305	return url
	306
9068dba1 MT	307	@property
	308	def hostname(self):
	309	return self._info.hostname
	310
	311	@property
	312	def path(self):
	313	return self._info.path
940227cb MT	314
	315	@property
	316	def address(self):
b898caea MT	317	for addr in self.addresses4:
	318	return addr
	319
	320	for addr in self.addresses6:
	321	return addr
940227cb	322
9068dba1 MT	323	@property
	324	def owner(self):
	325	return self._info.owner
	326
0673d1b0 MT	327	@property
0673d1b0 MT	328	def location(self):
119f55d7	329	if self.__location is None:
9068dba1	330	self.__location = self.geoip.get_location(self.address)
0673d1b0 MT	331
	332	return self.__location
	333
	334	@property
	335	def latitude(self):
9068dba1 MT	336	if self.location:
9068dba1 MT	337	return self.location.latitude
0673d1b0 MT	338
	339	@property
	340	def longitude(self):
9068dba1 MT	341	if self.location:
9068dba1 MT	342	return self.location.longitude
0673d1b0 MT	343
	344	@property
	345	def coordinates(self):
	346	return (self.latitude, self.longitude)
	347
	348	@property
	349	def coordiante_str(self):
	350	coordinates = []
	351
	352	for i in self.coordinates:
	353	coordinates.append("%s" % i)
	354
	355	return ",".join(coordinates)
	356
940227cb MT	357	@property
940227cb MT	358	def country_code(self):
9068dba1 MT	359	if self.location:
9068dba1 MT	360	return self.location.country
940227cb	361
0673d1b0 MT	362	@property
0673d1b0 MT	363	def country_name(self):
119f55d7	364	if self.__country_name is None:
9068dba1	365	self.__country_name = self.geoip.get_country_name(self.country_code)
119f55d7 MT	366
119f55d7 MT	367	return self.__country_name
0673d1b0 MT	368
0673d1b0 MT	369	@property
9068dba1 MT	370	def location_str(self):
	371	location = []
	372
	373	if self._info.location:
	374	location.append(self._info.location)
0673d1b0	375
9068dba1 MT	376	elif self.location:
	377	location.append(self.location.city)
	378	location.append(self.country_name)
	379
	380	return ", ".join([s for s in location if s])
0673d1b0 MT	381
0673d1b0 MT	382	@property
9068dba1 MT	383	def asn(self):
	384	if not hasattr(self, "__asn"):
	385	self.__asn = self.geoip.get_asn(self.address)
0673d1b0	386
9068dba1	387	return self.__asn
0673d1b0	388
940227cb MT	389	@property
	390	def filelist(self):
	391	filelist = self.db.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self.id)
	392	return [f.filename for f in filelist]
	393
54af860e MT	394	@property
54af860e MT	395	def prefix(self):
54af860e MT	396	return ""
54af860e MT	397
9068dba1 MT	398	@property
	399	def url(self):
	400	return self._info.url
	401
	402	def build_url(self, filename):
	403	return urlparse.urljoin(self.url, filename)
	404
	405	@property
	406	def last_update(self):
	407	return self._info.last_update
	408
	409	@property
	410	def state(self):
	411	return self._info.state
	412
940227cb MT	413	def set_state(self, state):
	414	logging.info("Setting state of %s to %s" % (self.hostname, state))
	415
	416	if self.state == state:
	417	return
	418
9068dba1	419	self.db.execute("UPDATE mirrors SET state = %s WHERE id = %s", state, self.id)
940227cb MT	420
940227cb MT	421	# Reload changed settings
bd17b7d1 MT	422	if hasattr(self, "_info"):
bd17b7d1 MT	423	self._info["state"] = state
940227cb	424
9068dba1 MT	425	@property
	426	def enabled(self):
	427	return self._info.enabled
	428
	429	@property
	430	def disabled(self):
	431	return not self.enabled
	432
940227cb MT	433	def check(self):
	434	logging.info("Running check for mirror %s" % self.hostname)
	435
	436	self.check_timestamp()
	437	self.check_filelist()
	438
	439	def check_state(self):
	440	logging.debug("Checking state of mirror %s" % self.id)
	441
9068dba1	442	if not self.enabled:
940227cb	443	self.set_state("DOWN")
9068dba1 MT	444	return
	445
	446	now = datetime.datetime.utcnow()
	447
	448	time_delta = now - self.last_update
	449	time_diff = time_delta.total_seconds()
940227cb	450
9068dba1 MT	451	time_down = self.settings.get_int("mirrors_time_down", 32460*60)
9068dba1 MT	452	if time_diff >= time_down:
940227cb	453	self.set_state("DOWN")
9068dba1	454	return
940227cb	455
9068dba1 MT	456	time_outofsync = self.settings.get_int("mirrors_time_outofsync", 66060)
	457	if time_diff >= time_outofsync:
	458	self.set_state("OUTOFSYNC")
940227cb MT	459	return
940227cb MT	460
9068dba1 MT	461	self.set_state("UP")
	462
	463	def check_timestamp(self):
940227cb MT	464	http = tornado.httpclient.AsyncHTTPClient()
	465
	466	http.fetch(self.url + ".timestamp",
54af860e	467	headers={ "Pragma" : "no-cache" },
940227cb MT	468	callback=self.__check_timestamp_response)
	469
	470	def __check_timestamp_response(self, response):
	471	if response.error:
	472	logging.debug("Error getting timestamp from %s" % self.hostname)
a3ee39ce	473	self.set_state("DOWN")
940227cb MT	474	return
	475
	476	try:
	477	timestamp = int(response.body.strip())
	478	except ValueError:
	479	timestamp = 0
	480
ea324f48	481	timestamp = datetime.datetime.utcfromtimestamp(timestamp)
9068dba1 MT	482
9068dba1 MT	483	self.db.execute("UPDATE mirrors SET last_update = %s WHERE id = %s",
940227cb MT	484	timestamp, self.id)
	485
	486	# Reload changed settings
bd17b7d1 MT	487	if hasattr(self, "_info"):
bd17b7d1 MT	488	self._info["timestamp"] = timestamp
940227cb MT	489
	490	self.check_state()
	491
	492	logging.info("Successfully updated timestamp from %s" % self.hostname)
	493
	494	def check_filelist(self):
54af860e	495	# XXX need to remove data from disabled mirrors
9068dba1	496	if not self.enabled:
940227cb MT	497	return
	498
	499	http = tornado.httpclient.AsyncHTTPClient()
	500
	501	http.fetch(self.url + ".filelist",
54af860e	502	headers={ "Pragma" : "no-cache" },
940227cb MT	503	callback=self.__check_filelist_response)
	504
	505	def __check_filelist_response(self, response):
	506	if response.error:
	507	logging.debug("Error getting timestamp from %s" % self.hostname)
	508	return
	509
56b9c1d8	510	files = self.filelist
940227cb MT	511
940227cb MT	512	for file in response.body.splitlines():
56b9c1d8 MT	513	file = os.path.join(self.prefix, file)
	514
	515	if file in files:
	516	files.remove(file)
	517	continue
	518
940227cb	519	self.db.execute("INSERT INTO mirror_files(mirror, filename) VALUES(%s, %s)",
56b9c1d8 MT	520	self.id, file)
	521
	522	for file in files:
	523	self.db.execute("DELETE FROM mirror_files WHERE mirror=%s AND filename=%s",
	524	self.id, file)
940227cb MT	525
	526	logging.info("Successfully updated mirror filelist from %s" % self.hostname)
	527
54af860e MT	528	@property
54af860e MT	529	def prefer_for_countries(self):
0673d1b0 MT	530	countries = self._info.get("prefer_for_countries", "")
	531	if countries:
	532	return sorted(countries.split(", "))
54af860e	533
0673d1b0 MT	534	return []
	535
	536	@property
	537	def prefer_for_countries_names(self):
9068dba1 MT	538	countries = [self.geoip.get_country_name(c.upper()) for c in self.prefer_for_countries]
	539
	540	return sorted(countries)
54af860e	541
119f55d7	542	def distance_to(self, location, ignore_preference=False):
0673d1b0	543	if not location:
9068dba1	544	return None
940227cb	545
9068dba1 MT	546	country_code = None
	547	if location.country:
	548	country_code = location.country.lower()
	549
	550	if not ignore_preference and country_code in self.prefer_for_countries:
0673d1b0 MT	551	return 0
0673d1b0 MT	552
9068dba1 MT	553	# http://www.movable-type.co.uk/scripts/latlong.html
	554
	555	if self.latitude is None:
	556	return None
	557
	558	if self.longitude is None:
	559	return None
	560
	561	earth = 6371 # km
	562	delta_lat = math.radians(self.latitude - location.latitude)
	563	delta_lon = math.radians(self.longitude - location.longitude)
	564
	565	lat1 = math.radians(self.latitude)
	566	lat2 = math.radians(location.latitude)
	567
	568	a = math.sin(delta_lat / 2) ** 2
	569	a += math.cos(lat1) * math.cos(lat2) * (math.sin(delta_lon / 2) ** 2)
0673d1b0	570
9068dba1 MT	571	b1 = math.sqrt(a)
9068dba1 MT	572	b2 = math.sqrt(1 - a)
0673d1b0	573
9068dba1 MT	574	c = 2 * math.atan2(b1, b2)
	575
	576	return c * earth
0673d1b0 MT	577
	578	def traffic(self, since):
	579	# XXX needs to be done better
	580
	581	files = {}
	582	for entry in self.db.query("SELECT filename, filesize FROM files"):
	583	files[entry.filename] = entry.filesize
	584
	585	query = "SELECT COUNT(filename) as count, filename FROM log_download WHERE mirror = %s"
	586	query += " AND date >= %s GROUP BY filename"
	587
	588	traffic = 0
	589	for entry in self.db.query(query, self.id, since):
	590	if files.has_key(entry.filename):
	591	traffic += entry.count * files[entry.filename]
	592
	593	return traffic
	594
	595	@property
	596	def priority(self):
f1f7eb7e	597	return self._info.get("priority", 10)
940227cb	598
bd17b7d1 MT	599	@property
bd17b7d1 MT	600	def development(self):
ea324f48	601	return self._info.get("mirrorlist_devel", False)
bd17b7d1 MT	602
	603	@property
	604	def mirrorlist(self):
9068dba1 MT	605	return self._info.get("mirrorlist", False)
	606
	607	@property
	608	def addresses(self):
	609	if not hasattr(self, "__addresses"):
b898caea MT	610	try:
	611	addrinfo = socket.getaddrinfo(self.hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM)
	612	except:
	613	raise Exception("Could not resolve %s" % self.hostname)
9068dba1 MT	614
	615	ret = []
	616	for family, socktype, proto, canonname, address in addrinfo:
	617	if family == socket.AF_INET:
	618	address, port = address
	619	elif family == socket.AF_INET6:
	620	address, port, flowid, scopeid = address
	621	ret.append((family, address))
	622
	623	self.__addresses = ret
	624
	625	return self.__addresses
	626
	627	@property
	628	def addresses6(self):
	629	return [address for family, address in self.addresses if family == socket.AF_INET6]
	630
	631	@property
	632	def addresses4(self):
	633	return [address for family, address in self.addresses if family == socket.AF_INET]