From 119f55d73fb3b5e8b6def6d264204ae3a1befe12 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Thu, 29 Nov 2012 18:33:14 +0100 Subject: [PATCH] Use new geoip database. --- www/templates/mirrors-item.html | 29 ++++++++++--- www/templates/mirrors.html | 4 +- www/translations/webapp.pot | 28 +++++++++--- www/webapp/backend/geoip.py | 75 ++++++++++++++++++--------------- www/webapp/backend/mirrors.py | 39 +++++++++++------ www/webapp/handlers_mirrors.py | 11 +++-- 6 files changed, 120 insertions(+), 66 deletions(-) diff --git a/www/templates/mirrors-item.html b/www/templates/mirrors-item.html index 2cbb5c66..cc375a2d 100644 --- a/www/templates/mirrors-item.html +++ b/www/templates/mirrors-item.html @@ -28,10 +28,16 @@ {% if item.prefer_for_countries %} - {{ _("Preferred for") }}: + {{ _("Preferred for") }} {{ locale.list(item.prefer_for_countries_names) }} {% end %} + {% if client_distance %} + + {{ _("Your distance to this mirror") }} + {{ "%.1f km" % client_distance }} + + {% end %} {{ _("Go to mirror") }} @@ -43,9 +49,20 @@

{{ _("The mirror %s is located in %s.") % (item.hostname, item.location_str) }}

-

- {{ _( -

+ + {% if item.longitude and item.latitude %} + +

+ {{ _("View larger map") }} + - + © OpenStreetMap contributors, CC-BY-SA +

+

+ {{ _("The location of the mirror server is estimated by the IP address.") }} +

+ {% else %} + {{ _("The location of the mirror server could not be estimated.") }} + {% end %} {% end block %} diff --git a/www/templates/mirrors.html b/www/templates/mirrors.html index 0e719087..1937f80f 100644 --- a/www/templates/mirrors.html +++ b/www/templates/mirrors.html @@ -21,7 +21,7 @@

{% end %} - {% if other_mirrors %} + {% if preferred_mirrors %}

{{ _("Mirror servers nearby") }}

{{ modules.MirrorsTable(preferred_mirrors) }} @@ -29,6 +29,6 @@ {{ modules.MirrorsTable(other_mirrors) }} {% else %}

{{ _("Worldwide mirror servers") }}

- {{ modules.MirrorsTable(preferred_mirrors) }} + {{ modules.MirrorsTable(other_mirrors) }} {% end %} {% end block %} diff --git a/www/translations/webapp.pot b/www/translations/webapp.pot index f7986937..8b7e827b 100644 --- a/www/translations/webapp.pot +++ b/www/translations/webapp.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2012-11-08 21:41+0100\n" +"POT-Creation-Date: 2012-11-29 18:31+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -22,12 +22,12 @@ msgstr "" msgid "Wish" msgstr "" -#: templates/wishlist/wish.html:16 +#: templates/wishlist/wish.html:18 #, python-format msgid "Launched: %s" msgstr "" -#: templates/wishlist/wish.html:20 +#: templates/wishlist/wish.html:23 #, python-format msgid "Funding ends: %s" msgstr "" @@ -203,18 +203,34 @@ msgid "Preferred for" msgstr "" #: templates/mirrors-item.html:37 +msgid "Your distance to this mirror" +msgstr "" + +#: templates/mirrors-item.html:43 msgid "Go to mirror" msgstr "" -#: templates/mirrors-item.html:42 +#: templates/mirrors-item.html:48 msgid "Mirror location" msgstr "" -#: templates/mirrors-item.html:44 +#: templates/mirrors-item.html:50 #, python-format msgid "The mirror %s is located in %s." msgstr "" +#: templates/mirrors-item.html:58 +msgid "View larger map" +msgstr "" + +#: templates/mirrors-item.html:63 +msgid "The location of the mirror server is estimated by the IP address." +msgstr "" + +#: templates/mirrors-item.html:66 +msgid "The location of the mirror server could not be estimated." +msgstr "" + #: templates/admin-downloads-mirrors.html:5 templates/admin-downloads.html:5 msgid "Download statistics" msgstr "" @@ -1411,7 +1427,7 @@ msgstr "" msgid "%s to %s" msgstr "" -#: webapp/backend/wishlist.py:128 +#: webapp/backend/wishlist.py:135 msgid "Checkout this crowdfunding wish from #ipfire:" msgstr "" diff --git a/www/webapp/backend/geoip.py b/www/webapp/backend/geoip.py index 123bc772..767488b0 100644 --- a/www/webapp/backend/geoip.py +++ b/www/webapp/backend/geoip.py @@ -9,54 +9,48 @@ from misc import Singleton class GeoIP(object): __metaclass__ = Singleton - def __init__(self): - self.__country_codes = self.db.query("SELECT code, name FROM iso3166_countries") - @property def db(self): return Databases().geoip - @property - def memcached(self): - return Memcached() - - def __encode_ip(self, addr): + def _encode_ip(self, addr): # We get a tuple if there were proxy headers. addr = addr.split(", ") if addr: addr = addr[-1] - # ip is calculated as described in http://ipinfodb.com/ip_database.php + # ip is calculated as described in http://dev.maxmind.com/geoip/csv a1, a2, a3, a4 = addr.split(".") - return int(((int(a1) * 256 + int(a2)) * 256 + int(a3)) * 256 + int(a4) + 100) + try: + a1 = int(a1) + a2 = int(a2) + a3 = int(a3) + a4 = int(a4) + except ValueError: + return 0 + + return (16777216 * a1) + (65536 * a2) + (256 * a3) + a4 def get_country(self, addr): - addr = self.__encode_ip(addr) + addr = self._encode_ip(addr) - mem_id = "geoip-country-%s" % addr - ret = self.memcached.get(mem_id) + ret = self.db.get("SELECT locations.country_code AS country_code FROM addresses \ + JOIN locations ON locations.id = addresses.location \ + WHERE %s BETWEEN start_ip_num AND end_ip_num LIMIT 1", addr) - if not ret: - ret = self.db.get("SELECT * FROM ip_group_country WHERE ip_start <= %s \ - ORDER BY ip_start DESC LIMIT 1;", addr).country_code.lower() - self.memcached.set(mem_id, ret, 3600) - - return ret + if ret: + return ret.country_code def get_all(self, addr): - addr = self.__encode_ip(addr) + addr = self._encode_ip(addr) - mem_id = "geoip-all-%s" % addr - ret = self.memcached.get(mem_id) + ret = self.db.get("SELECT locations.* FROM addresses \ + JOIN locations ON locations.id = addresses.location \ + WHERE %s BETWEEN start_ip_num AND end_ip_num LIMIT 1", addr) if not ret: - # XXX should be done with a join - location = self.db.get("SELECT location FROM ip_group_city WHERE ip_start <= %s \ - ORDER BY ip_start DESC LIMIT 1;", addr).location - - ret = self.db.get("SELECT * FROM locations WHERE id = %s", int(location)) - self.memcached.set(mem_id, ret, 3600) + return # If location was not determinable if ret.latitude == 0 and ret.longitude == 0: @@ -65,13 +59,24 @@ class GeoIP(object): return ret def get_country_name(self, code): - name = "Unknown" - - code = code.upper() - for country in self.__country_codes: - if country.code == code: - name = country.name - break + name = "Unkown" + + codes = { + "A1" : "Anonymous Proxy", + "A2" : "Satellite Provider", + "EU" : "Europe", + "AP" : "Asia/Pacific Region", + } + + # Return description of some exceptional codes. + try: + return codes[code] + except KeyError: + pass + + ret = self.db.get("SELECT name FROM iso3166_countries WHERE code = %s LIMIT 1", code) + if ret: + name = ret.name # Fix some weird strings name = re.sub(r"(.*) (.* Republic of)", r"\2 \1", name) diff --git a/www/webapp/backend/mirrors.py b/www/webapp/backend/mirrors.py index 87c237dd..38040464 100644 --- a/www/webapp/backend/mirrors.py +++ b/www/webapp/backend/mirrors.py @@ -148,9 +148,13 @@ class Mirrors(object): mirrors = [] all_mirrors = self.list() + location = GeoIP().get_all(addr) + if not location: + return None + while all_mirrors and len(mirrors) <= 2 and distance <= 270: for mirror in all_mirrors: - if mirror.distance_to(addr) <= distance: + if mirror.distance_to(location) <= distance: mirrors.append(mirror) all_mirrors.remove(mirror) @@ -241,15 +245,17 @@ class MirrorSet(object): mirrors = [] - while len(mirrors) <= 2 and distance <= 270: - for mirror in self._mirrors: - if mirror in mirrors: - continue + location = GeoIP().get_all(addr) + if location: + while len(mirrors) <= 2 and distance <= 270: + for mirror in self._mirrors: + if mirror in mirrors: + continue - if mirror.distance_to(addr) <= distance: - mirrors.append(mirror) + if mirror.distance_to(location) <= distance: + mirrors.append(mirror) - distance *= 1.2 + distance *= 1.2 return MirrorSet(mirrors) @@ -269,6 +275,9 @@ class Mirror(object): self.reload() + self.__location = None + self.__country_name = None + def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.url) @@ -314,7 +323,7 @@ class Mirror(object): @property def location(self): - if not hasattr(self, "__location"): + if self.__location is None: self.__location = GeoIP().get_all(self.address) return self.__location @@ -342,11 +351,14 @@ class Mirror(object): @property def country_code(self): - return GeoIP().get_country(self.address).lower() or "unknown" + return self.location.country_code.lower() or "unknown" @property def country_name(self): - return GeoIP().get_country_name(self.country_code) + if self.__country_name is None: + self.__country_name = GeoIP().get_country_name(self.country_code) + + return self.__country_name @property def city(self): @@ -484,12 +496,11 @@ class Mirror(object): def prefer_for_countries_names(self): return sorted([GeoIP().get_country_name(c) for c in self.prefer_for_countries]) - def distance_to(self, addr): - location = GeoIP().get_all(addr) + def distance_to(self, location, ignore_preference=False): if not location: return 0 - if location.country_code.lower() in self.prefer_for_countries: + if not ignore_preference and location.country_code.lower() in self.prefer_for_countries: return 0 distance_vector = ( diff --git a/www/webapp/handlers_mirrors.py b/www/webapp/handlers_mirrors.py index 8ae911df..d1a8a640 100644 --- a/www/webapp/handlers_mirrors.py +++ b/www/webapp/handlers_mirrors.py @@ -24,13 +24,18 @@ class MirrorIndexHandler(BaseHandler): class MirrorItemHandler(BaseHandler): def get(self, id): - _ = self.locale.translate - mirror = self.mirrors.get(id) if not mirror: raise tornado.web.HTTPError(404) - self.render("mirrors-item.html", item=mirror) + ip_addr = self.get_argument("addr", self.request.remote_ip) + client_location = self.geoip.get_all(ip_addr) + + client_distance = mirror.distance_to(client_location, ignore_preference=True) + client_distance *= 111.32 # to km + + self.render("mirrors-item.html", item=mirror, + client_distance=client_distance) class MirrorHandler(BaseHandler): -- 2.47.3