src/backend/geoip.py

   1 #!/usr/bin/python
   2
   3 import ipaddress
   4 import logging
   5 import pycares
   6 import re
   7 import socket
   8 import tornado.gen
   9 import tornado.platform.caresresolver
  10
  11 from . import countries
  12
  13 from .decorators import *
  14 from .misc import Object
  15
  16 BLACKLISTS = (
  17         "access.redhawk.org",
  18         "all.spamblock.unit.liu.se",
  19         "b.barracudacentral.org",
  20         "bl.deadbeef.com",
  21         #"bl.emailbasura.org",
  22         "bl.spamcannibal.org",
  23         "bl.spamcop.net",
  24         "blackholes.five-ten-sg.com",
  25         #"blackholes.mail-abuse.org",
  26         "blacklist.sci.kun.nl",
  27         "blacklist.woody.ch",
  28         "bogons.cymru.com",
  29         "bsb.spamlookup.net",
  30         "cbl.abuseat.org",
  31         #"cbl.anti-spam.org.cn",
  32         #"cblless.anti-spam.org.cn",
  33         #"cblplus.anti-spam.org.cn",
  34         #"cdl.anti-spam.org.cn",
  35         #"combined.njabl.org",
  36         "combined.rbl.msrbl.net",
  37         "csi.cloudmark.com",
  38         "db.wpbl.info",
  39         #"dialups.mail-abuse.org",
  40         "dnsbl-1.uceprotect.net",
  41         "dnsbl-2.uceprotect.net",
  42         "dnsbl-3.uceprotect.net",
  43         "dnsbl.abuse.ch",
  44         "dnsbl.cyberlogic.net",
  45         "dnsbl.dronebl.org",
  46         "dnsbl.inps.de",
  47         "dnsbl.kempt.net",
  48         #"dnsbl.njabl.org",
  49         "dnsbl.sorbs.net",
  50         "dob.sibl.support-intelligence.net",
  51         "drone.abuse.ch",
  52         "dsn.rfc-ignorant.org",
  53         "duinv.aupads.org",
  54         #"dul.blackhole.cantv.net",
  55         "dul.dnsbl.sorbs.net",
  56         "vdul.ru",
  57         "dyna.spamrats.com",
  58         "dynablock.sorbs.net",
  59         #"dyndns.rbl.jp",
  60         "dynip.rothen.com",
  61         "forbidden.icm.edu.pl",
  62         "http.dnsbl.sorbs.net",
  63         "httpbl.abuse.ch",
  64         "images.rbl.msrbl.net",
  65         "ips.backscatterer.org",
  66         "ix.dnsbl.manitu.net",
  67         "korea.services.net",
  68         "mail.people.it",
  69         "misc.dnsbl.sorbs.net",
  70         "multi.surbl.org",
  71         "netblock.pedantic.org",
  72         "noptr.spamrats.com",
  73         "opm.tornevall.org",
  74         "orvedb.aupads.org",
  75         "pbl.spamhaus.org",
  76         "phishing.rbl.msrbl.net",
  77         "psbl.surriel.com",
  78         "query.senderbase.org",
  79         #"rbl-plus.mail-abuse.org",
  80         "rbl.efnetrbl.org",
  81         "rbl.interserver.net",
  82         "rbl.spamlab.com",
  83         "rbl.suresupport.com",
  84         "relays.bl.gweep.ca",
  85         "relays.bl.kundenserver.de",
  86         #"relays.mail-abuse.org",
  87         "relays.nether.net",
  88         "residential.block.transip.nl",
  89         #"rot.blackhole.cantv.net",
  90         "sbl.spamhaus.org",
  91         #"short.rbl.jp",
  92         "smtp.dnsbl.sorbs.net",
  93         "socks.dnsbl.sorbs.net",
  94         "spam.abuse.ch",
  95         "spam.dnsbl.sorbs.net",
  96         "spam.rbl.msrbl.net",
  97         "spam.spamrats.com",
  98         "spamguard.leadmon.net",
  99         "spamlist.or.kr",
 100         "spamrbl.imp.ch",
 101         "tor.dan.me.uk",
 102         "ubl.lashback.com",
 103         "ubl.unsubscore.com",
 104         "uribl.swinog.ch",
 105         #"url.rbl.jp",
 106         "virbl.bit.nl",
 107         #"virus.rbl.jp",
 108         "virus.rbl.msrbl.net",
 109         "web.dnsbl.sorbs.net",
 110         "wormrbl.imp.ch",
 111         "xbl.spamhaus.org",
 112         "zen.spamhaus.org",
 113         "zombie.dnsbl.sorbs.net",
 114 )
 115
 116 class Resolver(tornado.platform.caresresolver.CaresResolver):
 117         def initialize(self, **kwargs):
 118                 super().initialize()
 119
 120                 # Overwrite Channel
 121                 self.channel = pycares.Channel(sock_state_cb=self._sock_state_cb, **kwargs)
 122
 123         @tornado.gen.coroutine
 124         def query(self, name, type=pycares.QUERY_TYPE_A):
 125                 # Create a new Future
 126                 fut = tornado.gen.Future()
 127
 128                 # Perform the query
 129                 self.channel.query(name, type, lambda result, error: fut.set_result((result, error)))
 130
 131                 # Wait for the response
 132                 result, error = yield fut
 133
 134                 # Handle any errors
 135                 if error:
 136                         # NXDOMAIN
 137                         if error == pycares.errno.ARES_ENOTFOUND:
 138                                 return
 139
 140                         # Ignore responses with no data
 141                         elif error == pycares.errno.ARES_ENODATA:
 142                                 return
 143
 144                         raise IOError(
 145                                 "C-Ares returned error %s: %s while resolving %s"
 146                                 % (error, pycares.errno.strerror(error), name)
 147                         )
 148
 149                 # Return the result
 150                 return result
 151
 152
 153 class GeoIP(Object):
 154         @lazy_property
 155         def resolver(self):
 156                 return Resolver(tries=2, timeout=2, domains=[])
 157
 158         def lookup(self, address):
 159                 return Address(self.backend, address)
 160
 161         def guess_address_family(self, addr):
 162                 if ":" in addr:
 163                         return 6
 164
 165                 return 4
 166
 167         def get_country(self, addr):
 168                 ret = self.get_all(addr)
 169
 170                 if ret:
 171                         return ret.country
 172
 173         def get_location(self, addr):
 174                 query = "SELECT * FROM geoip \
 175                         WHERE %s BETWEEN start_ip AND end_ip LIMIT 1"
 176
 177                 return self.db.get(query, addr)
 178
 179         def get_asn(self, addr):
 180                 query = "SELECT asn FROM geoip_asn \
 181                         WHERE %s BETWEEN start_ip AND end_ip LIMIT 1"
 182
 183                 ret = self.db.get(query, addr)
 184
 185                 if ret:
 186                         return ret.asn
 187
 188         def get_all(self, addr):
 189                 location = self.get_location(addr)
 190
 191                 if location:
 192                         location["asn"] = self.get_asn(addr)
 193
 194                 return location
 195
 196         _countries = {
 197                 "A1" : "Anonymous Proxy",
 198                 "A2" : "Satellite Provider",
 199                 "AP" : "Asia/Pacific Region",
 200                 "EU" : "Europe",
 201         }
 202
 203         def get_country_name(self, code):
 204                 return countries.get_name(code)
 205
 206
 207 class Address(Object):
 208         def init(self, address):
 209                 self.address = ipaddress.ip_address(address)
 210
 211         def __str__(self):
 212                 return "%s" % self.address
 213
 214         @property
 215         def family(self):
 216                 if isinstance(self.address, ipaddress.IPv6Address):
 217                         return socket.AF_INET6
 218                 elif isinstance(self.address, ipaddress.IPv4Address):
 219                         return socket.AF_INET
 220
 221         # Blacklist
 222
 223         def _make_blacklist_rr(self, blacklist):
 224                 if self.family == socket.AF_INET6:
 225                         octets = list(self.address.exploded.replace(":", ""))
 226                 elif self.family == socket.AF_INET:
 227                         octets = str(self.address).split(".")
 228                 else:
 229                         raise NotImplementedError("Unknown IP protocol")
 230
 231                 # Reverse the list
 232                 octets.reverse()
 233
 234                 # Append suffix
 235                 octets.append(blacklist)
 236
 237                 return ".".join(octets)
 238
 239         @tornado.gen.coroutine
 240         def _resolve_blacklist(self, blacklist):
 241                 # Get resource record name
 242                 rr = self._make_blacklist_rr(blacklist)
 243
 244                 # Get query type from IP protocol version
 245                 if self.family == socket.AF_INET6:
 246                         type = pycares.QUERY_TYPE_AAAA
 247                 elif self.family == socket.AF_INET:
 248                         type = pycares.QUERY_TYPE_A
 249                 else:
 250                         raise NotImplementedError("Unknown IP protocol")
 251
 252                 # Run query
 253                 try:
 254                         res = yield self.backend.geoip.resolver.query(rr, type=type)
 255                 except IOError as e:
 256                         logging.warning(e)
 257
 258                         return None, "%s" % e
 259
 260                 # Not found
 261                 if not res:
 262                         return False, None
 263
 264                 # If the IP address is on a blacklist, we will try to fetch the TXT record
 265                 reason = yield self.backend.geoip.resolver.query(rr, type=pycares.QUERY_TYPE_TXT)
 266
 267                 # Take the first reason
 268                 if reason:
 269                         for i in reason:
 270                                 return True, i.text.decode()
 271
 272                 # Blocked, but no reason
 273                 return True, None
 274
 275         @tornado.gen.coroutine
 276         def get_blacklists(self):
 277                 blacklists = yield { bl : self._resolve_blacklist(bl) for bl in BLACKLISTS }
 278
 279                 return blacklists