src/backend/geoip.py

   1 #!/usr/bin/python
   2
   3 import ipaddress
   4 import logging
   5 import pycares
   6 import re
   7 import socket
   8 import tornado.gen
   9 import tornado.platform.caresresolver
  10
  11 from . import countries
  12
  13 from .decorators import *
  14 from .misc import Object
  15
  16 BLACKLISTS = {
  17         "access.redhawk.org" : False,
  18         "all.de.bl.blocklist.de" : False,
  19         "all.spamblock.unit.liu.se" : False,
  20         "b.barracudacentral.org" : False,
  21         "bl.deadbeef.com" : False,
  22         #"bl.emailbasura.org" : False,
  23         "bl.spamcannibal.org" : False,
  24         "bl.spamcop.net" : False,
  25         "blackholes.five-ten-sg.com" : False,
  26         #"blackholes.mail-abuse.org" : False,
  27         "blacklist.sci.kun.nl" : False,
  28         "blacklist.woody.ch" : False,
  29         "bogons.cymru.com" : False,
  30         "bsb.spamlookup.net" : False,
  31         "cbl.abuseat.org" : False,
  32         #"cbl.anti-spam.org.cn" : False,
  33         #"cblless.anti-spam.org.cn" : False,
  34         #"cblplus.anti-spam.org.cn" : False,
  35         #"cdl.anti-spam.org.cn" : False,
  36         #"combined.njabl.org" : False,
  37         "combined.rbl.msrbl.net" : False,
  38         "csi.cloudmark.com" : False,
  39         "db.wpbl.info" : False,
  40         #"dialups.mail-abuse.org" : False,
  41         "dnsbl-1.uceprotect.net" : False,
  42         "dnsbl-2.uceprotect.net" : False,
  43         "dnsbl-3.uceprotect.net" : False,
  44         "dnsbl.abuse.ch" : False,
  45         "dnsbl.cyberlogic.net" : False,
  46         "dnsbl.dronebl.org" : False,
  47         "dnsbl.inps.de" : False,
  48         "dnsbl.kempt.net" : False,
  49         #"dnsbl.njabl.org" : False,
  50         "dnsbl.sorbs.net" : False,
  51         "dob.sibl.support-intelligence.net" : False,
  52         "drone.abuse.ch" : False,
  53         "dsn.rfc-ignorant.org" : False,
  54         "duinv.aupads.org" : False,
  55         #"dul.blackhole.cantv.net" : False,
  56         "dul.dnsbl.sorbs.net" : False,
  57         "vdul.ru" : False,
  58         "dyna.spamrats.com" : False,
  59         "dynablock.sorbs.net" : False,
  60         #"dyndns.rbl.jp" : False,
  61         "dynip.rothen.com" : False,
  62         "forbidden.icm.edu.pl" : False,
  63         "http.dnsbl.sorbs.net" : False,
  64         "httpbl.abuse.ch" : False,
  65         "images.rbl.msrbl.net" : False,
  66         "ips.backscatterer.org" : False,
  67         "ix.dnsbl.manitu.net" : False,
  68         "korea.services.net" : False,
  69         "mail.people.it" : False,
  70         "misc.dnsbl.sorbs.net" : False,
  71         "multi.surbl.org" : False,
  72         "netblock.pedantic.org" : False,
  73         "noptr.spamrats.com" : False,
  74         "opm.tornevall.org" : False,
  75         "orvedb.aupads.org" : False,
  76         "pbl.spamhaus.org" : False,
  77         "phishing.rbl.msrbl.net" : False,
  78         "psbl.surriel.com" : False,
  79         "query.senderbase.org" : False,
  80         #"rbl-plus.mail-abuse.org" : False,
  81         "rbl.efnetrbl.org" : False,
  82         "rbl.interserver.net" : False,
  83         "rbl.spamlab.com" : False,
  84         "rbl.suresupport.com" : False,
  85         "relays.bl.gweep.ca" : False,
  86         "relays.bl.kundenserver.de" : False,
  87         #"relays.mail-abuse.org" : False,
  88         "relays.nether.net" : False,
  89         "residential.block.transip.nl" : False,
  90         #"rot.blackhole.cantv.net" : False,
  91         "sbl.spamhaus.org" : True,
  92         #"short.rbl.jp" : False,
  93         "smtp.dnsbl.sorbs.net" : False,
  94         "socks.dnsbl.sorbs.net" : False,
  95         "spam.abuse.ch" : False,
  96         "spam.dnsbl.sorbs.net" : False,
  97         "spam.rbl.msrbl.net" : False,
  98         "spam.spamrats.com" : False,
  99         "spamguard.leadmon.net" : False,
 100         "spamlist.or.kr" : False,
 101         "spamrbl.imp.ch" : False,
 102         "tor.dan.me.uk" : False,
 103         "ubl.lashback.com" : False,
 104         "ubl.unsubscore.com" : False,
 105         "uribl.swinog.ch" : False,
 106         #"url.rbl.jp" : False,
 107         "virbl.bit.nl" : False,
 108         #"virus.rbl.jp" : False,
 109         "virus.rbl.msrbl.net" : False,
 110         "web.dnsbl.sorbs.net" : False,
 111         "wormrbl.imp.ch" : False,
 112         "xbl.spamhaus.org" : False,
 113         "zen.spamhaus.org" : False,
 114         "zombie.dnsbl.sorbs.net" : False,
 115 }
 116
 117 class Resolver(tornado.platform.caresresolver.CaresResolver):
 118         def initialize(self, **kwargs):
 119                 super().initialize()
 120
 121                 # Overwrite Channel
 122                 self.channel = pycares.Channel(sock_state_cb=self._sock_state_cb, **kwargs)
 123
 124         @tornado.gen.coroutine
 125         def query(self, name, type=pycares.QUERY_TYPE_A):
 126                 # Create a new Future
 127                 fut = tornado.gen.Future()
 128
 129                 # Perform the query
 130                 self.channel.query(name, type, lambda result, error: fut.set_result((result, error)))
 131
 132                 # Wait for the response
 133                 result, error = yield fut
 134
 135                 # Handle any errors
 136                 if error:
 137                         # NXDOMAIN
 138                         if error == pycares.errno.ARES_ENOTFOUND:
 139                                 return
 140
 141                         # Ignore responses with no data
 142                         elif error == pycares.errno.ARES_ENODATA:
 143                                 return
 144
 145                         raise IOError(
 146                                 "C-Ares returned error %s: %s while resolving %s"
 147                                 % (error, pycares.errno.strerror(error), name)
 148                         )
 149
 150                 # Return the result
 151                 return result
 152
 153
 154 class GeoIP(Object):
 155         @lazy_property
 156         def resolver(self):
 157                 return Resolver(tries=2, timeout=2, domains=[])
 158
 159         def lookup(self, address):
 160                 return Address(self.backend, address)
 161
 162         def guess_address_family(self, addr):
 163                 if ":" in addr:
 164                         return 6
 165
 166                 return 4
 167
 168         def get_country(self, addr):
 169                 ret = self.get_all(addr)
 170
 171                 if ret:
 172                         return ret.country
 173
 174         def get_location(self, addr):
 175                 query = "SELECT * FROM geoip \
 176                         WHERE %s BETWEEN start_ip AND end_ip LIMIT 1"
 177
 178                 return self.db.get(query, addr)
 179
 180         def get_asn(self, addr):
 181                 query = "SELECT asn FROM geoip_asn \
 182                         WHERE %s BETWEEN start_ip AND end_ip LIMIT 1"
 183
 184                 ret = self.db.get(query, addr)
 185
 186                 if ret:
 187                         return ret.asn
 188
 189         def get_all(self, addr):
 190                 location = self.get_location(addr)
 191
 192                 if location:
 193                         location["asn"] = self.get_asn(addr)
 194
 195                 return location
 196
 197         _countries = {
 198                 "A1" : "Anonymous Proxy",
 199                 "A2" : "Satellite Provider",
 200                 "AP" : "Asia/Pacific Region",
 201                 "EU" : "Europe",
 202         }
 203
 204         def get_country_name(self, code):
 205                 return countries.get_name(code)
 206
 207
 208 class Address(Object):
 209         def init(self, address):
 210                 self.address = ipaddress.ip_address(address)
 211
 212         def __str__(self):
 213                 return "%s" % self.address
 214
 215         @property
 216         def family(self):
 217                 if isinstance(self.address, ipaddress.IPv6Address):
 218                         return socket.AF_INET6
 219                 elif isinstance(self.address, ipaddress.IPv4Address):
 220                         return socket.AF_INET
 221
 222         # Blacklist
 223
 224         def _make_blacklist_rr(self, blacklist):
 225                 if self.family == socket.AF_INET6:
 226                         octets = list(self.address.exploded.replace(":", ""))
 227                 elif self.family == socket.AF_INET:
 228                         octets = str(self.address).split(".")
 229                 else:
 230                         raise NotImplementedError("Unknown IP protocol")
 231
 232                 # Reverse the list
 233                 octets.reverse()
 234
 235                 # Append suffix
 236                 octets.append(blacklist)
 237
 238                 return ".".join(octets)
 239
 240         @tornado.gen.coroutine
 241         def _resolve_blacklist(self, blacklist):
 242                 # Get resource record name
 243                 rr = self._make_blacklist_rr(blacklist)
 244
 245                 # Get query type from IP protocol version
 246                 if self.family == socket.AF_INET6:
 247                         type = pycares.QUERY_TYPE_AAAA
 248                 elif self.family == socket.AF_INET:
 249                         type = pycares.QUERY_TYPE_A
 250                 else:
 251                         raise NotImplementedError("Unknown IP protocol")
 252
 253                 # Run query
 254                 try:
 255                         res = yield self.backend.geoip.resolver.query(rr, type=type)
 256                 except IOError as e:
 257                         logging.warning(e)
 258
 259                         return None, "%s" % e
 260
 261                 # Not found
 262                 if not res:
 263                         logging.debug("%s is not blacklisted on %s" % (self, blacklist))
 264                         return False, None
 265
 266                 # If the IP address is on a blacklist, we will try to fetch the TXT record
 267                 reason = yield self.backend.geoip.resolver.query(rr, type=pycares.QUERY_TYPE_TXT)
 268
 269                 # Log result
 270                 logging.debug("%s is blacklisted on %s: %s" % (self, blacklist, reason or "N/A"))
 271
 272                 # Take the first reason
 273                 if reason:
 274                         for i in reason:
 275                                 return True, i.text.decode()
 276
 277                 # Blocked, but no reason
 278                 return True, None
 279
 280         @tornado.gen.coroutine
 281         def get_blacklists(self, important_only=False):
 282                 blacklists = yield { bl : self._resolve_blacklist(bl) for bl in BLACKLISTS if not important_only or BLACKLISTS[bl] }
 283
 284                 return blacklists
 285
 286         @tornado.gen.coroutine
 287         def is_blacklisted(self):
 288                 logging.debug("Checking if %s is blacklisted..." % self)
 289
 290                 # Perform checks
 291                 blacklists = yield self.get_blacklists(important_only=True)
 292
 293                 # If we are blacklisted on one list, this one is screwed
 294                 for code, reason in blacklists.values():
 295                         if code:
 296                                 return True