src/backend/geoip.py

   1 #!/usr/bin/python
   2
   3 import ipaddress
   4 import logging
   5 import pycares
   6 import re
   7 import socket
   8 import tornado.gen
   9 import tornado.platform.caresresolver
  10
  11 from . import countries
  12
  13 from .decorators import *
  14 from .misc import Object
  15
  16 BLACKLISTS = {
  17         "access.redhawk.org" : False,
  18         "all.spamblock.unit.liu.se" : False,
  19         "b.barracudacentral.org" : False,
  20         "bl.deadbeef.com" : False,
  21         #"bl.emailbasura.org" : False,
  22         "bl.spamcannibal.org" : False,
  23         "bl.spamcop.net" : False,
  24         "blackholes.five-ten-sg.com" : False,
  25         #"blackholes.mail-abuse.org" : False,
  26         "blacklist.sci.kun.nl" : False,
  27         "blacklist.woody.ch" : False,
  28         "bogons.cymru.com" : False,
  29         "bsb.spamlookup.net" : False,
  30         "cbl.abuseat.org" : False,
  31         #"cbl.anti-spam.org.cn" : False,
  32         #"cblless.anti-spam.org.cn" : False,
  33         #"cblplus.anti-spam.org.cn" : False,
  34         #"cdl.anti-spam.org.cn" : False,
  35         #"combined.njabl.org" : False,
  36         "combined.rbl.msrbl.net" : False,
  37         "csi.cloudmark.com" : False,
  38         "db.wpbl.info" : False,
  39         #"dialups.mail-abuse.org" : False,
  40         "dnsbl-1.uceprotect.net" : False,
  41         "dnsbl-2.uceprotect.net" : False,
  42         "dnsbl-3.uceprotect.net" : False,
  43         "dnsbl.abuse.ch" : False,
  44         "dnsbl.cyberlogic.net" : False,
  45         "dnsbl.dronebl.org" : False,
  46         "dnsbl.inps.de" : False,
  47         "dnsbl.kempt.net" : False,
  48         #"dnsbl.njabl.org" : False,
  49         "dnsbl.sorbs.net" : False,
  50         "dob.sibl.support-intelligence.net" : False,
  51         "drone.abuse.ch" : False,
  52         "dsn.rfc-ignorant.org" : False,
  53         "duinv.aupads.org" : False,
  54         #"dul.blackhole.cantv.net" : False,
  55         "dul.dnsbl.sorbs.net" : False,
  56         "vdul.ru" : False,
  57         "dyna.spamrats.com" : False,
  58         "dynablock.sorbs.net" : False,
  59         #"dyndns.rbl.jp" : False,
  60         "dynip.rothen.com" : False,
  61         "forbidden.icm.edu.pl" : False,
  62         "http.dnsbl.sorbs.net" : False,
  63         "httpbl.abuse.ch" : False,
  64         "images.rbl.msrbl.net" : False,
  65         "ips.backscatterer.org" : False,
  66         "ix.dnsbl.manitu.net" : False,
  67         "korea.services.net" : False,
  68         "mail.people.it" : False,
  69         "misc.dnsbl.sorbs.net" : False,
  70         "multi.surbl.org" : False,
  71         "netblock.pedantic.org" : False,
  72         "noptr.spamrats.com" : False,
  73         "opm.tornevall.org" : False,
  74         "orvedb.aupads.org" : False,
  75         "pbl.spamhaus.org" : False,
  76         "phishing.rbl.msrbl.net" : False,
  77         "psbl.surriel.com" : False,
  78         "query.senderbase.org" : False,
  79         #"rbl-plus.mail-abuse.org" : False,
  80         "rbl.efnetrbl.org" : False,
  81         "rbl.interserver.net" : False,
  82         "rbl.spamlab.com" : False,
  83         "rbl.suresupport.com" : False,
  84         "relays.bl.gweep.ca" : False,
  85         "relays.bl.kundenserver.de" : False,
  86         #"relays.mail-abuse.org" : False,
  87         "relays.nether.net" : False,
  88         "residential.block.transip.nl" : False,
  89         #"rot.blackhole.cantv.net" : False,
  90         "sbl.spamhaus.org" : True,
  91         #"short.rbl.jp" : False,
  92         "smtp.dnsbl.sorbs.net" : False,
  93         "socks.dnsbl.sorbs.net" : False,
  94         "spam.abuse.ch" : False,
  95         "spam.dnsbl.sorbs.net" : False,
  96         "spam.rbl.msrbl.net" : False,
  97         "spam.spamrats.com" : False,
  98         "spamguard.leadmon.net" : False,
  99         "spamlist.or.kr" : False,
 100         "spamrbl.imp.ch" : False,
 101         "tor.dan.me.uk" : False,
 102         "ubl.lashback.com" : False,
 103         "ubl.unsubscore.com" : False,
 104         "uribl.swinog.ch" : False,
 105         #"url.rbl.jp" : False,
 106         "virbl.bit.nl" : False,
 107         #"virus.rbl.jp" : False,
 108         "virus.rbl.msrbl.net" : False,
 109         "web.dnsbl.sorbs.net" : False,
 110         "wormrbl.imp.ch" : False,
 111         "xbl.spamhaus.org" : True,
 112         "zen.spamhaus.org" : False,
 113         "zombie.dnsbl.sorbs.net" : False,
 114 }
 115
 116 class Resolver(tornado.platform.caresresolver.CaresResolver):
 117         def initialize(self, **kwargs):
 118                 super().initialize()
 119
 120                 # Overwrite Channel
 121                 self.channel = pycares.Channel(sock_state_cb=self._sock_state_cb, **kwargs)
 122
 123         @tornado.gen.coroutine
 124         def query(self, name, type=pycares.QUERY_TYPE_A):
 125                 # Create a new Future
 126                 fut = tornado.gen.Future()
 127
 128                 # Perform the query
 129                 self.channel.query(name, type, lambda result, error: fut.set_result((result, error)))
 130
 131                 # Wait for the response
 132                 result, error = yield fut
 133
 134                 # Handle any errors
 135                 if error:
 136                         # NXDOMAIN
 137                         if error == pycares.errno.ARES_ENOTFOUND:
 138                                 return
 139
 140                         # Ignore responses with no data
 141                         elif error == pycares.errno.ARES_ENODATA:
 142                                 return
 143
 144                         raise IOError(
 145                                 "C-Ares returned error %s: %s while resolving %s"
 146                                 % (error, pycares.errno.strerror(error), name)
 147                         )
 148
 149                 # Return the result
 150                 return result
 151
 152
 153 class GeoIP(Object):
 154         @lazy_property
 155         def resolver(self):
 156                 return Resolver(tries=2, timeout=2, domains=[])
 157
 158         def lookup(self, address):
 159                 return Address(self.backend, address)
 160
 161         def guess_address_family(self, addr):
 162                 if ":" in addr:
 163                         return 6
 164
 165                 return 4
 166
 167         def get_country(self, addr):
 168                 ret = self.get_all(addr)
 169
 170                 if ret:
 171                         return ret.country
 172
 173         def get_location(self, addr):
 174                 query = "SELECT * FROM geoip \
 175                         WHERE %s BETWEEN start_ip AND end_ip LIMIT 1"
 176
 177                 return self.db.get(query, addr)
 178
 179         def get_asn(self, addr):
 180                 query = "SELECT asn FROM geoip_asn \
 181                         WHERE %s BETWEEN start_ip AND end_ip LIMIT 1"
 182
 183                 ret = self.db.get(query, addr)
 184
 185                 if ret:
 186                         return ret.asn
 187
 188         def get_all(self, addr):
 189                 location = self.get_location(addr)
 190
 191                 if location:
 192                         location["asn"] = self.get_asn(addr)
 193
 194                 return location
 195
 196         _countries = {
 197                 "A1" : "Anonymous Proxy",
 198                 "A2" : "Satellite Provider",
 199                 "AP" : "Asia/Pacific Region",
 200                 "EU" : "Europe",
 201         }
 202
 203         def get_country_name(self, code):
 204                 return countries.get_name(code)
 205
 206
 207 class Address(Object):
 208         def init(self, address):
 209                 self.address = ipaddress.ip_address(address)
 210
 211         def __str__(self):
 212                 return "%s" % self.address
 213
 214         @property
 215         def family(self):
 216                 if isinstance(self.address, ipaddress.IPv6Address):
 217                         return socket.AF_INET6
 218                 elif isinstance(self.address, ipaddress.IPv4Address):
 219                         return socket.AF_INET
 220
 221         # Blacklist
 222
 223         def _make_blacklist_rr(self, blacklist):
 224                 if self.family == socket.AF_INET6:
 225                         octets = list(self.address.exploded.replace(":", ""))
 226                 elif self.family == socket.AF_INET:
 227                         octets = str(self.address).split(".")
 228                 else:
 229                         raise NotImplementedError("Unknown IP protocol")
 230
 231                 # Reverse the list
 232                 octets.reverse()
 233
 234                 # Append suffix
 235                 octets.append(blacklist)
 236
 237                 return ".".join(octets)
 238
 239         @tornado.gen.coroutine
 240         def _resolve_blacklist(self, blacklist):
 241                 # Get resource record name
 242                 rr = self._make_blacklist_rr(blacklist)
 243
 244                 # Get query type from IP protocol version
 245                 if self.family == socket.AF_INET6:
 246                         type = pycares.QUERY_TYPE_AAAA
 247                 elif self.family == socket.AF_INET:
 248                         type = pycares.QUERY_TYPE_A
 249                 else:
 250                         raise NotImplementedError("Unknown IP protocol")
 251
 252                 # Run query
 253                 try:
 254                         res = yield self.backend.geoip.resolver.query(rr, type=type)
 255                 except IOError as e:
 256                         logging.warning(e)
 257
 258                         return None, "%s" % e
 259
 260                 # Not found
 261                 if not res:
 262                         logging.debug("%s is not blacklisted on %s" % (self, blacklist))
 263                         return False, None
 264
 265                 # If the IP address is on a blacklist, we will try to fetch the TXT record
 266                 reason = yield self.backend.geoip.resolver.query(rr, type=pycares.QUERY_TYPE_TXT)
 267
 268                 # Log result
 269                 logging.debug("%s is blacklisted on %s: %s" % (self, blacklist, reason or "N/A"))
 270
 271                 # Take the first reason
 272                 if reason:
 273                         for i in reason:
 274                                 return True, i.text.decode()
 275
 276                 # Blocked, but no reason
 277                 return True, None
 278
 279         @tornado.gen.coroutine
 280         def get_blacklists(self, important_only=False):
 281                 blacklists = yield { bl : self._resolve_blacklist(bl) for bl in BLACKLISTS if not important_only or BLACKLISTS[bl] }
 282
 283                 return blacklists
 284
 285         @tornado.gen.coroutine
 286         def is_blacklisted(self):
 287                 logging.debug("Checking if %s is blacklisted..." % self)
 288
 289                 # Perform checks
 290                 blacklists = yield self.get_blacklists(important_only=True)
 291
 292                 # If we are blacklisted on one list, this one is screwed
 293                 for code, reason in blacklists.values():
 294                         if code:
 295                                 return True