src/backend/geoip.py

   1 #!/usr/bin/python
   2
   3 import ipaddress
   4 import logging
   5 import pycares
   6 import re
   7 import socket
   8 import tornado.gen
   9 import tornado.platform.caresresolver
  10
  11 from . import countries
  12
  13 from .decorators import *
  14 from .misc import Object
  15
  16 # These lists are used to block access to the webapp
  17 BLOCKLISTS = (
  18         "sbl.spamhaus.org",
  19         "xbl.spamhaus.org",
  20 )
  21
  22 BLACKLISTS = {
  23         "access.redhawk.org" : False,
  24         "all.de.bl.blocklist.de" : False,
  25         "all.spamblock.unit.liu.se" : False,
  26         "b.barracudacentral.org" : False,
  27         "bl.deadbeef.com" : False,
  28         #"bl.emailbasura.org" : False,
  29         "bl.spamcannibal.org" : False,
  30         "bl.spamcop.net" : False,
  31         "blackholes.five-ten-sg.com" : False,
  32         #"blackholes.mail-abuse.org" : False,
  33         "blacklist.sci.kun.nl" : False,
  34         "blacklist.woody.ch" : False,
  35         "bogons.cymru.com" : False,
  36         "bsb.spamlookup.net" : False,
  37         "cbl.abuseat.org" : False,
  38         #"cbl.anti-spam.org.cn" : False,
  39         #"cblless.anti-spam.org.cn" : False,
  40         #"cblplus.anti-spam.org.cn" : False,
  41         #"cdl.anti-spam.org.cn" : False,
  42         #"combined.njabl.org" : False,
  43         "combined.rbl.msrbl.net" : False,
  44         "csi.cloudmark.com" : False,
  45         "db.wpbl.info" : False,
  46         #"dialups.mail-abuse.org" : False,
  47         "dnsbl-1.uceprotect.net" : False,
  48         "dnsbl-2.uceprotect.net" : False,
  49         "dnsbl-3.uceprotect.net" : False,
  50         "dnsbl.abuse.ch" : False,
  51         "dnsbl.cyberlogic.net" : False,
  52         "dnsbl.dronebl.org" : False,
  53         "dnsbl.inps.de" : False,
  54         "dnsbl.kempt.net" : False,
  55         #"dnsbl.njabl.org" : False,
  56         "dnsbl.sorbs.net" : False,
  57         "dob.sibl.support-intelligence.net" : False,
  58         "drone.abuse.ch" : False,
  59         "dsn.rfc-ignorant.org" : False,
  60         "duinv.aupads.org" : False,
  61         #"dul.blackhole.cantv.net" : False,
  62         "dul.dnsbl.sorbs.net" : False,
  63         "vdul.ru" : False,
  64         "dyna.spamrats.com" : False,
  65         "dynablock.sorbs.net" : False,
  66         #"dyndns.rbl.jp" : False,
  67         "dynip.rothen.com" : False,
  68         "forbidden.icm.edu.pl" : False,
  69         "http.dnsbl.sorbs.net" : False,
  70         "httpbl.abuse.ch" : False,
  71         "images.rbl.msrbl.net" : False,
  72         "ips.backscatterer.org" : False,
  73         "ix.dnsbl.manitu.net" : False,
  74         "korea.services.net" : False,
  75         "mail.people.it" : False,
  76         "misc.dnsbl.sorbs.net" : False,
  77         "multi.surbl.org" : False,
  78         "netblock.pedantic.org" : False,
  79         "noptr.spamrats.com" : False,
  80         "opm.tornevall.org" : False,
  81         "orvedb.aupads.org" : False,
  82         "pbl.spamhaus.org" : False,
  83         "phishing.rbl.msrbl.net" : False,
  84         "psbl.surriel.com" : False,
  85         "query.senderbase.org" : False,
  86         #"rbl-plus.mail-abuse.org" : False,
  87         "rbl.efnetrbl.org" : False,
  88         "rbl.interserver.net" : False,
  89         "rbl.spamlab.com" : False,
  90         "rbl.suresupport.com" : False,
  91         "relays.bl.gweep.ca" : False,
  92         "relays.bl.kundenserver.de" : False,
  93         #"relays.mail-abuse.org" : False,
  94         "relays.nether.net" : False,
  95         "residential.block.transip.nl" : False,
  96         #"rot.blackhole.cantv.net" : False,
  97         "sbl.spamhaus.org" : True,
  98         #"short.rbl.jp" : False,
  99         "smtp.dnsbl.sorbs.net" : False,
 100         "socks.dnsbl.sorbs.net" : False,
 101         "spam.abuse.ch" : False,
 102         "spam.dnsbl.sorbs.net" : False,
 103         "spam.rbl.msrbl.net" : False,
 104         "spam.spamrats.com" : False,
 105         "spamguard.leadmon.net" : False,
 106         "spamlist.or.kr" : False,
 107         "spamrbl.imp.ch" : False,
 108         "tor.dan.me.uk" : False,
 109         "ubl.lashback.com" : False,
 110         "ubl.unsubscore.com" : False,
 111         "uribl.swinog.ch" : False,
 112         #"url.rbl.jp" : False,
 113         "virbl.bit.nl" : False,
 114         #"virus.rbl.jp" : False,
 115         "virus.rbl.msrbl.net" : False,
 116         "web.dnsbl.sorbs.net" : False,
 117         "wormrbl.imp.ch" : False,
 118         "xbl.spamhaus.org" : True,
 119         "zen.spamhaus.org" : False,
 120         "zombie.dnsbl.sorbs.net" : False,
 121 }
 122
 123 class Resolver(tornado.platform.caresresolver.CaresResolver):
 124         def initialize(self, **kwargs):
 125                 super().initialize()
 126
 127                 # Overwrite Channel
 128                 self.channel = pycares.Channel(sock_state_cb=self._sock_state_cb, **kwargs)
 129
 130         @tornado.gen.coroutine
 131         def query(self, name, type=pycares.QUERY_TYPE_A):
 132                 # Create a new Future
 133                 fut = tornado.gen.Future()
 134
 135                 # Perform the query
 136                 self.channel.query(name, type, lambda result, error: fut.set_result((result, error)))
 137
 138                 # Wait for the response
 139                 result, error = yield fut
 140
 141                 # Handle any errors
 142                 if error:
 143                         # NXDOMAIN
 144                         if error == pycares.errno.ARES_ENOTFOUND:
 145                                 return
 146
 147                         # Ignore responses with no data
 148                         elif error == pycares.errno.ARES_ENODATA:
 149                                 return
 150
 151                         raise IOError(
 152                                 "C-Ares returned error %s: %s while resolving %s"
 153                                 % (error, pycares.errno.strerror(error), name)
 154                         )
 155
 156                 # Return the result
 157                 return result
 158
 159
 160 class GeoIP(Object):
 161         @lazy_property
 162         def resolver(self):
 163                 return Resolver(tries=2, timeout=2, domains=[])
 164
 165         def lookup(self, address):
 166                 return Address(self.backend, address)
 167
 168         def guess_address_family(self, addr):
 169                 if ":" in addr:
 170                         return 6
 171
 172                 return 4
 173
 174         def get_country(self, addr):
 175                 ret = self.get_all(addr)
 176
 177                 if ret:
 178                         return ret.country
 179
 180         def get_location(self, addr):
 181                 query = "SELECT * FROM geoip \
 182                         WHERE %s BETWEEN start_ip AND end_ip LIMIT 1"
 183
 184                 return self.db.get(query, addr)
 185
 186         def get_asn(self, addr):
 187                 query = "SELECT asn FROM geoip_asn \
 188                         WHERE %s BETWEEN start_ip AND end_ip LIMIT 1"
 189
 190                 ret = self.db.get(query, addr)
 191
 192                 if ret:
 193                         return ret.asn
 194
 195         def get_all(self, addr):
 196                 location = self.get_location(addr)
 197
 198                 if location:
 199                         location["asn"] = self.get_asn(addr)
 200
 201                 return location
 202
 203         _countries = {
 204                 "A1" : "Anonymous Proxy",
 205                 "A2" : "Satellite Provider",
 206                 "AP" : "Asia/Pacific Region",
 207                 "EU" : "Europe",
 208         }
 209
 210         def get_country_name(self, code):
 211                 return countries.get_name(code)
 212
 213         @tornado.gen.coroutine
 214         def test_blacklist(self, address):
 215                 address = self.lookup(address)
 216
 217                 # Determne blacklist status
 218                 status = yield address.is_blacklisted()
 219
 220                 print("Blacklist status for %s: %s" % (address, status))
 221
 222
 223 class Address(Object):
 224         def init(self, address):
 225                 self.address = ipaddress.ip_address(address)
 226
 227         def __str__(self):
 228                 return "%s" % self.address
 229
 230         @property
 231         def family(self):
 232                 if isinstance(self.address, ipaddress.IPv6Address):
 233                         return socket.AF_INET6
 234                 elif isinstance(self.address, ipaddress.IPv4Address):
 235                         return socket.AF_INET
 236
 237         # Blacklist
 238
 239         def _make_blacklist_rr(self, blacklist):
 240                 if self.family == socket.AF_INET6:
 241                         octets = list(self.address.exploded.replace(":", ""))
 242                 elif self.family == socket.AF_INET:
 243                         octets = str(self.address).split(".")
 244                 else:
 245                         raise NotImplementedError("Unknown IP protocol")
 246
 247                 # Reverse the list
 248                 octets.reverse()
 249
 250                 # Append suffix
 251                 octets.append(blacklist)
 252
 253                 return ".".join(octets)
 254
 255         @tornado.gen.coroutine
 256         def _resolve_blacklist(self, blacklist):
 257                 return_code = None
 258
 259                 # Get resource record name
 260                 rr = self._make_blacklist_rr(blacklist)
 261
 262                 # Get query type from IP protocol version
 263                 if self.family == socket.AF_INET6:
 264                         type = pycares.QUERY_TYPE_AAAA
 265                 elif self.family == socket.AF_INET:
 266                         type = pycares.QUERY_TYPE_A
 267                 else:
 268                         raise NotImplementedError("Unknown IP protocol")
 269
 270                 # Run query
 271                 try:
 272                         res = yield self.backend.geoip.resolver.query(rr, type=type)
 273                 except IOError as e:
 274                         logging.warning(e)
 275
 276                         return return_code, "%s" % e
 277
 278                 # Not found
 279                 if not res:
 280                         logging.debug("%s is not blacklisted on %s" % (self, blacklist))
 281                         return return_code, None
 282
 283                 # Extract return code from DNS response
 284                 for row in res:
 285                         return_code = row.host
 286                         break
 287
 288                 # If the IP address is on a blacklist, we will try to fetch the TXT record
 289                 reason = yield self.backend.geoip.resolver.query(rr, type=pycares.QUERY_TYPE_TXT)
 290
 291                 # Log result
 292                 logging.debug("%s is blacklisted on %s: %s" % (self, blacklist, reason or "N/A"))
 293
 294                 # Take the first reason
 295                 if reason:
 296                         for i in reason:
 297                                 return return_code, i.text
 298
 299                 # Blocked, but no reason
 300                 return return_code, None
 301
 302         @tornado.gen.coroutine
 303         def get_blacklists(self, important_only=False):
 304                 blacklists = yield { bl : self._resolve_blacklist(bl) for bl in BLACKLISTS if not important_only or BLACKLISTS[bl] }
 305
 306                 return blacklists
 307
 308         @tornado.gen.coroutine
 309         def is_blacklisted(self):
 310                 logging.debug("Checking if %s is blacklisted..." % self)
 311
 312                 # Perform checks
 313                 blacklists = yield { bl : self._resolve_blacklist(bl) for bl in BLOCKLISTS }
 314
 315                 # If we are blacklisted on one list, this one is screwed
 316                 for bl in blacklists:
 317                         code, message = blacklists[bl]
 318
 319                         logging.debug("Response from %s is: %s (%s)" % (bl, code, message))
 320
 321                         # Exclude matches on SBLCSS
 322                         if bl == "sbl.spamhaus.org" and code == "127.0.0.3":
 323                                 continue
 324
 325                         # Consider the host blocked for any non-zero return code
 326                         if code:
 327                                 return True