blocking: Replace spamhaus XBL against blocklist.de

[ipfire.org.git] / src / backend / geoip.py

#!/usr/bin/python

import ipaddress
import logging
import pycares
import re
import socket
import tornado.gen
import tornado.platform.caresresolver

from . import countries

from .decorators import *
from .misc import Object

BLACKLISTS = {
	"access.redhawk.org" : False,
	"all.de.bl.blocklist.de" : True,
	"all.spamblock.unit.liu.se" : False,
	"b.barracudacentral.org" : False,
	"bl.deadbeef.com" : False,
	#"bl.emailbasura.org" : False,
	"bl.spamcannibal.org" : False,
	"bl.spamcop.net" : False,
	"blackholes.five-ten-sg.com" : False,
	#"blackholes.mail-abuse.org" : False,
	"blacklist.sci.kun.nl" : False,
	"blacklist.woody.ch" : False,
	"bogons.cymru.com" : False,
	"bsb.spamlookup.net" : False,
	"cbl.abuseat.org" : False,
	#"cbl.anti-spam.org.cn" : False,
	#"cblless.anti-spam.org.cn" : False,
	#"cblplus.anti-spam.org.cn" : False,
	#"cdl.anti-spam.org.cn" : False,
	#"combined.njabl.org" : False,
	"combined.rbl.msrbl.net" : False,
	"csi.cloudmark.com" : False,
	"db.wpbl.info" : False,
	#"dialups.mail-abuse.org" : False,
	"dnsbl-1.uceprotect.net" : False,
	"dnsbl-2.uceprotect.net" : False,
	"dnsbl-3.uceprotect.net" : False,
	"dnsbl.abuse.ch" : False,
	"dnsbl.cyberlogic.net" : False,
	"dnsbl.dronebl.org" : False,
	"dnsbl.inps.de" : False,
	"dnsbl.kempt.net" : False,
	#"dnsbl.njabl.org" : False,
	"dnsbl.sorbs.net" : False,
	"dob.sibl.support-intelligence.net" : False,
	"drone.abuse.ch" : False,
	"dsn.rfc-ignorant.org" : False,
	"duinv.aupads.org" : False,
	#"dul.blackhole.cantv.net" : False,
	"dul.dnsbl.sorbs.net" : False,
	"vdul.ru" : False,
	"dyna.spamrats.com" : False,
	"dynablock.sorbs.net" : False,
	#"dyndns.rbl.jp" : False,
	"dynip.rothen.com" : False,
	"forbidden.icm.edu.pl" : False,
	"http.dnsbl.sorbs.net" : False,
	"httpbl.abuse.ch" : False,
	"images.rbl.msrbl.net" : False,
	"ips.backscatterer.org" : False,
	"ix.dnsbl.manitu.net" : False,
	"korea.services.net" : False,
	"mail.people.it" : False,
	"misc.dnsbl.sorbs.net" : False,
	"multi.surbl.org" : False,
	"netblock.pedantic.org" : False,
	"noptr.spamrats.com" : False,
	"opm.tornevall.org" : False,
	"orvedb.aupads.org" : False,
	"pbl.spamhaus.org" : False,
	"phishing.rbl.msrbl.net" : False,
	"psbl.surriel.com" : False,
	"query.senderbase.org" : False,
	#"rbl-plus.mail-abuse.org" : False,
	"rbl.efnetrbl.org" : False,
	"rbl.interserver.net" : False,
	"rbl.spamlab.com" : False,
	"rbl.suresupport.com" : False,
	"relays.bl.gweep.ca" : False,
	"relays.bl.kundenserver.de" : False,
	#"relays.mail-abuse.org" : False,
	"relays.nether.net" : False,
	"residential.block.transip.nl" : False,
	#"rot.blackhole.cantv.net" : False,
	"sbl.spamhaus.org" : True,
	#"short.rbl.jp" : False,
	"smtp.dnsbl.sorbs.net" : False,
	"socks.dnsbl.sorbs.net" : False,
	"spam.abuse.ch" : False,
	"spam.dnsbl.sorbs.net" : False,
	"spam.rbl.msrbl.net" : False,
	"spam.spamrats.com" : False,
	"spamguard.leadmon.net" : False,
	"spamlist.or.kr" : False,
	"spamrbl.imp.ch" : False,
	"tor.dan.me.uk" : False,
	"ubl.lashback.com" : False,
	"ubl.unsubscore.com" : False,
	"uribl.swinog.ch" : False,
	#"url.rbl.jp" : False,
	"virbl.bit.nl" : False,
	#"virus.rbl.jp" : False,
	"virus.rbl.msrbl.net" : False,
	"web.dnsbl.sorbs.net" : False,
	"wormrbl.imp.ch" : False,
	"xbl.spamhaus.org" : False,
	"zen.spamhaus.org" : False,
	"zombie.dnsbl.sorbs.net" : False,
}

class Resolver(tornado.platform.caresresolver.CaresResolver):
	def initialize(self, **kwargs):
		super().initialize()

		# Overwrite Channel
		self.channel = pycares.Channel(sock_state_cb=self._sock_state_cb, **kwargs)

	@tornado.gen.coroutine
	def query(self, name, type=pycares.QUERY_TYPE_A):
		# Create a new Future
		fut = tornado.gen.Future()

		# Perform the query
		self.channel.query(name, type, lambda result, error: fut.set_result((result, error)))

		# Wait for the response
		result, error = yield fut

		# Handle any errors
		if error:
			# NXDOMAIN
			if error == pycares.errno.ARES_ENOTFOUND:
				return

			# Ignore responses with no data
			elif error == pycares.errno.ARES_ENODATA:
				return

			raise IOError(
				"C-Ares returned error %s: %s while resolving %s"
				% (error, pycares.errno.strerror(error), name)
			)

		# Return the result
		return result


class GeoIP(Object):
	@lazy_property
	def resolver(self):
		return Resolver(tries=2, timeout=2, domains=[])

	def lookup(self, address):
		return Address(self.backend, address)

	def guess_address_family(self, addr):
		if ":" in addr:
			return 6

		return 4

	def get_country(self, addr):
		ret = self.get_all(addr)

		if ret:
			return ret.country

	def get_location(self, addr):
		query = "SELECT * FROM geoip \
			WHERE %s BETWEEN start_ip AND end_ip LIMIT 1"

		return self.db.get(query, addr)

	def get_asn(self, addr):
		query = "SELECT asn FROM geoip_asn \
			WHERE %s BETWEEN start_ip AND end_ip LIMIT 1"

		ret = self.db.get(query, addr)

		if ret:
			return ret.asn

	def get_all(self, addr):
		location = self.get_location(addr)

		if location:
			location["asn"] = self.get_asn(addr)

		return location

	_countries = {
		"A1" : "Anonymous Proxy",
		"A2" : "Satellite Provider",
		"AP" : "Asia/Pacific Region",
		"EU" : "Europe",
	}

	def get_country_name(self, code):
		return countries.get_name(code)


class Address(Object):
	def init(self, address):
		self.address = ipaddress.ip_address(address)

	def __str__(self):
		return "%s" % self.address

	@property
	def family(self):
		if isinstance(self.address, ipaddress.IPv6Address):
			return socket.AF_INET6
		elif isinstance(self.address, ipaddress.IPv4Address):
			return socket.AF_INET

	# Blacklist

	def _make_blacklist_rr(self, blacklist):
		if self.family == socket.AF_INET6:
			octets = list(self.address.exploded.replace(":", ""))
		elif self.family == socket.AF_INET:
			octets = str(self.address).split(".")
		else:
			raise NotImplementedError("Unknown IP protocol")

		# Reverse the list
		octets.reverse()

		# Append suffix
		octets.append(blacklist)

		return ".".join(octets)

	@tornado.gen.coroutine
	def _resolve_blacklist(self, blacklist):
		# Get resource record name
		rr = self._make_blacklist_rr(blacklist)

		# Get query type from IP protocol version
		if self.family == socket.AF_INET6:
			type = pycares.QUERY_TYPE_AAAA
		elif self.family == socket.AF_INET:
			type = pycares.QUERY_TYPE_A
		else:
			raise NotImplementedError("Unknown IP protocol")

		# Run query
		try:
			res = yield self.backend.geoip.resolver.query(rr, type=type)
		except IOError as e:
			logging.warning(e)

			return None, "%s" % e

		# Not found
		if not res:
			logging.debug("%s is not blacklisted on %s" % (self, blacklist))
			return False, None

		# If the IP address is on a blacklist, we will try to fetch the TXT record
		reason = yield self.backend.geoip.resolver.query(rr, type=pycares.QUERY_TYPE_TXT)

		# Log result
		logging.debug("%s is blacklisted on %s: %s" % (self, blacklist, reason or "N/A"))

		# Take the first reason
		if reason:
			for i in reason:
				return True, i.text.decode()

		# Blocked, but no reason
		return True, None

	@tornado.gen.coroutine
	def get_blacklists(self, important_only=False):
		blacklists = yield { bl : self._resolve_blacklist(bl) for bl in BLACKLISTS if not important_only or BLACKLISTS[bl] }

		return blacklists

	@tornado.gen.coroutine
	def is_blacklisted(self):
		logging.debug("Checking if %s is blacklisted..." % self)

		# Perform checks
		blacklists = yield self.get_blacklists(important_only=True)

		# If we are blacklisted on one list, this one is screwed
		for code, reason in blacklists.values():
			if code:
				return True