fireinfo/fireinfod

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import ipaddr
   5 import logging
   6 import pymongo
   7 import re
   8 import simplejson
   9 import tornado.database
  10 import tornado.httpserver
  11 import tornado.ioloop
  12 import tornado.options
  13 import tornado.web
  14
  15 import backend
  16
  17 DATABASE_HOST = ["irma.ipfire.org", "madeye.ipfire.org"]
  18 DATABASE_NAME = "stasy"
  19
  20 DEFAULT_HOST = "www.ipfire.org"
  21
  22 MIN_PROFILE_VERSION = 0
  23 MAX_PROFILE_VERSION = 0
  24
  25 class Profile(dict):
  26         def __getattr__(self, key):
  27                 try:
  28                         return self[key]
  29                 except KeyError:
  30                         raise AttributeError, key
  31
  32         def __setattr__(self, key, val):
  33                 self[key] = val
  34
  35
  36 class Fireinfod(tornado.web.Application):
  37         def __init__(self, **kwargs):
  38                 settings = dict(
  39                         debug = False,
  40                         default_host = DEFAULT_HOST,
  41                         gzip = True,
  42                 )
  43                 settings.update(kwargs)
  44
  45                 tornado.web.Application.__init__(self, **settings)
  46
  47                 # Establish database connection
  48                 self.connection = pymongo.Connection(DATABASE_HOST)
  49                 self.db = self.connection[DATABASE_NAME]
  50                 logging.info("Successfully connected to database: %s:%s" % \
  51                         (self.connection.host, self.connection.port))
  52
  53                 self.add_handlers(r"fireinfo.ipfire.org", [
  54                         (r"/", tornado.web.RedirectHandler, { "url" : "http://www.ipfire.org/" }),
  55                         (r"/send/([a-z0-9]+)", ProfileSendHandler),
  56                         (r"/debug", DebugHandler),
  57                 ])
  58
  59                 # ipfire.org
  60                 #  this should not be neccessary (see default_host) but some versions
  61                 #  of tornado have a bug.
  62                 self.add_handlers(r".*", [
  63                         (r".*", tornado.web.RedirectHandler, { "url" : "http://" + DEFAULT_HOST + "/" })
  64                 ])
  65
  66         def __del__(self):
  67                 logging.debug("Disconnecting from database")
  68                 self.connection.disconnect()
  69
  70         @property
  71         def ioloop(self):
  72                 return tornado.ioloop.IOLoop.instance()
  73
  74         def start(self, port=9001):
  75                 logging.info("Starting application")
  76
  77                 http_server = tornado.httpserver.HTTPServer(self, xheaders=True)
  78                 http_server.listen(port)
  79
  80                 # Register automatic cleanup for old profiles, etc.
  81                 automatic_cleanup = tornado.ioloop.PeriodicCallback(
  82                         self.automatic_cleanup, 60*60*1000)
  83                 automatic_cleanup.start()
  84
  85                 self.ioloop.start()
  86
  87         def stop(self):
  88                 logging.info("Stopping application")
  89                 self.ioloop.stop()
  90
  91         def db_get_collection(self, name):
  92                 return pymongo.collection.Collection(self.db, name)
  93
  94         @property
  95         def profiles(self):
  96                 return self.db_get_collection("profiles")
  97
  98         @property
  99         def archives(self):
 100                 return self.db_get_collection("archives")
 101
 102         def automatic_cleanup(self):
 103                 logging.info("Starting automatic cleanup...")
 104
 105                 # Remove all profiles that were not updated since 4 weeks.
 106                 not_updated_since = datetime.datetime.utcnow() - \
 107                         datetime.timedelta(weeks=4)
 108
 109                 self.move_profiles({ "updated" : { "$lt" : not_updated_since }})
 110
 111         def move_profiles(self, find):
 112                 """
 113                         Move all profiles by the "find" criteria.
 114                 """
 115                 for p in self.profiles.find(find):
 116                         self.archives.save(p)
 117                 self.profiles.remove(find)
 118
 119
 120 class BaseHandler(tornado.web.RequestHandler):
 121         @property
 122         def geoip(self):
 123                 return backend.GeoIP()
 124
 125         @property
 126         def db(self):
 127                 return self.application.db
 128
 129         def db_get_collection(self, name):
 130                 return self.application.db_get_collection(name)
 131
 132         @property
 133         def db_collections(self):
 134                 return [self.db_get_collection(c) for c in self.db.collection_names()]
 135
 136
 137 DEBUG_STR = """
 138 Database information:
 139         Host: %(db_host)s:%(db_port)s
 140
 141         All nodes: %(db_nodes)s
 142
 143         %(collections)s
 144
 145 """
 146
 147 DEBUG_COLLECTION_STR = """
 148         Collection: %(name)s
 149                 Total documents: %(count)d
 150 """
 151
 152 class DebugHandler(BaseHandler):
 153         def get(self):
 154                 # This handler is only available in debugging mode.
 155                 if not self.application.settings["debug"]:
 156                         return tornado.web.HTTPError(404)
 157
 158                 self.set_header("Content-type", "text/plain")
 159
 160                 conn, db = (self.application.connection, self.db)
 161
 162                 debug_info = dict(
 163                         db_host = conn.host,
 164                         db_port = conn.port,
 165                         db_nodes = list(conn.nodes),
 166                 )
 167
 168                 collections = []
 169                 for collection in self.db_collections:
 170                         collections.append(DEBUG_COLLECTION_STR % {
 171                                 "name" : collection.name, "count" : collection.count(),
 172                         })
 173                 debug_info["collections"] = "".join(collections)
 174
 175                 self.write(DEBUG_STR % debug_info)
 176                 self.finish()
 177
 178
 179 class ProfileSendHandler(BaseHandler):
 180         @property
 181         def archives(self):
 182                 return self.application.archives
 183
 184         @property
 185         def profiles(self):
 186                 return self.application.profiles
 187
 188         def prepare(self):
 189                 # Create an empty profile.
 190                 self.profile = Profile()
 191
 192         def __check_attributes(self, profile):
 193                 """
 194                         Check for attributes that must be provided,
 195                 """
 196
 197                 attributes = (
 198                         "private_id",
 199                         "profile_version",
 200                         "public_id",
 201                         "updated",
 202                 )
 203                 for attr in attributes:
 204                         if not profile.has_key(attr):
 205                                 raise tornado.web.HTTPError(400, "Profile lacks '%s' attribute: %s" % (attr, profile))
 206
 207         def __check_valid_ids(self, profile):
 208                 """
 209                         Check if IDs contain valid data.
 210                 """
 211
 212                 for id in ("public_id", "private_id"):
 213                         if re.match(r"^([a-f0-9]{40})$", "%s" % profile[id]) is None:
 214                                 raise tornado.web.HTTPError(400, "ID '%s' has wrong format: %s" % (id, profile))
 215
 216         def __check_equal_ids(self, profile):
 217                 """
 218                         Check if public_id and private_id are equal.
 219                 """
 220
 221                 if profile.public_id == profile.private_id:
 222                         raise tornado.web.HTTPError(400, "Public and private IDs are equal: %s" % profile)
 223
 224         def __check_matching_ids(self, profile):
 225                 """
 226                         Check if a profile with the given public_id is already in the
 227                         database. If so we need to check if the private_id matches.
 228                 """
 229                 p = self.profiles.find_one({ "public_id" : profile["public_id"]})
 230                 if not p:
 231                         return
 232
 233                 p = Profile(p)
 234                 if p.private_id != profile.private_id:
 235                         raise tornado.web.HTTPError(400, "Mismatch of private_id: %s" % profile)
 236
 237         def __check_profile_version(self, profile):
 238                 """
 239                         Check if this version of the server software does support the
 240                         received profile.
 241                 """
 242                 version = profile.profile_version
 243
 244                 if version < MIN_PROFILE_VERSION or version > MAX_PROFILE_VERSION:
 245                         raise tornado.web.HTTPError(400,
 246                                 "Profile version is not supported: %s" % version)
 247
 248         def check_profile(self):
 249                 """
 250                         This method checks if the blob is sane.
 251                 """
 252
 253                 checks = (
 254                         self.__check_attributes,
 255                         self.__check_valid_ids,
 256                         self.__check_equal_ids,
 257                         self.__check_profile_version,
 258                         # These checks require at least one database query and should be done
 259                         # at last.
 260                         self.__check_matching_ids,
 261                 )
 262
 263                 for check in checks:
 264                         check(self.profile)
 265
 266                 # If we got here, everything is okay and we can go on...
 267
 268         def move_profiles(self, find):
 269                 self.application.move_profiles(find)
 270
 271         # The GET method is only allowed in debugging mode.
 272         def get(self, public_id):
 273                 if not self.application.settings["debug"]:
 274                         return tornado.web.HTTPError(405)
 275
 276                 return self.post(public_id)
 277
 278         def post(self, public_id):
 279                 profile = self.get_argument("profile", None)
 280
 281                 # Send "400 bad request" if no profile was provided
 282                 if not profile:
 283                         raise tornado.web.HTTPError(400, "No profile received.")
 284
 285                 # Try to decode the profile.
 286                 try:
 287                         self.profile.update(simplejson.loads(profile))
 288                 except simplejson.decoder.JSONDecodeError, e:
 289                         raise tornado.web.HTTPError(400, "Profile could not be decoded: %s" % e)
 290
 291                 # Create a shortcut and overwrite public_id from query string
 292                 profile = self.profile
 293                 profile.public_id = public_id
 294
 295                 # Add timestamp to the profile
 296                 profile.updated = datetime.datetime.utcnow()
 297
 298                 # Check if profile contains proper data.
 299                 self.check_profile()
 300
 301                 # Get GeoIP information if address is not defined in rfc1918
 302                 addr = ipaddr.IPAddress(self.request.remote_ip)
 303                 if not addr.is_private:
 304                         profile.geoip = self.geoip.get_all(self.request.remote_ip)
 305
 306                 # Move previous profiles to archive and keep only the latest one
 307                 # in profiles. This will make full table lookups faster.
 308                 self.move_profiles({ "public_id" : profile.public_id })
 309
 310                 # Write profile to database
 311                 id = self.profiles.save(profile)
 312
 313                 self.write("Your profile was successfully saved to the database.")
 314                 self.finish()
 315
 316                 logging.debug("Saved profile: %s" % profile)
 317
 318
 319 if __name__ == "__main__":
 320         app = Fireinfod()
 321
 322         app.start()