b5e57589b595c39b921cc32f6808b4141ed86b3b
[location/libloc.git] / src / python / location.in
1 #!/usr/bin/python3
2 ###############################################################################
3 # #
4 # libloc - A library to determine the location of someone on the Internet #
5 # #
6 # Copyright (C) 2017 IPFire Development Team <info@ipfire.org> #
7 # #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
12 # #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
17 # #
18 ###############################################################################
19
20 import argparse
21 import datetime
22 import ipaddress
23 import logging
24 import os
25 import re
26 import shutil
27 import socket
28 import sys
29 import time
30
31 # Load our location module
32 import location
33 import location.downloader
34 import location.export
35
36 from location.i18n import _
37
38 # Setup logging
39 log = logging.getLogger("location")
40
41 # Output formatters
42
43 class CLI(object):
44 def parse_cli(self):
45 parser = argparse.ArgumentParser(
46 description=_("Location Database Command Line Interface"),
47 )
48 subparsers = parser.add_subparsers()
49
50 # Global configuration flags
51 parser.add_argument("--debug", action="store_true",
52 help=_("Enable debug output"))
53 parser.add_argument("--quiet", action="store_true",
54 help=_("Enable quiet mode"))
55
56 # version
57 parser.add_argument("--version", action="version",
58 version="%(prog)s @VERSION@")
59
60 # database
61 parser.add_argument("--database", "-d",
62 default="@databasedir@/database.db", help=_("Path to database"),
63 )
64
65 # public key
66 parser.add_argument("--public-key", "-k",
67 default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
68 )
69
70 # Show the database version
71 version = subparsers.add_parser("version",
72 help=_("Show database version"))
73 version.set_defaults(func=self.handle_version)
74
75 # lookup an IP address
76 lookup = subparsers.add_parser("lookup",
77 help=_("Lookup one or multiple IP addresses"),
78 )
79 lookup.add_argument("address", nargs="+")
80 lookup.set_defaults(func=self.handle_lookup)
81
82 # Dump the whole database
83 dump = subparsers.add_parser("dump",
84 help=_("Dump the entire database"),
85 )
86 dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
87 dump.set_defaults(func=self.handle_dump)
88
89 # Update
90 update = subparsers.add_parser("update", help=_("Update database"))
91 update.add_argument("--cron",
92 help=_("Update the library only once per interval"),
93 choices=("daily", "weekly", "monthly"),
94 )
95 update.set_defaults(func=self.handle_update)
96
97 # Verify
98 verify = subparsers.add_parser("verify",
99 help=_("Verify the downloaded database"))
100 verify.set_defaults(func=self.handle_verify)
101
102 # Get AS
103 get_as = subparsers.add_parser("get-as",
104 help=_("Get information about one or multiple Autonomous Systems"),
105 )
106 get_as.add_argument("asn", nargs="+")
107 get_as.set_defaults(func=self.handle_get_as)
108
109 # Search for AS
110 search_as = subparsers.add_parser("search-as",
111 help=_("Search for Autonomous Systems that match the string"),
112 )
113 search_as.add_argument("query", nargs=1)
114 search_as.set_defaults(func=self.handle_search_as)
115
116 # List all networks in an AS
117 list_networks_by_as = subparsers.add_parser("list-networks-by-as",
118 help=_("Lists all networks in an AS"),
119 )
120 list_networks_by_as.add_argument("asn", nargs=1, type=int)
121 list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
122 list_networks_by_as.add_argument("--format",
123 choices=location.export.formats.keys(), default="list")
124 list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
125
126 # List all networks in a country
127 list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
128 help=_("Lists all networks in a country"),
129 )
130 list_networks_by_cc.add_argument("country_code", nargs=1)
131 list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
132 list_networks_by_cc.add_argument("--format",
133 choices=location.export.formats.keys(), default="list")
134 list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
135
136 # List all networks with flags
137 list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
138 help=_("Lists all networks with flags"),
139 )
140 list_networks_by_flags.add_argument("--anonymous-proxy",
141 action="store_true", help=_("Anonymous Proxies"),
142 )
143 list_networks_by_flags.add_argument("--satellite-provider",
144 action="store_true", help=_("Satellite Providers"),
145 )
146 list_networks_by_flags.add_argument("--anycast",
147 action="store_true", help=_("Anycasts"),
148 )
149 list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
150 list_networks_by_flags.add_argument("--format",
151 choices=location.export.formats.keys(), default="list")
152 list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
153
154 # List countries
155 list_countries = subparsers.add_parser("list-countries",
156 help=_("Lists all countries"),
157 )
158 list_countries.add_argument("--show-name",
159 action="store_true", help=_("Show the name of the country"),
160 )
161 list_countries.add_argument("--show-continent",
162 action="store_true", help=_("Show the continent"),
163 )
164 list_countries.set_defaults(func=self.handle_list_countries)
165
166 # Export
167 export = subparsers.add_parser("export",
168 help=_("Exports data in many formats to load it into packet filters"),
169 )
170 export.add_argument("--format", help=_("Output format"),
171 choices=location.export.formats.keys(), default="list")
172 export.add_argument("--directory", help=_("Output directory"), required=True)
173 export.add_argument("--family",
174 help=_("Specify address family"), choices=("ipv6", "ipv4"),
175 )
176 export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
177 export.set_defaults(func=self.handle_export)
178
179 args = parser.parse_args()
180
181 # Configure logging
182 if args.debug:
183 location.logger.set_level(logging.DEBUG)
184 elif args.quiet:
185 location.logger.set_level(logging.WARNING)
186
187 # Print usage if no action was given
188 if not "func" in args:
189 parser.print_usage()
190 sys.exit(2)
191
192 return args
193
194 def run(self):
195 # Parse command line arguments
196 args = self.parse_cli()
197
198 # Open database
199 try:
200 db = location.Database(args.database)
201 except FileNotFoundError as e:
202 # Allow continuing without a database
203 if args.func == self.handle_update:
204 db = None
205
206 else:
207 sys.stderr.write("location: Could not open database %s: %s\n" \
208 % (args.database, e))
209 sys.exit(1)
210
211 # Translate family (if present)
212 if "family" in args:
213 if args.family == "ipv6":
214 args.family = socket.AF_INET6
215 elif args.family == "ipv4":
216 args.family = socket.AF_INET
217 else:
218 args.family = 0
219
220 # Call function
221 try:
222 ret = args.func(db, args)
223
224 # Catch invalid inputs
225 except ValueError as e:
226 sys.stderr.write("%s\n" % e)
227 ret = 2
228
229 # Return with exit code
230 if ret:
231 sys.exit(ret)
232
233 # Otherwise just exit
234 sys.exit(0)
235
236 def handle_version(self, db, ns):
237 """
238 Print the version of the database
239 """
240 t = time.strftime(
241 "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
242 )
243
244 print(t)
245
246 def handle_lookup(self, db, ns):
247 ret = 0
248
249 format = " %-24s: %s"
250
251 for address in ns.address:
252 try:
253 network = db.lookup(address)
254 except ValueError:
255 print(_("Invalid IP address: %s") % address, file=sys.stderr)
256
257 args = {
258 "address" : address,
259 "network" : network,
260 }
261
262 # Nothing found?
263 if not network:
264 print(_("Nothing found for %(address)s") % args, file=sys.stderr)
265 ret = 1
266 continue
267
268 print("%s:" % address)
269 print(format % (_("Network"), network))
270
271 # Print country
272 if network.country_code:
273 country = db.get_country(network.country_code)
274
275 print(format % (
276 _("Country"),
277 country.name if country else network.country_code),
278 )
279
280 # Print AS information
281 if network.asn:
282 autonomous_system = db.get_as(network.asn)
283
284 print(format % (
285 _("Autonomous System"),
286 autonomous_system or "AS%s" % network.asn),
287 )
288
289 # Anonymous Proxy
290 if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
291 print(format % (
292 _("Anonymous Proxy"), _("yes"),
293 ))
294
295 # Satellite Provider
296 if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
297 print(format % (
298 _("Satellite Provider"), _("yes"),
299 ))
300
301 # Anycast
302 if network.has_flag(location.NETWORK_FLAG_ANYCAST):
303 print(format % (
304 _("Anycast"), _("yes"),
305 ))
306
307 return ret
308
309 def handle_dump(self, db, ns):
310 # Use output file or write to stdout
311 f = ns.output or sys.stdout
312
313 # Format everything like this
314 format = "%-24s %s\n"
315
316 # Write metadata
317 f.write("#\n# Location Database Export\n#\n")
318
319 f.write("# Generated: %s\n" % time.strftime(
320 "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
321 ))
322
323 if db.vendor:
324 f.write("# Vendor: %s\n" % db.vendor)
325
326 if db.license:
327 f.write("# License: %s\n" % db.license)
328
329 f.write("#\n")
330
331 if db.description:
332 for line in db.description.splitlines():
333 line = "# %s" % line
334 f.write("%s\n" % line.rstrip())
335
336 f.write("#\n")
337
338 # Iterate over all ASes
339 for a in db.ases:
340 f.write("\n")
341 f.write(format % ("aut-num:", "AS%s" % a.number))
342 f.write(format % ("name:", a.name))
343
344 flags = {
345 location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:",
346 location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
347 location.NETWORK_FLAG_ANYCAST : "is-anycast:",
348 }
349
350 # Iterate over all networks
351 for n in db.networks:
352 f.write("\n")
353 f.write(format % ("net:", n))
354
355 if n.country_code:
356 f.write(format % ("country:", n.country_code))
357
358 if n.asn:
359 f.write(format % ("aut-num:", n.asn))
360
361 # Print all flags
362 for flag in flags:
363 if n.has_flag(flag):
364 f.write(format % (flags[flag], "yes"))
365
366 def handle_get_as(self, db, ns):
367 """
368 Gets information about Autonomous Systems
369 """
370 ret = 0
371
372 for asn in ns.asn:
373 try:
374 asn = int(asn)
375 except ValueError:
376 print(_("Invalid ASN: %s") % asn, file=sys.stderr)
377 ret = 1
378 continue
379
380 # Fetch AS from database
381 a = db.get_as(asn)
382
383 # Nothing found
384 if not a:
385 print(_("Could not find AS%s") % asn, file=sys.stderr)
386 ret = 1
387 continue
388
389 print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
390
391 return ret
392
393 def handle_search_as(self, db, ns):
394 for query in ns.query:
395 # Print all matches ASes
396 for a in db.search_as(query):
397 print(a)
398
399 def handle_update(self, db, ns):
400 if ns.cron and db:
401 now = datetime.datetime.utcnow()
402
403 # Parse the database timestamp
404 t = datetime.datetime.utcfromtimestamp(db.created_at)
405
406 if ns.cron == "daily":
407 delta = datetime.timedelta(days=1)
408 elif ns.cron == "weekly":
409 delta = datetime.timedelta(days=7)
410 elif ns.cron == "monthly":
411 delta = datetime.timedelta(days=30)
412
413 # Check if the database has recently been updated
414 if t >= (now - delta):
415 log.info(
416 _("The database has been updated recently (%s)") % \
417 format_timedelta(now - t),
418 )
419 return 3
420
421 # Fetch the timestamp we need from DNS
422 t = location.discover_latest_version()
423
424 # Parse timestamp into datetime format
425 timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
426
427 # Check the version of the local database
428 if db and timestamp and db.created_at >= timestamp.timestamp():
429 log.info("Already on the latest version")
430 return
431
432 # Download the database into the correct directory
433 tmpdir = os.path.dirname(ns.database)
434
435 # Create a downloader
436 d = location.downloader.Downloader()
437
438 # Try downloading a new database
439 try:
440 t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
441
442 # If no file could be downloaded, log a message
443 except FileNotFoundError as e:
444 log.error("Could not download a new database")
445 return 1
446
447 # If we have not received a new file, there is nothing to do
448 if not t:
449 return 3
450
451 # Move temporary file to destination
452 shutil.move(t.name, ns.database)
453
454 return 0
455
456 def handle_verify(self, db, ns):
457 # Verify the database
458 with open(ns.public_key, "r") as f:
459 if not db.verify(f):
460 log.error("Could not verify database")
461 return 1
462
463 # Success
464 log.debug("Database successfully verified")
465 return 0
466
467 def __get_output_formatter(self, ns):
468 try:
469 cls = location.export.formats[ns.format]
470 except KeyError:
471 cls = location.export.OutputFormatter
472
473 return cls
474
475 def handle_list_countries(self, db, ns):
476 for country in db.countries:
477 line = [
478 country.code,
479 ]
480
481 if ns.show_continent:
482 line.append(country.continent_code)
483
484 if ns.show_name:
485 line.append(country.name)
486
487 # Format the output
488 line = " ".join(line)
489
490 # Print the output
491 print(line)
492
493 def handle_list_networks_by_as(self, db, ns):
494 writer = self.__get_output_formatter(ns)
495
496 for asn in ns.asn:
497 f = writer(sys.stdout, prefix="AS%s" % asn)
498
499 # Print all matching networks
500 for n in db.search_networks(asn=asn, family=ns.family):
501 f.write(n)
502
503 f.finish()
504
505 def handle_list_networks_by_cc(self, db, ns):
506 writer = self.__get_output_formatter(ns)
507
508 for country_code in ns.country_code:
509 # Open standard output
510 f = writer(sys.stdout, prefix=country_code)
511
512 # Print all matching networks
513 for n in db.search_networks(country_code=country_code, family=ns.family):
514 f.write(n)
515
516 f.finish()
517
518 def handle_list_networks_by_flags(self, db, ns):
519 flags = 0
520
521 if ns.anonymous_proxy:
522 flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
523
524 if ns.satellite_provider:
525 flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
526
527 if ns.anycast:
528 flags |= location.NETWORK_FLAG_ANYCAST
529
530 if not flags:
531 raise ValueError(_("You must at least pass one flag"))
532
533 writer = self.__get_output_formatter(ns)
534 f = writer(sys.stdout, prefix="custom")
535
536 for n in db.search_networks(flags=flags, family=ns.family):
537 f.write(n)
538
539 f.finish()
540
541 def handle_export(self, db, ns):
542 countries, asns = [], []
543
544 # Translate family
545 if ns.family:
546 families = [ ns.family ]
547 else:
548 families = [ socket.AF_INET6, socket.AF_INET ]
549
550 for object in ns.objects:
551 m = re.match("^AS(\d+)$", object)
552 if m:
553 object = int(m.group(1))
554
555 asns.append(object)
556
557 elif location.country_code_is_valid(object) \
558 or object in ("A1", "A2", "A3"):
559 countries.append(object)
560
561 else:
562 log.warning("Invalid argument: %s" % object)
563 continue
564
565 # Default to exporting all countries
566 if not countries and not asns:
567 countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
568
569 # Select the output format
570 writer = self.__get_output_formatter(ns)
571
572 e = location.export.Exporter(db, writer)
573 e.export(ns.directory, countries=countries, asns=asns, families=families)
574
575
576 def format_timedelta(t):
577 s = []
578
579 if t.days:
580 s.append(
581 _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
582 )
583
584 hours = t.seconds // 3600
585 if hours:
586 s.append(
587 _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
588 )
589
590 minutes = (t.seconds % 3600) // 60
591 if minutes:
592 s.append(
593 _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
594 )
595
596 seconds = t.seconds % 60
597 if t.seconds:
598 s.append(
599 _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
600 )
601
602 if not s:
603 return _("Now")
604
605 return _("%s ago") % ", ".join(s)
606
607 def main():
608 # Run the command line interface
609 c = CLI()
610 c.run()
611
612 main()