location.in: do not confuse UTC with local time zones
[location/libloc.git] / src / python / location.in
1 #!/usr/bin/python3
2 ###############################################################################
3 # #
4 # libloc - A library to determine the location of someone on the Internet #
5 # #
6 # Copyright (C) 2017 IPFire Development Team <info@ipfire.org> #
7 # #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
12 # #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
17 # #
18 ###############################################################################
19
20 import argparse
21 import datetime
22 import ipaddress
23 import logging
24 import os
25 import re
26 import shutil
27 import socket
28 import sys
29 import time
30
31 # Load our location module
32 import location
33 import location.downloader
34 import location.export
35
36 from location.i18n import _
37
38 # Setup logging
39 log = logging.getLogger("location")
40
41 # Output formatters
42
43 class CLI(object):
44 def parse_cli(self):
45 parser = argparse.ArgumentParser(
46 description=_("Location Database Command Line Interface"),
47 )
48 subparsers = parser.add_subparsers()
49
50 # Global configuration flags
51 parser.add_argument("--debug", action="store_true",
52 help=_("Enable debug output"))
53 parser.add_argument("--quiet", action="store_true",
54 help=_("Enable quiet mode"))
55
56 # version
57 parser.add_argument("--version", action="version",
58 version="%(prog)s @VERSION@")
59
60 # database
61 parser.add_argument("--database", "-d",
62 default="@databasedir@/database.db", help=_("Path to database"),
63 )
64
65 # public key
66 parser.add_argument("--public-key", "-k",
67 default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
68 )
69
70 # Show the database version
71 version = subparsers.add_parser("version",
72 help=_("Show database version"))
73 version.set_defaults(func=self.handle_version)
74
75 # lookup an IP address
76 lookup = subparsers.add_parser("lookup",
77 help=_("Lookup one or multiple IP addresses"),
78 )
79 lookup.add_argument("address", nargs="+")
80 lookup.set_defaults(func=self.handle_lookup)
81
82 # Dump the whole database
83 dump = subparsers.add_parser("dump",
84 help=_("Dump the entire database"),
85 )
86 dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
87 dump.set_defaults(func=self.handle_dump)
88
89 # Update
90 update = subparsers.add_parser("update", help=_("Update database"))
91 update.add_argument("--cron",
92 help=_("Update the library only once per interval"),
93 choices=("daily", "weekly", "monthly"),
94 )
95 update.set_defaults(func=self.handle_update)
96
97 # Verify
98 verify = subparsers.add_parser("verify",
99 help=_("Verify the downloaded database"))
100 verify.set_defaults(func=self.handle_verify)
101
102 # Get AS
103 get_as = subparsers.add_parser("get-as",
104 help=_("Get information about one or multiple Autonomous Systems"),
105 )
106 get_as.add_argument("asn", nargs="+")
107 get_as.set_defaults(func=self.handle_get_as)
108
109 # Search for AS
110 search_as = subparsers.add_parser("search-as",
111 help=_("Search for Autonomous Systems that match the string"),
112 )
113 search_as.add_argument("query", nargs=1)
114 search_as.set_defaults(func=self.handle_search_as)
115
116 # List all networks in an AS
117 list_networks_by_as = subparsers.add_parser("list-networks-by-as",
118 help=_("Lists all networks in an AS"),
119 )
120 list_networks_by_as.add_argument("asn", nargs=1, type=int)
121 list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
122 list_networks_by_as.add_argument("--format",
123 choices=location.export.formats.keys(), default="list")
124 list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
125
126 # List all networks in a country
127 list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
128 help=_("Lists all networks in a country"),
129 )
130 list_networks_by_cc.add_argument("country_code", nargs=1)
131 list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
132 list_networks_by_cc.add_argument("--format",
133 choices=location.export.formats.keys(), default="list")
134 list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
135
136 # List all networks with flags
137 list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
138 help=_("Lists all networks with flags"),
139 )
140 list_networks_by_flags.add_argument("--anonymous-proxy",
141 action="store_true", help=_("Anonymous Proxies"),
142 )
143 list_networks_by_flags.add_argument("--satellite-provider",
144 action="store_true", help=_("Satellite Providers"),
145 )
146 list_networks_by_flags.add_argument("--anycast",
147 action="store_true", help=_("Anycasts"),
148 )
149 list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
150 list_networks_by_flags.add_argument("--format",
151 choices=location.export.formats.keys(), default="list")
152 list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
153
154 # List countries
155 list_countries = subparsers.add_parser("list-countries",
156 help=_("Lists all countries"),
157 )
158 list_countries.add_argument("--show-name",
159 action="store_true", help=_("Show the name of the country"),
160 )
161 list_countries.add_argument("--show-continent",
162 action="store_true", help=_("Show the continent"),
163 )
164 list_countries.set_defaults(func=self.handle_list_countries)
165
166 # Export
167 export = subparsers.add_parser("export",
168 help=_("Exports data in many formats to load it into packet filters"),
169 )
170 export.add_argument("--format", help=_("Output format"),
171 choices=location.export.formats.keys(), default="list")
172 export.add_argument("--directory", help=_("Output directory"), required=True)
173 export.add_argument("--family",
174 help=_("Specify address family"), choices=("ipv6", "ipv4"),
175 )
176 export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
177 export.set_defaults(func=self.handle_export)
178
179 args = parser.parse_args()
180
181 # Configure logging
182 if args.debug:
183 location.logger.set_level(logging.DEBUG)
184 elif args.quiet:
185 location.logger.set_level(logging.WARNING)
186
187 # Print usage if no action was given
188 if not "func" in args:
189 parser.print_usage()
190 sys.exit(2)
191
192 return args
193
194 def run(self):
195 # Parse command line arguments
196 args = self.parse_cli()
197
198 # Open database
199 try:
200 db = location.Database(args.database)
201 except FileNotFoundError as e:
202 # Allow continuing without a database
203 if args.func == self.handle_update:
204 db = None
205
206 else:
207 sys.stderr.write("location: Could not open database %s: %s\n" \
208 % (args.database, e))
209 sys.exit(1)
210
211 # Translate family (if present)
212 if "family" in args:
213 if args.family == "ipv6":
214 args.family = socket.AF_INET6
215 elif args.family == "ipv4":
216 args.family = socket.AF_INET
217 else:
218 args.family = 0
219
220 # Call function
221 try:
222 ret = args.func(db, args)
223
224 # Catch invalid inputs
225 except ValueError as e:
226 sys.stderr.write("%s\n" % e)
227 ret = 2
228
229 # Return with exit code
230 if ret:
231 sys.exit(ret)
232
233 # Otherwise just exit
234 sys.exit(0)
235
236 def handle_version(self, db, ns):
237 """
238 Print the version of the database
239 """
240 t = time.strftime(
241 "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
242 )
243
244 print(t)
245
246 def handle_lookup(self, db, ns):
247 ret = 0
248
249 format = " %-24s: %s"
250
251 for address in ns.address:
252 try:
253 network = db.lookup(address)
254 except ValueError:
255 print(_("Invalid IP address: %s") % address, file=sys.stderr)
256
257 args = {
258 "address" : address,
259 "network" : network,
260 }
261
262 # Nothing found?
263 if not network:
264 print(_("Nothing found for %(address)s") % args, file=sys.stderr)
265 ret = 1
266 continue
267
268 print("%s:" % address)
269 print(format % (_("Network"), network))
270
271 # Print country
272 if network.country_code:
273 country = db.get_country(network.country_code)
274
275 print(format % (
276 _("Country"),
277 country.name if country else network.country_code),
278 )
279
280 # Print AS information
281 if network.asn:
282 autonomous_system = db.get_as(network.asn)
283
284 print(format % (
285 _("Autonomous System"),
286 autonomous_system or "AS%s" % network.asn),
287 )
288
289 # Anonymous Proxy
290 if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
291 print(format % (
292 _("Anonymous Proxy"), _("yes"),
293 ))
294
295 # Satellite Provider
296 if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
297 print(format % (
298 _("Satellite Provider"), _("yes"),
299 ))
300
301 # Anycast
302 if network.has_flag(location.NETWORK_FLAG_ANYCAST):
303 print(format % (
304 _("Anycast"), _("yes"),
305 ))
306
307 return ret
308
309 def handle_dump(self, db, ns):
310 # Use output file or write to stdout
311 f = ns.output or sys.stdout
312
313 # Format everything like this
314 format = "%-24s %s\n"
315
316 # Write metadata
317 f.write("#\n# Location Database Export\n#\n")
318
319 f.write("# Generated: %s\n" % time.strftime(
320 "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
321 ))
322
323 if db.vendor:
324 f.write("# Vendor: %s\n" % db.vendor)
325
326 if db.license:
327 f.write("# License: %s\n" % db.license)
328
329 f.write("#\n")
330
331 if db.description:
332 for line in db.description.splitlines():
333 line = "# %s" % line
334 f.write("%s\n" % line.rstrip())
335
336 f.write("#\n")
337
338 # Iterate over all ASes
339 for a in db.ases:
340 f.write("\n")
341 f.write(format % ("aut-num:", "AS%s" % a.number))
342 f.write(format % ("name:", a.name))
343
344 flags = {
345 location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:",
346 location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
347 location.NETWORK_FLAG_ANYCAST : "is-anycast:",
348 }
349
350 # Iterate over all networks
351 for n in db.networks:
352 f.write("\n")
353 f.write(format % ("net:", n))
354
355 if n.country_code:
356 f.write(format % ("country:", n.country_code))
357
358 if n.asn:
359 f.write(format % ("aut-num:", n.asn))
360
361 # Print all flags
362 for flag in flags:
363 if n.has_flag(flag):
364 f.write(format % (flags[flag], "yes"))
365
366 def handle_get_as(self, db, ns):
367 """
368 Gets information about Autonomous Systems
369 """
370 ret = 0
371
372 for asn in ns.asn:
373 try:
374 asn = int(asn)
375 except ValueError:
376 print(_("Invalid ASN: %s") % asn, file=sys.stderr)
377 ret = 1
378 continue
379
380 # Fetch AS from database
381 a = db.get_as(asn)
382
383 # Nothing found
384 if not a:
385 print(_("Could not find AS%s") % asn, file=sys.stderr)
386 ret = 1
387 continue
388
389 print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
390
391 return ret
392
393 def handle_search_as(self, db, ns):
394 for query in ns.query:
395 # Print all matches ASes
396 for a in db.search_as(query):
397 print(a)
398
399 def handle_update(self, db, ns):
400 if ns.cron and db:
401 now = time.time()
402
403 if ns.cron == "daily":
404 delta = datetime.timedelta(days=1)
405 elif ns.cron == "weekly":
406 delta = datetime.timedelta(days=7)
407 elif ns.cron == "monthly":
408 delta = datetime.timedelta(days=30)
409
410 delta = delta.total_seconds()
411
412 # Check if the database has recently been updated
413 if db.created_at >= (now - delta):
414 log.info(
415 _("The database has been updated recently"),
416 )
417 return 3
418
419 # Fetch the timestamp we need from DNS
420 t = location.discover_latest_version()
421
422 # Check the version of the local database
423 if db and t and db.created_at >= t:
424 log.info("Already on the latest version")
425 return
426
427 # Download the database into the correct directory
428 tmpdir = os.path.dirname(ns.database)
429
430 # Create a downloader
431 d = location.downloader.Downloader()
432
433 # Try downloading a new database
434 try:
435 t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
436
437 # If no file could be downloaded, log a message
438 except FileNotFoundError as e:
439 log.error("Could not download a new database")
440 return 1
441
442 # If we have not received a new file, there is nothing to do
443 if not t:
444 return 3
445
446 # Move temporary file to destination
447 shutil.move(t.name, ns.database)
448
449 return 0
450
451 def handle_verify(self, db, ns):
452 # Verify the database
453 with open(ns.public_key, "r") as f:
454 if not db.verify(f):
455 log.error("Could not verify database")
456 return 1
457
458 # Success
459 log.debug("Database successfully verified")
460 return 0
461
462 def __get_output_formatter(self, ns):
463 try:
464 cls = location.export.formats[ns.format]
465 except KeyError:
466 cls = location.export.OutputFormatter
467
468 return cls
469
470 def handle_list_countries(self, db, ns):
471 for country in db.countries:
472 line = [
473 country.code,
474 ]
475
476 if ns.show_continent:
477 line.append(country.continent_code)
478
479 if ns.show_name:
480 line.append(country.name)
481
482 # Format the output
483 line = " ".join(line)
484
485 # Print the output
486 print(line)
487
488 def handle_list_networks_by_as(self, db, ns):
489 writer = self.__get_output_formatter(ns)
490
491 for asn in ns.asn:
492 f = writer(sys.stdout, prefix="AS%s" % asn)
493
494 # Print all matching networks
495 for n in db.search_networks(asn=asn, family=ns.family):
496 f.write(n)
497
498 f.finish()
499
500 def handle_list_networks_by_cc(self, db, ns):
501 writer = self.__get_output_formatter(ns)
502
503 for country_code in ns.country_code:
504 # Open standard output
505 f = writer(sys.stdout, prefix=country_code)
506
507 # Print all matching networks
508 for n in db.search_networks(country_code=country_code, family=ns.family):
509 f.write(n)
510
511 f.finish()
512
513 def handle_list_networks_by_flags(self, db, ns):
514 flags = 0
515
516 if ns.anonymous_proxy:
517 flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
518
519 if ns.satellite_provider:
520 flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
521
522 if ns.anycast:
523 flags |= location.NETWORK_FLAG_ANYCAST
524
525 if not flags:
526 raise ValueError(_("You must at least pass one flag"))
527
528 writer = self.__get_output_formatter(ns)
529 f = writer(sys.stdout, prefix="custom")
530
531 for n in db.search_networks(flags=flags, family=ns.family):
532 f.write(n)
533
534 f.finish()
535
536 def handle_export(self, db, ns):
537 countries, asns = [], []
538
539 # Translate family
540 if ns.family:
541 families = [ ns.family ]
542 else:
543 families = [ socket.AF_INET6, socket.AF_INET ]
544
545 for object in ns.objects:
546 m = re.match("^AS(\d+)$", object)
547 if m:
548 object = int(m.group(1))
549
550 asns.append(object)
551
552 elif location.country_code_is_valid(object) \
553 or object in ("A1", "A2", "A3"):
554 countries.append(object)
555
556 else:
557 log.warning("Invalid argument: %s" % object)
558 continue
559
560 # Default to exporting all countries
561 if not countries and not asns:
562 countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
563
564 # Select the output format
565 writer = self.__get_output_formatter(ns)
566
567 e = location.export.Exporter(db, writer)
568 e.export(ns.directory, countries=countries, asns=asns, families=families)
569
570
571 def format_timedelta(t):
572 s = []
573
574 if t.days:
575 s.append(
576 _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
577 )
578
579 hours = t.seconds // 3600
580 if hours:
581 s.append(
582 _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
583 )
584
585 minutes = (t.seconds % 3600) // 60
586 if minutes:
587 s.append(
588 _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
589 )
590
591 seconds = t.seconds % 60
592 if t.seconds:
593 s.append(
594 _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
595 )
596
597 if not s:
598 return _("Now")
599
600 return _("%s ago") % ", ".join(s)
601
602 def main():
603 # Run the command line interface
604 c = CLI()
605 c.run()
606
607 main()