location update: Remove double conversion of timestamps
[location/libloc.git] / src / python / location.in
1 #!/usr/bin/python3
2 ###############################################################################
3 # #
4 # libloc - A library to determine the location of someone on the Internet #
5 # #
6 # Copyright (C) 2017 IPFire Development Team <info@ipfire.org> #
7 # #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
12 # #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
17 # #
18 ###############################################################################
19
20 import argparse
21 import datetime
22 import ipaddress
23 import logging
24 import os
25 import re
26 import shutil
27 import socket
28 import sys
29 import time
30
31 # Load our location module
32 import location
33 import location.downloader
34 import location.export
35
36 from location.i18n import _
37
38 # Setup logging
39 log = logging.getLogger("location")
40
41 # Output formatters
42
43 class CLI(object):
44 def parse_cli(self):
45 parser = argparse.ArgumentParser(
46 description=_("Location Database Command Line Interface"),
47 )
48 subparsers = parser.add_subparsers()
49
50 # Global configuration flags
51 parser.add_argument("--debug", action="store_true",
52 help=_("Enable debug output"))
53 parser.add_argument("--quiet", action="store_true",
54 help=_("Enable quiet mode"))
55
56 # version
57 parser.add_argument("--version", action="version",
58 version="%(prog)s @VERSION@")
59
60 # database
61 parser.add_argument("--database", "-d",
62 default="@databasedir@/database.db", help=_("Path to database"),
63 )
64
65 # public key
66 parser.add_argument("--public-key", "-k",
67 default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
68 )
69
70 # Show the database version
71 version = subparsers.add_parser("version",
72 help=_("Show database version"))
73 version.set_defaults(func=self.handle_version)
74
75 # lookup an IP address
76 lookup = subparsers.add_parser("lookup",
77 help=_("Lookup one or multiple IP addresses"),
78 )
79 lookup.add_argument("address", nargs="+")
80 lookup.set_defaults(func=self.handle_lookup)
81
82 # Dump the whole database
83 dump = subparsers.add_parser("dump",
84 help=_("Dump the entire database"),
85 )
86 dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
87 dump.set_defaults(func=self.handle_dump)
88
89 # Update
90 update = subparsers.add_parser("update", help=_("Update database"))
91 update.add_argument("--cron",
92 help=_("Update the library only once per interval"),
93 choices=("daily", "weekly", "monthly"),
94 )
95 update.set_defaults(func=self.handle_update)
96
97 # Verify
98 verify = subparsers.add_parser("verify",
99 help=_("Verify the downloaded database"))
100 verify.set_defaults(func=self.handle_verify)
101
102 # Get AS
103 get_as = subparsers.add_parser("get-as",
104 help=_("Get information about one or multiple Autonomous Systems"),
105 )
106 get_as.add_argument("asn", nargs="+")
107 get_as.set_defaults(func=self.handle_get_as)
108
109 # Search for AS
110 search_as = subparsers.add_parser("search-as",
111 help=_("Search for Autonomous Systems that match the string"),
112 )
113 search_as.add_argument("query", nargs=1)
114 search_as.set_defaults(func=self.handle_search_as)
115
116 # List all networks in an AS
117 list_networks_by_as = subparsers.add_parser("list-networks-by-as",
118 help=_("Lists all networks in an AS"),
119 )
120 list_networks_by_as.add_argument("asn", nargs=1, type=int)
121 list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
122 list_networks_by_as.add_argument("--format",
123 choices=location.export.formats.keys(), default="list")
124 list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
125
126 # List all networks in a country
127 list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
128 help=_("Lists all networks in a country"),
129 )
130 list_networks_by_cc.add_argument("country_code", nargs=1)
131 list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
132 list_networks_by_cc.add_argument("--format",
133 choices=location.export.formats.keys(), default="list")
134 list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
135
136 # List all networks with flags
137 list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
138 help=_("Lists all networks with flags"),
139 )
140 list_networks_by_flags.add_argument("--anonymous-proxy",
141 action="store_true", help=_("Anonymous Proxies"),
142 )
143 list_networks_by_flags.add_argument("--satellite-provider",
144 action="store_true", help=_("Satellite Providers"),
145 )
146 list_networks_by_flags.add_argument("--anycast",
147 action="store_true", help=_("Anycasts"),
148 )
149 list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
150 list_networks_by_flags.add_argument("--format",
151 choices=location.export.formats.keys(), default="list")
152 list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
153
154 # List countries
155 list_countries = subparsers.add_parser("list-countries",
156 help=_("Lists all countries"),
157 )
158 list_countries.add_argument("--show-name",
159 action="store_true", help=_("Show the name of the country"),
160 )
161 list_countries.add_argument("--show-continent",
162 action="store_true", help=_("Show the continent"),
163 )
164 list_countries.set_defaults(func=self.handle_list_countries)
165
166 # Export
167 export = subparsers.add_parser("export",
168 help=_("Exports data in many formats to load it into packet filters"),
169 )
170 export.add_argument("--format", help=_("Output format"),
171 choices=location.export.formats.keys(), default="list")
172 export.add_argument("--directory", help=_("Output directory"), required=True)
173 export.add_argument("--family",
174 help=_("Specify address family"), choices=("ipv6", "ipv4"),
175 )
176 export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
177 export.set_defaults(func=self.handle_export)
178
179 args = parser.parse_args()
180
181 # Configure logging
182 if args.debug:
183 location.logger.set_level(logging.DEBUG)
184 elif args.quiet:
185 location.logger.set_level(logging.WARNING)
186
187 # Print usage if no action was given
188 if not "func" in args:
189 parser.print_usage()
190 sys.exit(2)
191
192 return args
193
194 def run(self):
195 # Parse command line arguments
196 args = self.parse_cli()
197
198 # Open database
199 try:
200 db = location.Database(args.database)
201 except FileNotFoundError as e:
202 # Allow continuing without a database
203 if args.func == self.handle_update:
204 db = None
205
206 else:
207 sys.stderr.write("location: Could not open database %s: %s\n" \
208 % (args.database, e))
209 sys.exit(1)
210
211 # Translate family (if present)
212 if "family" in args:
213 if args.family == "ipv6":
214 args.family = socket.AF_INET6
215 elif args.family == "ipv4":
216 args.family = socket.AF_INET
217 else:
218 args.family = 0
219
220 # Call function
221 try:
222 ret = args.func(db, args)
223
224 # Catch invalid inputs
225 except ValueError as e:
226 sys.stderr.write("%s\n" % e)
227 ret = 2
228
229 # Return with exit code
230 if ret:
231 sys.exit(ret)
232
233 # Otherwise just exit
234 sys.exit(0)
235
236 def handle_version(self, db, ns):
237 """
238 Print the version of the database
239 """
240 t = time.strftime(
241 "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
242 )
243
244 print(t)
245
246 def handle_lookup(self, db, ns):
247 ret = 0
248
249 format = " %-24s: %s"
250
251 for address in ns.address:
252 try:
253 network = db.lookup(address)
254 except ValueError:
255 print(_("Invalid IP address: %s") % address, file=sys.stderr)
256
257 args = {
258 "address" : address,
259 "network" : network,
260 }
261
262 # Nothing found?
263 if not network:
264 print(_("Nothing found for %(address)s") % args, file=sys.stderr)
265 ret = 1
266 continue
267
268 print("%s:" % address)
269 print(format % (_("Network"), network))
270
271 # Print country
272 if network.country_code:
273 country = db.get_country(network.country_code)
274
275 print(format % (
276 _("Country"),
277 country.name if country else network.country_code),
278 )
279
280 # Print AS information
281 if network.asn:
282 autonomous_system = db.get_as(network.asn)
283
284 print(format % (
285 _("Autonomous System"),
286 autonomous_system or "AS%s" % network.asn),
287 )
288
289 # Anonymous Proxy
290 if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
291 print(format % (
292 _("Anonymous Proxy"), _("yes"),
293 ))
294
295 # Satellite Provider
296 if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
297 print(format % (
298 _("Satellite Provider"), _("yes"),
299 ))
300
301 # Anycast
302 if network.has_flag(location.NETWORK_FLAG_ANYCAST):
303 print(format % (
304 _("Anycast"), _("yes"),
305 ))
306
307 return ret
308
309 def handle_dump(self, db, ns):
310 # Use output file or write to stdout
311 f = ns.output or sys.stdout
312
313 # Format everything like this
314 format = "%-24s %s\n"
315
316 # Write metadata
317 f.write("#\n# Location Database Export\n#\n")
318
319 f.write("# Generated: %s\n" % time.strftime(
320 "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
321 ))
322
323 if db.vendor:
324 f.write("# Vendor: %s\n" % db.vendor)
325
326 if db.license:
327 f.write("# License: %s\n" % db.license)
328
329 f.write("#\n")
330
331 if db.description:
332 for line in db.description.splitlines():
333 line = "# %s" % line
334 f.write("%s\n" % line.rstrip())
335
336 f.write("#\n")
337
338 # Iterate over all ASes
339 for a in db.ases:
340 f.write("\n")
341 f.write(format % ("aut-num:", "AS%s" % a.number))
342 f.write(format % ("name:", a.name))
343
344 flags = {
345 location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:",
346 location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
347 location.NETWORK_FLAG_ANYCAST : "is-anycast:",
348 }
349
350 # Iterate over all networks
351 for n in db.networks:
352 f.write("\n")
353 f.write(format % ("net:", n))
354
355 if n.country_code:
356 f.write(format % ("country:", n.country_code))
357
358 if n.asn:
359 f.write(format % ("aut-num:", n.asn))
360
361 # Print all flags
362 for flag in flags:
363 if n.has_flag(flag):
364 f.write(format % (flags[flag], "yes"))
365
366 def handle_get_as(self, db, ns):
367 """
368 Gets information about Autonomous Systems
369 """
370 ret = 0
371
372 for asn in ns.asn:
373 try:
374 asn = int(asn)
375 except ValueError:
376 print(_("Invalid ASN: %s") % asn, file=sys.stderr)
377 ret = 1
378 continue
379
380 # Fetch AS from database
381 a = db.get_as(asn)
382
383 # Nothing found
384 if not a:
385 print(_("Could not find AS%s") % asn, file=sys.stderr)
386 ret = 1
387 continue
388
389 print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
390
391 return ret
392
393 def handle_search_as(self, db, ns):
394 for query in ns.query:
395 # Print all matches ASes
396 for a in db.search_as(query):
397 print(a)
398
399 def handle_update(self, db, ns):
400 if ns.cron and db:
401 now = datetime.datetime.utcnow()
402
403 # Parse the database timestamp
404 t = datetime.datetime.utcfromtimestamp(db.created_at)
405
406 if ns.cron == "daily":
407 delta = datetime.timedelta(days=1)
408 elif ns.cron == "weekly":
409 delta = datetime.timedelta(days=7)
410 elif ns.cron == "monthly":
411 delta = datetime.timedelta(days=30)
412
413 # Check if the database has recently been updated
414 if t >= (now - delta):
415 log.info(
416 _("The database has been updated recently (%s)") % \
417 format_timedelta(now - t),
418 )
419 return 3
420
421 # Fetch the timestamp we need from DNS
422 t = location.discover_latest_version()
423
424 # Check the version of the local database
425 if db and t and db.created_at >= t:
426 log.info("Already on the latest version")
427 return
428
429 # Download the database into the correct directory
430 tmpdir = os.path.dirname(ns.database)
431
432 # Create a downloader
433 d = location.downloader.Downloader()
434
435 # Try downloading a new database
436 try:
437 t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
438
439 # If no file could be downloaded, log a message
440 except FileNotFoundError as e:
441 log.error("Could not download a new database")
442 return 1
443
444 # If we have not received a new file, there is nothing to do
445 if not t:
446 return 3
447
448 # Move temporary file to destination
449 shutil.move(t.name, ns.database)
450
451 return 0
452
453 def handle_verify(self, db, ns):
454 # Verify the database
455 with open(ns.public_key, "r") as f:
456 if not db.verify(f):
457 log.error("Could not verify database")
458 return 1
459
460 # Success
461 log.debug("Database successfully verified")
462 return 0
463
464 def __get_output_formatter(self, ns):
465 try:
466 cls = location.export.formats[ns.format]
467 except KeyError:
468 cls = location.export.OutputFormatter
469
470 return cls
471
472 def handle_list_countries(self, db, ns):
473 for country in db.countries:
474 line = [
475 country.code,
476 ]
477
478 if ns.show_continent:
479 line.append(country.continent_code)
480
481 if ns.show_name:
482 line.append(country.name)
483
484 # Format the output
485 line = " ".join(line)
486
487 # Print the output
488 print(line)
489
490 def handle_list_networks_by_as(self, db, ns):
491 writer = self.__get_output_formatter(ns)
492
493 for asn in ns.asn:
494 f = writer(sys.stdout, prefix="AS%s" % asn)
495
496 # Print all matching networks
497 for n in db.search_networks(asn=asn, family=ns.family):
498 f.write(n)
499
500 f.finish()
501
502 def handle_list_networks_by_cc(self, db, ns):
503 writer = self.__get_output_formatter(ns)
504
505 for country_code in ns.country_code:
506 # Open standard output
507 f = writer(sys.stdout, prefix=country_code)
508
509 # Print all matching networks
510 for n in db.search_networks(country_code=country_code, family=ns.family):
511 f.write(n)
512
513 f.finish()
514
515 def handle_list_networks_by_flags(self, db, ns):
516 flags = 0
517
518 if ns.anonymous_proxy:
519 flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
520
521 if ns.satellite_provider:
522 flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
523
524 if ns.anycast:
525 flags |= location.NETWORK_FLAG_ANYCAST
526
527 if not flags:
528 raise ValueError(_("You must at least pass one flag"))
529
530 writer = self.__get_output_formatter(ns)
531 f = writer(sys.stdout, prefix="custom")
532
533 for n in db.search_networks(flags=flags, family=ns.family):
534 f.write(n)
535
536 f.finish()
537
538 def handle_export(self, db, ns):
539 countries, asns = [], []
540
541 # Translate family
542 if ns.family:
543 families = [ ns.family ]
544 else:
545 families = [ socket.AF_INET6, socket.AF_INET ]
546
547 for object in ns.objects:
548 m = re.match("^AS(\d+)$", object)
549 if m:
550 object = int(m.group(1))
551
552 asns.append(object)
553
554 elif location.country_code_is_valid(object) \
555 or object in ("A1", "A2", "A3"):
556 countries.append(object)
557
558 else:
559 log.warning("Invalid argument: %s" % object)
560 continue
561
562 # Default to exporting all countries
563 if not countries and not asns:
564 countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
565
566 # Select the output format
567 writer = self.__get_output_formatter(ns)
568
569 e = location.export.Exporter(db, writer)
570 e.export(ns.directory, countries=countries, asns=asns, families=families)
571
572
573 def format_timedelta(t):
574 s = []
575
576 if t.days:
577 s.append(
578 _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
579 )
580
581 hours = t.seconds // 3600
582 if hours:
583 s.append(
584 _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
585 )
586
587 minutes = (t.seconds % 3600) // 60
588 if minutes:
589 s.append(
590 _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
591 )
592
593 seconds = t.seconds % 60
594 if t.seconds:
595 s.append(
596 _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
597 )
598
599 if not s:
600 return _("Now")
601
602 return _("%s ago") % ", ".join(s)
603
604 def main():
605 # Run the command line interface
606 c = CLI()
607 c.run()
608
609 main()