location update: Remove double conversion of timestamps
[location/libloc.git] / src / python / location.in
CommitLineData
5118a4b8
MT
1#!/usr/bin/python3
2###############################################################################
3# #
4# libloc - A library to determine the location of someone on the Internet #
5# #
6# Copyright (C) 2017 IPFire Development Team <info@ipfire.org> #
7# #
8# This library is free software; you can redistribute it and/or #
9# modify it under the terms of the GNU Lesser General Public #
10# License as published by the Free Software Foundation; either #
11# version 2.1 of the License, or (at your option) any later version. #
12# #
13# This library is distributed in the hope that it will be useful, #
14# but WITHOUT ANY WARRANTY; without even the implied warranty of #
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16# Lesser General Public License for more details. #
17# #
18###############################################################################
19
20import argparse
a6f1e346 21import datetime
4439e317 22import ipaddress
a6f1e346 23import logging
4439e317 24import os
88ef7e9c 25import re
a6f1e346 26import shutil
4439e317 27import socket
5118a4b8 28import sys
a68a46f5 29import time
5118a4b8
MT
30
31# Load our location module
32import location
a6f1e346 33import location.downloader
88ef7e9c
MT
34import location.export
35
7dccb767 36from location.i18n import _
5118a4b8 37
a6f1e346
MT
38# Setup logging
39log = logging.getLogger("location")
40
4439e317
MT
41# Output formatters
42
5118a4b8 43class CLI(object):
5118a4b8
MT
44 def parse_cli(self):
45 parser = argparse.ArgumentParser(
46 description=_("Location Database Command Line Interface"),
47 )
48 subparsers = parser.add_subparsers()
49
50 # Global configuration flags
51 parser.add_argument("--debug", action="store_true",
52 help=_("Enable debug output"))
bc1f5f53
MT
53 parser.add_argument("--quiet", action="store_true",
54 help=_("Enable quiet mode"))
5118a4b8 55
ddb184be
MT
56 # version
57 parser.add_argument("--version", action="version",
d2714e4a 58 version="%(prog)s @VERSION@")
ddb184be 59
2538ed9a
MT
60 # database
61 parser.add_argument("--database", "-d",
62 default="@databasedir@/database.db", help=_("Path to database"),
63 )
64
726f9984
MT
65 # public key
66 parser.add_argument("--public-key", "-k",
67 default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
68 )
69
9f64f1eb
MT
70 # Show the database version
71 version = subparsers.add_parser("version",
72 help=_("Show database version"))
73 version.set_defaults(func=self.handle_version)
74
5118a4b8
MT
75 # lookup an IP address
76 lookup = subparsers.add_parser("lookup",
77 help=_("Lookup one or multiple IP addresses"),
78 )
79 lookup.add_argument("address", nargs="+")
80 lookup.set_defaults(func=self.handle_lookup)
81
a68a46f5
MT
82 # Dump the whole database
83 dump = subparsers.add_parser("dump",
84 help=_("Dump the entire database"),
85 )
86 dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
87 dump.set_defaults(func=self.handle_dump)
88
a6f1e346
MT
89 # Update
90 update = subparsers.add_parser("update", help=_("Update database"))
97769987
MT
91 update.add_argument("--cron",
92 help=_("Update the library only once per interval"),
93 choices=("daily", "weekly", "monthly"),
94 )
a6f1e346
MT
95 update.set_defaults(func=self.handle_update)
96
97 # Verify
98 verify = subparsers.add_parser("verify",
99 help=_("Verify the downloaded database"))
100 verify.set_defaults(func=self.handle_verify)
101
fadc1af0
MT
102 # Get AS
103 get_as = subparsers.add_parser("get-as",
104 help=_("Get information about one or multiple Autonomous Systems"),
105 )
106 get_as.add_argument("asn", nargs="+")
107 get_as.set_defaults(func=self.handle_get_as)
108
da3e360e
MT
109 # Search for AS
110 search_as = subparsers.add_parser("search-as",
111 help=_("Search for Autonomous Systems that match the string"),
112 )
113 search_as.add_argument("query", nargs=1)
114 search_as.set_defaults(func=self.handle_search_as)
115
43154ed7
MT
116 # List all networks in an AS
117 list_networks_by_as = subparsers.add_parser("list-networks-by-as",
118 help=_("Lists all networks in an AS"),
119 )
120 list_networks_by_as.add_argument("asn", nargs=1, type=int)
44e5ef71 121 list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
88ef7e9c
MT
122 list_networks_by_as.add_argument("--format",
123 choices=location.export.formats.keys(), default="list")
43154ed7
MT
124 list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
125
ccc7ab4e 126 # List all networks in a country
b5cdfad7 127 list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
ccc7ab4e
MT
128 help=_("Lists all networks in a country"),
129 )
b5cdfad7 130 list_networks_by_cc.add_argument("country_code", nargs=1)
44e5ef71 131 list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
88ef7e9c
MT
132 list_networks_by_cc.add_argument("--format",
133 choices=location.export.formats.keys(), default="list")
b5cdfad7 134 list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
ccc7ab4e 135
bbdb2e0a
MT
136 # List all networks with flags
137 list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
138 help=_("Lists all networks with flags"),
139 )
140 list_networks_by_flags.add_argument("--anonymous-proxy",
141 action="store_true", help=_("Anonymous Proxies"),
142 )
143 list_networks_by_flags.add_argument("--satellite-provider",
144 action="store_true", help=_("Satellite Providers"),
145 )
146 list_networks_by_flags.add_argument("--anycast",
147 action="store_true", help=_("Anycasts"),
148 )
44e5ef71 149 list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
88ef7e9c
MT
150 list_networks_by_flags.add_argument("--format",
151 choices=location.export.formats.keys(), default="list")
bbdb2e0a
MT
152 list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
153
fa9a3663
MT
154 # List countries
155 list_countries = subparsers.add_parser("list-countries",
156 help=_("Lists all countries"),
157 )
158 list_countries.add_argument("--show-name",
159 action="store_true", help=_("Show the name of the country"),
160 )
161 list_countries.add_argument("--show-continent",
162 action="store_true", help=_("Show the continent"),
163 )
164 list_countries.set_defaults(func=self.handle_list_countries)
165
88ef7e9c
MT
166 # Export
167 export = subparsers.add_parser("export",
168 help=_("Exports data in many formats to load it into packet filters"),
169 )
170 export.add_argument("--format", help=_("Output format"),
171 choices=location.export.formats.keys(), default="list")
172 export.add_argument("--directory", help=_("Output directory"), required=True)
173 export.add_argument("--family",
174 help=_("Specify address family"), choices=("ipv6", "ipv4"),
175 )
10fa313b 176 export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
88ef7e9c
MT
177 export.set_defaults(func=self.handle_export)
178
78f37815
MT
179 args = parser.parse_args()
180
bc1f5f53 181 # Configure logging
f9de5e61
MT
182 if args.debug:
183 location.logger.set_level(logging.DEBUG)
bc1f5f53
MT
184 elif args.quiet:
185 location.logger.set_level(logging.WARNING)
f9de5e61 186
78f37815
MT
187 # Print usage if no action was given
188 if not "func" in args:
189 parser.print_usage()
190 sys.exit(2)
191
192 return args
5118a4b8
MT
193
194 def run(self):
195 # Parse command line arguments
196 args = self.parse_cli()
197
2538ed9a
MT
198 # Open database
199 try:
200 db = location.Database(args.database)
201 except FileNotFoundError as e:
2201d904
MT
202 # Allow continuing without a database
203 if args.func == self.handle_update:
204 db = None
205
206 else:
207 sys.stderr.write("location: Could not open database %s: %s\n" \
208 % (args.database, e))
209 sys.exit(1)
2538ed9a 210
6961aaf3
MT
211 # Translate family (if present)
212 if "family" in args:
213 if args.family == "ipv6":
214 args.family = socket.AF_INET6
215 elif args.family == "ipv4":
216 args.family = socket.AF_INET
217 else:
218 args.family = 0
44e5ef71 219
5118a4b8 220 # Call function
228d0e74
MT
221 try:
222 ret = args.func(db, args)
223
224 # Catch invalid inputs
225 except ValueError as e:
226 sys.stderr.write("%s\n" % e)
227 ret = 2
5118a4b8
MT
228
229 # Return with exit code
230 if ret:
231 sys.exit(ret)
232
233 # Otherwise just exit
234 sys.exit(0)
235
9f64f1eb
MT
236 def handle_version(self, db, ns):
237 """
238 Print the version of the database
239 """
240 t = time.strftime(
241 "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
242 )
243
244 print(t)
245
2538ed9a 246 def handle_lookup(self, db, ns):
5118a4b8
MT
247 ret = 0
248
fbf925c8
MT
249 format = " %-24s: %s"
250
5118a4b8
MT
251 for address in ns.address:
252 try:
fbf925c8 253 network = db.lookup(address)
5118a4b8 254 except ValueError:
9f2f5d13 255 print(_("Invalid IP address: %s") % address, file=sys.stderr)
5118a4b8
MT
256
257 args = {
258 "address" : address,
fbf925c8 259 "network" : network,
5118a4b8
MT
260 }
261
262 # Nothing found?
fbf925c8 263 if not network:
9f2f5d13 264 print(_("Nothing found for %(address)s") % args, file=sys.stderr)
5118a4b8
MT
265 ret = 1
266 continue
267
fbf925c8
MT
268 print("%s:" % address)
269 print(format % (_("Network"), network))
5118a4b8 270
fbf925c8
MT
271 # Print country
272 if network.country_code:
072ca100
MT
273 country = db.get_country(network.country_code)
274
275 print(format % (
276 _("Country"),
277 country.name if country else network.country_code),
278 )
5118a4b8 279
fbf925c8
MT
280 # Print AS information
281 if network.asn:
282 autonomous_system = db.get_as(network.asn)
5118a4b8 283
fbf925c8
MT
284 print(format % (
285 _("Autonomous System"),
286 autonomous_system or "AS%s" % network.asn),
287 )
5118a4b8 288
ee83fe2e
MT
289 # Anonymous Proxy
290 if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
291 print(format % (
292 _("Anonymous Proxy"), _("yes"),
293 ))
294
295 # Satellite Provider
296 if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
297 print(format % (
298 _("Satellite Provider"), _("yes"),
299 ))
300
301 # Anycast
302 if network.has_flag(location.NETWORK_FLAG_ANYCAST):
303 print(format % (
304 _("Anycast"), _("yes"),
305 ))
306
5118a4b8
MT
307 return ret
308
a68a46f5
MT
309 def handle_dump(self, db, ns):
310 # Use output file or write to stdout
311 f = ns.output or sys.stdout
312
d6d06375
MT
313 # Format everything like this
314 format = "%-24s %s\n"
315
a68a46f5
MT
316 # Write metadata
317 f.write("#\n# Location Database Export\n#\n")
318
319 f.write("# Generated: %s\n" % time.strftime(
320 "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
321 ))
322
323 if db.vendor:
324 f.write("# Vendor: %s\n" % db.vendor)
325
326 if db.license:
327 f.write("# License: %s\n" % db.license)
328
329 f.write("#\n")
330
331 if db.description:
332 for line in db.description.splitlines():
cfbae90f
PM
333 line = "# %s" % line
334 f.write("%s\n" % line.rstrip())
a68a46f5
MT
335
336 f.write("#\n")
337
338 # Iterate over all ASes
339 for a in db.ases:
340 f.write("\n")
d6d06375
MT
341 f.write(format % ("aut-num:", "AS%s" % a.number))
342 f.write(format % ("name:", a.name))
343
344 flags = {
345 location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:",
346 location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
347 location.NETWORK_FLAG_ANYCAST : "is-anycast:",
348 }
a68a46f5
MT
349
350 # Iterate over all networks
351 for n in db.networks:
352 f.write("\n")
d6d06375 353 f.write(format % ("net:", n))
a68a46f5
MT
354
355 if n.country_code:
d6d06375 356 f.write(format % ("country:", n.country_code))
a68a46f5
MT
357
358 if n.asn:
03cd8096 359 f.write(format % ("aut-num:", n.asn))
d6d06375
MT
360
361 # Print all flags
362 for flag in flags:
363 if n.has_flag(flag):
364 f.write(format % (flags[flag], "yes"))
a68a46f5 365
2538ed9a 366 def handle_get_as(self, db, ns):
fadc1af0
MT
367 """
368 Gets information about Autonomous Systems
369 """
370 ret = 0
371
372 for asn in ns.asn:
373 try:
374 asn = int(asn)
375 except ValueError:
9f2f5d13 376 print(_("Invalid ASN: %s") % asn, file=sys.stderr)
fadc1af0
MT
377 ret = 1
378 continue
379
380 # Fetch AS from database
2538ed9a 381 a = db.get_as(asn)
fadc1af0
MT
382
383 # Nothing found
384 if not a:
9f2f5d13 385 print(_("Could not find AS%s") % asn, file=sys.stderr)
fadc1af0
MT
386 ret = 1
387 continue
388
389 print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
390
391 return ret
5118a4b8 392
2538ed9a 393 def handle_search_as(self, db, ns):
da3e360e
MT
394 for query in ns.query:
395 # Print all matches ASes
2538ed9a 396 for a in db.search_as(query):
da3e360e
MT
397 print(a)
398
a6f1e346 399 def handle_update(self, db, ns):
97769987
MT
400 if ns.cron and db:
401 now = datetime.datetime.utcnow()
402
403 # Parse the database timestamp
5e63ddd3 404 t = datetime.datetime.utcfromtimestamp(db.created_at)
97769987
MT
405
406 if ns.cron == "daily":
407 delta = datetime.timedelta(days=1)
408 elif ns.cron == "weekly":
409 delta = datetime.timedelta(days=7)
410 elif ns.cron == "monthly":
411 delta = datetime.timedelta(days=30)
412
413 # Check if the database has recently been updated
414 if t >= (now - delta):
415 log.info(
f869f105 416 _("The database has been updated recently (%s)") % \
97769987
MT
417 format_timedelta(now - t),
418 )
a243587a 419 return 3
97769987 420
a6f1e346
MT
421 # Fetch the timestamp we need from DNS
422 t = location.discover_latest_version()
423
a6f1e346 424 # Check the version of the local database
0c74f6b1 425 if db and t and db.created_at >= t:
a6f1e346
MT
426 log.info("Already on the latest version")
427 return
428
429 # Download the database into the correct directory
430 tmpdir = os.path.dirname(ns.database)
431
432 # Create a downloader
433 d = location.downloader.Downloader()
434
435 # Try downloading a new database
436 try:
437 t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
438
439 # If no file could be downloaded, log a message
440 except FileNotFoundError as e:
441 log.error("Could not download a new database")
442 return 1
443
444 # If we have not received a new file, there is nothing to do
445 if not t:
446 return 3
447
448 # Move temporary file to destination
449 shutil.move(t.name, ns.database)
450
451 return 0
452
e99a7226 453 def handle_verify(self, db, ns):
a6f1e346
MT
454 # Verify the database
455 with open(ns.public_key, "r") as f:
456 if not db.verify(f):
457 log.error("Could not verify database")
458 return 1
459
460 # Success
461 log.debug("Database successfully verified")
462 return 0
463
4439e317
MT
464 def __get_output_formatter(self, ns):
465 try:
88ef7e9c 466 cls = location.export.formats[ns.format]
4439e317 467 except KeyError:
88ef7e9c 468 cls = location.export.OutputFormatter
4439e317 469
88ef7e9c 470 return cls
4439e317 471
fa9a3663
MT
472 def handle_list_countries(self, db, ns):
473 for country in db.countries:
474 line = [
475 country.code,
476 ]
477
478 if ns.show_continent:
479 line.append(country.continent_code)
480
481 if ns.show_name:
482 line.append(country.name)
483
484 # Format the output
485 line = " ".join(line)
486
487 # Print the output
488 print(line)
489
43154ed7 490 def handle_list_networks_by_as(self, db, ns):
88ef7e9c
MT
491 writer = self.__get_output_formatter(ns)
492
493 for asn in ns.asn:
494 f = writer(sys.stdout, prefix="AS%s" % asn)
495
496 # Print all matching networks
497 for n in db.search_networks(asn=asn, family=ns.family):
498 f.write(n)
499
500 f.finish()
43154ed7 501
ccc7ab4e 502 def handle_list_networks_by_cc(self, db, ns):
88ef7e9c
MT
503 writer = self.__get_output_formatter(ns)
504
505 for country_code in ns.country_code:
506 # Open standard output
507 f = writer(sys.stdout, prefix=country_code)
508
509 # Print all matching networks
510 for n in db.search_networks(country_code=country_code, family=ns.family):
511 f.write(n)
512
513 f.finish()
4439e317 514
bbdb2e0a
MT
515 def handle_list_networks_by_flags(self, db, ns):
516 flags = 0
517
518 if ns.anonymous_proxy:
519 flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
520
521 if ns.satellite_provider:
522 flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
523
524 if ns.anycast:
525 flags |= location.NETWORK_FLAG_ANYCAST
526
228d0e74
MT
527 if not flags:
528 raise ValueError(_("You must at least pass one flag"))
529
88ef7e9c
MT
530 writer = self.__get_output_formatter(ns)
531 f = writer(sys.stdout, prefix="custom")
532
533 for n in db.search_networks(flags=flags, family=ns.family):
534 f.write(n)
535
536 f.finish()
537
538 def handle_export(self, db, ns):
539 countries, asns = [], []
540
541 # Translate family
df072005
MT
542 if ns.family:
543 families = [ ns.family ]
88ef7e9c
MT
544 else:
545 families = [ socket.AF_INET6, socket.AF_INET ]
546
547 for object in ns.objects:
548 m = re.match("^AS(\d+)$", object)
549 if m:
550 object = int(m.group(1))
551
552 asns.append(object)
553
554 elif location.country_code_is_valid(object) \
555 or object in ("A1", "A2", "A3"):
556 countries.append(object)
557
558 else:
559 log.warning("Invalid argument: %s" % object)
560 continue
561
10fa313b 562 # Default to exporting all countries
88ef7e9c 563 if not countries and not asns:
10fa313b 564 countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
88ef7e9c
MT
565
566 # Select the output format
567 writer = self.__get_output_formatter(ns)
568
569 e = location.export.Exporter(db, writer)
570 e.export(ns.directory, countries=countries, asns=asns, families=families)
bbdb2e0a 571
ccc7ab4e 572
97769987
MT
573def format_timedelta(t):
574 s = []
575
576 if t.days:
577 s.append(
578 _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
579 )
580
581 hours = t.seconds // 3600
582 if hours:
583 s.append(
584 _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
585 )
586
587 minutes = (t.seconds % 3600) // 60
588 if minutes:
589 s.append(
590 _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
591 )
592
593 seconds = t.seconds % 60
594 if t.seconds:
595 s.append(
596 _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
597 )
598
599 if not s:
600 return _("Now")
601
602 return _("%s ago") % ", ".join(s)
603
5118a4b8
MT
604def main():
605 # Run the command line interface
606 c = CLI()
607 c.run()
608
609main()