location.in: do not confuse UTC with local time zones
[location/libloc.git] / src / python / location.in
CommitLineData
5118a4b8
MT
1#!/usr/bin/python3
2###############################################################################
3# #
4# libloc - A library to determine the location of someone on the Internet #
5# #
6# Copyright (C) 2017 IPFire Development Team <info@ipfire.org> #
7# #
8# This library is free software; you can redistribute it and/or #
9# modify it under the terms of the GNU Lesser General Public #
10# License as published by the Free Software Foundation; either #
11# version 2.1 of the License, or (at your option) any later version. #
12# #
13# This library is distributed in the hope that it will be useful, #
14# but WITHOUT ANY WARRANTY; without even the implied warranty of #
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16# Lesser General Public License for more details. #
17# #
18###############################################################################
19
20import argparse
a6f1e346 21import datetime
4439e317 22import ipaddress
a6f1e346 23import logging
4439e317 24import os
88ef7e9c 25import re
a6f1e346 26import shutil
4439e317 27import socket
5118a4b8 28import sys
a68a46f5 29import time
5118a4b8
MT
30
31# Load our location module
32import location
a6f1e346 33import location.downloader
88ef7e9c
MT
34import location.export
35
7dccb767 36from location.i18n import _
5118a4b8 37
a6f1e346
MT
38# Setup logging
39log = logging.getLogger("location")
40
4439e317
MT
41# Output formatters
42
5118a4b8 43class CLI(object):
5118a4b8
MT
44 def parse_cli(self):
45 parser = argparse.ArgumentParser(
46 description=_("Location Database Command Line Interface"),
47 )
48 subparsers = parser.add_subparsers()
49
50 # Global configuration flags
51 parser.add_argument("--debug", action="store_true",
52 help=_("Enable debug output"))
bc1f5f53
MT
53 parser.add_argument("--quiet", action="store_true",
54 help=_("Enable quiet mode"))
5118a4b8 55
ddb184be
MT
56 # version
57 parser.add_argument("--version", action="version",
d2714e4a 58 version="%(prog)s @VERSION@")
ddb184be 59
2538ed9a
MT
60 # database
61 parser.add_argument("--database", "-d",
62 default="@databasedir@/database.db", help=_("Path to database"),
63 )
64
726f9984
MT
65 # public key
66 parser.add_argument("--public-key", "-k",
67 default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
68 )
69
9f64f1eb
MT
70 # Show the database version
71 version = subparsers.add_parser("version",
72 help=_("Show database version"))
73 version.set_defaults(func=self.handle_version)
74
5118a4b8
MT
75 # lookup an IP address
76 lookup = subparsers.add_parser("lookup",
77 help=_("Lookup one or multiple IP addresses"),
78 )
79 lookup.add_argument("address", nargs="+")
80 lookup.set_defaults(func=self.handle_lookup)
81
a68a46f5
MT
82 # Dump the whole database
83 dump = subparsers.add_parser("dump",
84 help=_("Dump the entire database"),
85 )
86 dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
87 dump.set_defaults(func=self.handle_dump)
88
a6f1e346
MT
89 # Update
90 update = subparsers.add_parser("update", help=_("Update database"))
97769987
MT
91 update.add_argument("--cron",
92 help=_("Update the library only once per interval"),
93 choices=("daily", "weekly", "monthly"),
94 )
a6f1e346
MT
95 update.set_defaults(func=self.handle_update)
96
97 # Verify
98 verify = subparsers.add_parser("verify",
99 help=_("Verify the downloaded database"))
100 verify.set_defaults(func=self.handle_verify)
101
fadc1af0
MT
102 # Get AS
103 get_as = subparsers.add_parser("get-as",
104 help=_("Get information about one or multiple Autonomous Systems"),
105 )
106 get_as.add_argument("asn", nargs="+")
107 get_as.set_defaults(func=self.handle_get_as)
108
da3e360e
MT
109 # Search for AS
110 search_as = subparsers.add_parser("search-as",
111 help=_("Search for Autonomous Systems that match the string"),
112 )
113 search_as.add_argument("query", nargs=1)
114 search_as.set_defaults(func=self.handle_search_as)
115
43154ed7
MT
116 # List all networks in an AS
117 list_networks_by_as = subparsers.add_parser("list-networks-by-as",
118 help=_("Lists all networks in an AS"),
119 )
120 list_networks_by_as.add_argument("asn", nargs=1, type=int)
44e5ef71 121 list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
88ef7e9c
MT
122 list_networks_by_as.add_argument("--format",
123 choices=location.export.formats.keys(), default="list")
43154ed7
MT
124 list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
125
ccc7ab4e 126 # List all networks in a country
b5cdfad7 127 list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
ccc7ab4e
MT
128 help=_("Lists all networks in a country"),
129 )
b5cdfad7 130 list_networks_by_cc.add_argument("country_code", nargs=1)
44e5ef71 131 list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
88ef7e9c
MT
132 list_networks_by_cc.add_argument("--format",
133 choices=location.export.formats.keys(), default="list")
b5cdfad7 134 list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
ccc7ab4e 135
bbdb2e0a
MT
136 # List all networks with flags
137 list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
138 help=_("Lists all networks with flags"),
139 )
140 list_networks_by_flags.add_argument("--anonymous-proxy",
141 action="store_true", help=_("Anonymous Proxies"),
142 )
143 list_networks_by_flags.add_argument("--satellite-provider",
144 action="store_true", help=_("Satellite Providers"),
145 )
146 list_networks_by_flags.add_argument("--anycast",
147 action="store_true", help=_("Anycasts"),
148 )
44e5ef71 149 list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
88ef7e9c
MT
150 list_networks_by_flags.add_argument("--format",
151 choices=location.export.formats.keys(), default="list")
bbdb2e0a
MT
152 list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
153
fa9a3663
MT
154 # List countries
155 list_countries = subparsers.add_parser("list-countries",
156 help=_("Lists all countries"),
157 )
158 list_countries.add_argument("--show-name",
159 action="store_true", help=_("Show the name of the country"),
160 )
161 list_countries.add_argument("--show-continent",
162 action="store_true", help=_("Show the continent"),
163 )
164 list_countries.set_defaults(func=self.handle_list_countries)
165
88ef7e9c
MT
166 # Export
167 export = subparsers.add_parser("export",
168 help=_("Exports data in many formats to load it into packet filters"),
169 )
170 export.add_argument("--format", help=_("Output format"),
171 choices=location.export.formats.keys(), default="list")
172 export.add_argument("--directory", help=_("Output directory"), required=True)
173 export.add_argument("--family",
174 help=_("Specify address family"), choices=("ipv6", "ipv4"),
175 )
10fa313b 176 export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
88ef7e9c
MT
177 export.set_defaults(func=self.handle_export)
178
78f37815
MT
179 args = parser.parse_args()
180
bc1f5f53 181 # Configure logging
f9de5e61
MT
182 if args.debug:
183 location.logger.set_level(logging.DEBUG)
bc1f5f53
MT
184 elif args.quiet:
185 location.logger.set_level(logging.WARNING)
f9de5e61 186
78f37815
MT
187 # Print usage if no action was given
188 if not "func" in args:
189 parser.print_usage()
190 sys.exit(2)
191
192 return args
5118a4b8
MT
193
194 def run(self):
195 # Parse command line arguments
196 args = self.parse_cli()
197
2538ed9a
MT
198 # Open database
199 try:
200 db = location.Database(args.database)
201 except FileNotFoundError as e:
2201d904
MT
202 # Allow continuing without a database
203 if args.func == self.handle_update:
204 db = None
205
206 else:
207 sys.stderr.write("location: Could not open database %s: %s\n" \
208 % (args.database, e))
209 sys.exit(1)
2538ed9a 210
6961aaf3
MT
211 # Translate family (if present)
212 if "family" in args:
213 if args.family == "ipv6":
214 args.family = socket.AF_INET6
215 elif args.family == "ipv4":
216 args.family = socket.AF_INET
217 else:
218 args.family = 0
44e5ef71 219
5118a4b8 220 # Call function
228d0e74
MT
221 try:
222 ret = args.func(db, args)
223
224 # Catch invalid inputs
225 except ValueError as e:
226 sys.stderr.write("%s\n" % e)
227 ret = 2
5118a4b8
MT
228
229 # Return with exit code
230 if ret:
231 sys.exit(ret)
232
233 # Otherwise just exit
234 sys.exit(0)
235
9f64f1eb
MT
236 def handle_version(self, db, ns):
237 """
238 Print the version of the database
239 """
240 t = time.strftime(
241 "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
242 )
243
244 print(t)
245
2538ed9a 246 def handle_lookup(self, db, ns):
5118a4b8
MT
247 ret = 0
248
fbf925c8
MT
249 format = " %-24s: %s"
250
5118a4b8
MT
251 for address in ns.address:
252 try:
fbf925c8 253 network = db.lookup(address)
5118a4b8 254 except ValueError:
9f2f5d13 255 print(_("Invalid IP address: %s") % address, file=sys.stderr)
5118a4b8
MT
256
257 args = {
258 "address" : address,
fbf925c8 259 "network" : network,
5118a4b8
MT
260 }
261
262 # Nothing found?
fbf925c8 263 if not network:
9f2f5d13 264 print(_("Nothing found for %(address)s") % args, file=sys.stderr)
5118a4b8
MT
265 ret = 1
266 continue
267
fbf925c8
MT
268 print("%s:" % address)
269 print(format % (_("Network"), network))
5118a4b8 270
fbf925c8
MT
271 # Print country
272 if network.country_code:
072ca100
MT
273 country = db.get_country(network.country_code)
274
275 print(format % (
276 _("Country"),
277 country.name if country else network.country_code),
278 )
5118a4b8 279
fbf925c8
MT
280 # Print AS information
281 if network.asn:
282 autonomous_system = db.get_as(network.asn)
5118a4b8 283
fbf925c8
MT
284 print(format % (
285 _("Autonomous System"),
286 autonomous_system or "AS%s" % network.asn),
287 )
5118a4b8 288
ee83fe2e
MT
289 # Anonymous Proxy
290 if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
291 print(format % (
292 _("Anonymous Proxy"), _("yes"),
293 ))
294
295 # Satellite Provider
296 if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
297 print(format % (
298 _("Satellite Provider"), _("yes"),
299 ))
300
301 # Anycast
302 if network.has_flag(location.NETWORK_FLAG_ANYCAST):
303 print(format % (
304 _("Anycast"), _("yes"),
305 ))
306
5118a4b8
MT
307 return ret
308
a68a46f5
MT
309 def handle_dump(self, db, ns):
310 # Use output file or write to stdout
311 f = ns.output or sys.stdout
312
d6d06375
MT
313 # Format everything like this
314 format = "%-24s %s\n"
315
a68a46f5
MT
316 # Write metadata
317 f.write("#\n# Location Database Export\n#\n")
318
319 f.write("# Generated: %s\n" % time.strftime(
320 "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
321 ))
322
323 if db.vendor:
324 f.write("# Vendor: %s\n" % db.vendor)
325
326 if db.license:
327 f.write("# License: %s\n" % db.license)
328
329 f.write("#\n")
330
331 if db.description:
332 for line in db.description.splitlines():
cfbae90f
PM
333 line = "# %s" % line
334 f.write("%s\n" % line.rstrip())
a68a46f5
MT
335
336 f.write("#\n")
337
338 # Iterate over all ASes
339 for a in db.ases:
340 f.write("\n")
d6d06375
MT
341 f.write(format % ("aut-num:", "AS%s" % a.number))
342 f.write(format % ("name:", a.name))
343
344 flags = {
345 location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:",
346 location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
347 location.NETWORK_FLAG_ANYCAST : "is-anycast:",
348 }
a68a46f5
MT
349
350 # Iterate over all networks
351 for n in db.networks:
352 f.write("\n")
d6d06375 353 f.write(format % ("net:", n))
a68a46f5
MT
354
355 if n.country_code:
d6d06375 356 f.write(format % ("country:", n.country_code))
a68a46f5
MT
357
358 if n.asn:
03cd8096 359 f.write(format % ("aut-num:", n.asn))
d6d06375
MT
360
361 # Print all flags
362 for flag in flags:
363 if n.has_flag(flag):
364 f.write(format % (flags[flag], "yes"))
a68a46f5 365
2538ed9a 366 def handle_get_as(self, db, ns):
fadc1af0
MT
367 """
368 Gets information about Autonomous Systems
369 """
370 ret = 0
371
372 for asn in ns.asn:
373 try:
374 asn = int(asn)
375 except ValueError:
9f2f5d13 376 print(_("Invalid ASN: %s") % asn, file=sys.stderr)
fadc1af0
MT
377 ret = 1
378 continue
379
380 # Fetch AS from database
2538ed9a 381 a = db.get_as(asn)
fadc1af0
MT
382
383 # Nothing found
384 if not a:
9f2f5d13 385 print(_("Could not find AS%s") % asn, file=sys.stderr)
fadc1af0
MT
386 ret = 1
387 continue
388
389 print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
390
391 return ret
5118a4b8 392
2538ed9a 393 def handle_search_as(self, db, ns):
da3e360e
MT
394 for query in ns.query:
395 # Print all matches ASes
2538ed9a 396 for a in db.search_as(query):
da3e360e
MT
397 print(a)
398
a6f1e346 399 def handle_update(self, db, ns):
97769987 400 if ns.cron and db:
60c1ac03 401 now = time.time()
97769987
MT
402
403 if ns.cron == "daily":
404 delta = datetime.timedelta(days=1)
405 elif ns.cron == "weekly":
406 delta = datetime.timedelta(days=7)
407 elif ns.cron == "monthly":
408 delta = datetime.timedelta(days=30)
409
60c1ac03
PM
410 delta = delta.total_seconds()
411
97769987 412 # Check if the database has recently been updated
60c1ac03 413 if db.created_at >= (now - delta):
97769987 414 log.info(
60c1ac03 415 _("The database has been updated recently"),
97769987 416 )
a243587a 417 return 3
97769987 418
a6f1e346
MT
419 # Fetch the timestamp we need from DNS
420 t = location.discover_latest_version()
421
a6f1e346 422 # Check the version of the local database
0c74f6b1 423 if db and t and db.created_at >= t:
a6f1e346
MT
424 log.info("Already on the latest version")
425 return
426
427 # Download the database into the correct directory
428 tmpdir = os.path.dirname(ns.database)
429
430 # Create a downloader
431 d = location.downloader.Downloader()
432
433 # Try downloading a new database
434 try:
435 t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
436
437 # If no file could be downloaded, log a message
438 except FileNotFoundError as e:
439 log.error("Could not download a new database")
440 return 1
441
442 # If we have not received a new file, there is nothing to do
443 if not t:
444 return 3
445
446 # Move temporary file to destination
447 shutil.move(t.name, ns.database)
448
449 return 0
450
e99a7226 451 def handle_verify(self, db, ns):
a6f1e346
MT
452 # Verify the database
453 with open(ns.public_key, "r") as f:
454 if not db.verify(f):
455 log.error("Could not verify database")
456 return 1
457
458 # Success
459 log.debug("Database successfully verified")
460 return 0
461
4439e317
MT
462 def __get_output_formatter(self, ns):
463 try:
88ef7e9c 464 cls = location.export.formats[ns.format]
4439e317 465 except KeyError:
88ef7e9c 466 cls = location.export.OutputFormatter
4439e317 467
88ef7e9c 468 return cls
4439e317 469
fa9a3663
MT
470 def handle_list_countries(self, db, ns):
471 for country in db.countries:
472 line = [
473 country.code,
474 ]
475
476 if ns.show_continent:
477 line.append(country.continent_code)
478
479 if ns.show_name:
480 line.append(country.name)
481
482 # Format the output
483 line = " ".join(line)
484
485 # Print the output
486 print(line)
487
43154ed7 488 def handle_list_networks_by_as(self, db, ns):
88ef7e9c
MT
489 writer = self.__get_output_formatter(ns)
490
491 for asn in ns.asn:
492 f = writer(sys.stdout, prefix="AS%s" % asn)
493
494 # Print all matching networks
495 for n in db.search_networks(asn=asn, family=ns.family):
496 f.write(n)
497
498 f.finish()
43154ed7 499
ccc7ab4e 500 def handle_list_networks_by_cc(self, db, ns):
88ef7e9c
MT
501 writer = self.__get_output_formatter(ns)
502
503 for country_code in ns.country_code:
504 # Open standard output
505 f = writer(sys.stdout, prefix=country_code)
506
507 # Print all matching networks
508 for n in db.search_networks(country_code=country_code, family=ns.family):
509 f.write(n)
510
511 f.finish()
4439e317 512
bbdb2e0a
MT
513 def handle_list_networks_by_flags(self, db, ns):
514 flags = 0
515
516 if ns.anonymous_proxy:
517 flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
518
519 if ns.satellite_provider:
520 flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
521
522 if ns.anycast:
523 flags |= location.NETWORK_FLAG_ANYCAST
524
228d0e74
MT
525 if not flags:
526 raise ValueError(_("You must at least pass one flag"))
527
88ef7e9c
MT
528 writer = self.__get_output_formatter(ns)
529 f = writer(sys.stdout, prefix="custom")
530
531 for n in db.search_networks(flags=flags, family=ns.family):
532 f.write(n)
533
534 f.finish()
535
536 def handle_export(self, db, ns):
537 countries, asns = [], []
538
539 # Translate family
df072005
MT
540 if ns.family:
541 families = [ ns.family ]
88ef7e9c
MT
542 else:
543 families = [ socket.AF_INET6, socket.AF_INET ]
544
545 for object in ns.objects:
546 m = re.match("^AS(\d+)$", object)
547 if m:
548 object = int(m.group(1))
549
550 asns.append(object)
551
552 elif location.country_code_is_valid(object) \
553 or object in ("A1", "A2", "A3"):
554 countries.append(object)
555
556 else:
557 log.warning("Invalid argument: %s" % object)
558 continue
559
10fa313b 560 # Default to exporting all countries
88ef7e9c 561 if not countries and not asns:
10fa313b 562 countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
88ef7e9c
MT
563
564 # Select the output format
565 writer = self.__get_output_formatter(ns)
566
567 e = location.export.Exporter(db, writer)
568 e.export(ns.directory, countries=countries, asns=asns, families=families)
bbdb2e0a 569
ccc7ab4e 570
97769987
MT
571def format_timedelta(t):
572 s = []
573
574 if t.days:
575 s.append(
576 _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
577 )
578
579 hours = t.seconds // 3600
580 if hours:
581 s.append(
582 _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
583 )
584
585 minutes = (t.seconds % 3600) // 60
586 if minutes:
587 s.append(
588 _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
589 )
590
591 seconds = t.seconds % 60
592 if t.seconds:
593 s.append(
594 _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
595 )
596
597 if not s:
598 return _("Now")
599
600 return _("%s ago") % ", ".join(s)
601
5118a4b8
MT
602def main():
603 # Run the command line interface
604 c = CLI()
605 c.run()
606
607main()