]>
Commit | Line | Data |
---|---|---|
1f2c3ccb JS |
1 | #!/usr/bin/python3 |
2 | ############################################################################### | |
3 | # # | |
4 | # libloc - A library to determine the location of someone on the Internet # | |
5 | # # | |
6 | # Copyright (C) 2017-2021 IPFire Development Team <info@ipfire.org> # | |
7 | # # | |
8 | # This library is free software; you can redistribute it and/or # | |
9 | # modify it under the terms of the GNU Lesser General Public # | |
10 | # License as published by the Free Software Foundation; either # | |
11 | # version 2.1 of the License, or (at your option) any later version. # | |
12 | # # | |
13 | # This library is distributed in the hope that it will be useful, # | |
14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of # | |
15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # | |
16 | # Lesser General Public License for more details. # | |
17 | # # | |
18 | ############################################################################### | |
19 | ||
20 | import argparse | |
21 | import datetime | |
22 | import ipaddress | |
23 | import logging | |
24 | import os | |
25 | import re | |
26 | import shutil | |
27 | import socket | |
28 | import sys | |
29 | import time | |
30 | ||
31 | # Load our location module | |
32 | import location | |
33 | import location.downloader | |
34 | import location.export | |
35 | ||
36 | from location.i18n import _ | |
37 | ||
38 | # Setup logging | |
39 | log = logging.getLogger("location") | |
40 | ||
41 | # Output formatters | |
42 | ||
43 | class CLI(object): | |
44 | def parse_cli(self): | |
45 | parser = argparse.ArgumentParser( | |
46 | description=_("Location Database Command Line Interface"), | |
47 | ) | |
48 | subparsers = parser.add_subparsers() | |
49 | ||
50 | # Global configuration flags | |
51 | parser.add_argument("--debug", action="store_true", | |
52 | help=_("Enable debug output")) | |
53 | parser.add_argument("--quiet", action="store_true", | |
54 | help=_("Enable quiet mode")) | |
55 | ||
56 | # version | |
57 | parser.add_argument("--version", action="version", | |
58 | version="%(prog)s @VERSION@") | |
59 | ||
60 | # database | |
61 | parser.add_argument("--database", "-d", | |
aa9346d8 | 62 | default=location.DATABASE_PATH, help=_("Path to database"), |
1f2c3ccb JS |
63 | ) |
64 | ||
65 | # public key | |
66 | parser.add_argument("--public-key", "-k", | |
67 | default="@databasedir@/signing-key.pem", help=_("Public Signing Key"), | |
68 | ) | |
69 | ||
70 | # Show the database version | |
71 | version = subparsers.add_parser("version", | |
72 | help=_("Show database version")) | |
73 | version.set_defaults(func=self.handle_version) | |
74 | ||
75 | # lookup an IP address | |
76 | lookup = subparsers.add_parser("lookup", | |
77 | help=_("Lookup one or multiple IP addresses"), | |
78 | ) | |
79 | lookup.add_argument("address", nargs="+") | |
80 | lookup.set_defaults(func=self.handle_lookup) | |
81 | ||
82 | # Dump the whole database | |
83 | dump = subparsers.add_parser("dump", | |
84 | help=_("Dump the entire database"), | |
85 | ) | |
86 | dump.add_argument("output", nargs="?", type=argparse.FileType("w")) | |
87 | dump.set_defaults(func=self.handle_dump) | |
88 | ||
89 | # Update | |
90 | update = subparsers.add_parser("update", help=_("Update database")) | |
91 | update.add_argument("--cron", | |
92 | help=_("Update the library only once per interval"), | |
93 | choices=("daily", "weekly", "monthly"), | |
94 | ) | |
95 | update.set_defaults(func=self.handle_update) | |
96 | ||
97 | # Verify | |
98 | verify = subparsers.add_parser("verify", | |
99 | help=_("Verify the downloaded database")) | |
100 | verify.set_defaults(func=self.handle_verify) | |
101 | ||
102 | # Get AS | |
103 | get_as = subparsers.add_parser("get-as", | |
104 | help=_("Get information about one or multiple Autonomous Systems"), | |
105 | ) | |
106 | get_as.add_argument("asn", nargs="+") | |
107 | get_as.set_defaults(func=self.handle_get_as) | |
108 | ||
109 | # Search for AS | |
110 | search_as = subparsers.add_parser("search-as", | |
111 | help=_("Search for Autonomous Systems that match the string"), | |
112 | ) | |
113 | search_as.add_argument("query", nargs=1) | |
114 | search_as.set_defaults(func=self.handle_search_as) | |
115 | ||
116 | # List all networks in an AS | |
117 | list_networks_by_as = subparsers.add_parser("list-networks-by-as", | |
118 | help=_("Lists all networks in an AS"), | |
119 | ) | |
120 | list_networks_by_as.add_argument("asn", nargs=1, type=int) | |
121 | list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4")) | |
122 | list_networks_by_as.add_argument("--format", | |
123 | choices=location.export.formats.keys(), default="list") | |
124 | list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as) | |
125 | ||
126 | # List all networks in a country | |
127 | list_networks_by_cc = subparsers.add_parser("list-networks-by-cc", | |
128 | help=_("Lists all networks in a country"), | |
129 | ) | |
130 | list_networks_by_cc.add_argument("country_code", nargs=1) | |
131 | list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4")) | |
132 | list_networks_by_cc.add_argument("--format", | |
133 | choices=location.export.formats.keys(), default="list") | |
134 | list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc) | |
135 | ||
136 | # List all networks with flags | |
137 | list_networks_by_flags = subparsers.add_parser("list-networks-by-flags", | |
138 | help=_("Lists all networks with flags"), | |
139 | ) | |
140 | list_networks_by_flags.add_argument("--anonymous-proxy", | |
141 | action="store_true", help=_("Anonymous Proxies"), | |
142 | ) | |
143 | list_networks_by_flags.add_argument("--satellite-provider", | |
144 | action="store_true", help=_("Satellite Providers"), | |
145 | ) | |
146 | list_networks_by_flags.add_argument("--anycast", | |
147 | action="store_true", help=_("Anycasts"), | |
148 | ) | |
149 | list_networks_by_flags.add_argument("--drop", | |
150 | action="store_true", help=_("Hostile Networks safe to drop"), | |
151 | ) | |
152 | list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4")) | |
153 | list_networks_by_flags.add_argument("--format", | |
154 | choices=location.export.formats.keys(), default="list") | |
155 | list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags) | |
156 | ||
157 | # List bogons | |
158 | list_bogons = subparsers.add_parser("list-bogons", | |
159 | help=_("Lists all bogons"), | |
160 | ) | |
161 | list_bogons.add_argument("--family", choices=("ipv6", "ipv4")) | |
162 | list_bogons.add_argument("--format", | |
163 | choices=location.export.formats.keys(), default="list") | |
164 | list_bogons.set_defaults(func=self.handle_list_bogons) | |
165 | ||
166 | # List countries | |
167 | list_countries = subparsers.add_parser("list-countries", | |
168 | help=_("Lists all countries"), | |
169 | ) | |
170 | list_countries.add_argument("--show-name", | |
171 | action="store_true", help=_("Show the name of the country"), | |
172 | ) | |
173 | list_countries.add_argument("--show-continent", | |
174 | action="store_true", help=_("Show the continent"), | |
175 | ) | |
176 | list_countries.set_defaults(func=self.handle_list_countries) | |
177 | ||
178 | # Export | |
179 | export = subparsers.add_parser("export", | |
180 | help=_("Exports data in many formats to load it into packet filters"), | |
181 | ) | |
182 | export.add_argument("--format", help=_("Output format"), | |
183 | choices=location.export.formats.keys(), default="list") | |
184 | export.add_argument("--directory", help=_("Output directory")) | |
185 | export.add_argument("--family", | |
186 | help=_("Specify address family"), choices=("ipv6", "ipv4"), | |
187 | ) | |
188 | export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export")) | |
189 | export.set_defaults(func=self.handle_export) | |
190 | ||
191 | args = parser.parse_args() | |
192 | ||
193 | # Configure logging | |
194 | if args.debug: | |
195 | location.logger.set_level(logging.DEBUG) | |
196 | elif args.quiet: | |
197 | location.logger.set_level(logging.WARNING) | |
198 | ||
199 | # Print usage if no action was given | |
200 | if not "func" in args: | |
201 | parser.print_usage() | |
202 | sys.exit(2) | |
203 | ||
204 | return args | |
205 | ||
206 | def run(self): | |
207 | # Parse command line arguments | |
208 | args = self.parse_cli() | |
209 | ||
210 | # Open database | |
211 | try: | |
212 | db = location.Database(args.database) | |
213 | except FileNotFoundError as e: | |
214 | # Allow continuing without a database | |
215 | if args.func == self.handle_update: | |
216 | db = None | |
217 | ||
218 | else: | |
219 | sys.stderr.write("location: Could not open database %s: %s\n" \ | |
220 | % (args.database, e)) | |
221 | sys.exit(1) | |
222 | ||
223 | # Translate family (if present) | |
224 | if "family" in args: | |
225 | if args.family == "ipv6": | |
226 | args.family = socket.AF_INET6 | |
227 | elif args.family == "ipv4": | |
228 | args.family = socket.AF_INET | |
229 | else: | |
230 | args.family = 0 | |
231 | ||
232 | # Call function | |
233 | try: | |
234 | ret = args.func(db, args) | |
235 | ||
236 | # Catch invalid inputs | |
237 | except ValueError as e: | |
238 | sys.stderr.write("%s\n" % e) | |
239 | ret = 2 | |
240 | ||
241 | # Catch any other exceptions | |
242 | except Exception as e: | |
243 | sys.stderr.write("%s\n" % e) | |
244 | ret = 1 | |
245 | ||
246 | # Return with exit code | |
247 | if ret: | |
248 | sys.exit(ret) | |
249 | ||
250 | # Otherwise just exit | |
251 | sys.exit(0) | |
252 | ||
253 | def handle_version(self, db, ns): | |
254 | """ | |
255 | Print the version of the database | |
256 | """ | |
257 | t = time.strftime( | |
258 | "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at), | |
259 | ) | |
260 | ||
261 | print(t) | |
262 | ||
263 | def handle_lookup(self, db, ns): | |
264 | ret = 0 | |
265 | ||
266 | format = " %-24s: %s" | |
267 | ||
268 | for address in ns.address: | |
269 | try: | |
270 | network = db.lookup(address) | |
271 | except ValueError: | |
272 | print(_("Invalid IP address: %s") % address, file=sys.stderr) | |
273 | return 2 | |
274 | ||
275 | args = { | |
276 | "address" : address, | |
277 | "network" : network, | |
278 | } | |
279 | ||
280 | # Nothing found? | |
281 | if not network: | |
282 | print(_("Nothing found for %(address)s") % args, file=sys.stderr) | |
283 | ret = 1 | |
284 | continue | |
285 | ||
286 | print("%s:" % address) | |
287 | print(format % (_("Network"), network)) | |
288 | ||
289 | # Print country | |
290 | if network.country_code: | |
291 | country = db.get_country(network.country_code) | |
292 | ||
293 | print(format % ( | |
294 | _("Country"), | |
295 | country.name if country else network.country_code), | |
296 | ) | |
297 | ||
298 | # Print AS information | |
299 | if network.asn: | |
300 | autonomous_system = db.get_as(network.asn) | |
301 | ||
302 | print(format % ( | |
303 | _("Autonomous System"), | |
304 | autonomous_system or "AS%s" % network.asn), | |
305 | ) | |
306 | ||
307 | # Anonymous Proxy | |
308 | if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY): | |
309 | print(format % ( | |
310 | _("Anonymous Proxy"), _("yes"), | |
311 | )) | |
312 | ||
313 | # Satellite Provider | |
314 | if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER): | |
315 | print(format % ( | |
316 | _("Satellite Provider"), _("yes"), | |
317 | )) | |
318 | ||
319 | # Anycast | |
320 | if network.has_flag(location.NETWORK_FLAG_ANYCAST): | |
321 | print(format % ( | |
322 | _("Anycast"), _("yes"), | |
323 | )) | |
324 | ||
325 | # Hostile Network | |
326 | if network.has_flag(location.NETWORK_FLAG_DROP): | |
327 | print(format % ( | |
328 | _("Hostile Network safe to drop"), _("yes"), | |
329 | )) | |
330 | ||
331 | return ret | |
332 | ||
333 | def handle_dump(self, db, ns): | |
334 | # Use output file or write to stdout | |
335 | f = ns.output or sys.stdout | |
336 | ||
337 | # Format everything like this | |
338 | format = "%-24s %s\n" | |
339 | ||
340 | # Write metadata | |
341 | f.write("#\n# Location Database Export\n#\n") | |
342 | ||
343 | f.write("# Generated: %s\n" % time.strftime( | |
344 | "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at), | |
345 | )) | |
346 | ||
347 | if db.vendor: | |
348 | f.write("# Vendor: %s\n" % db.vendor) | |
349 | ||
350 | if db.license: | |
351 | f.write("# License: %s\n" % db.license) | |
352 | ||
353 | f.write("#\n") | |
354 | ||
355 | if db.description: | |
356 | for line in db.description.splitlines(): | |
357 | line = "# %s" % line | |
358 | f.write("%s\n" % line.rstrip()) | |
359 | ||
360 | f.write("#\n") | |
361 | ||
362 | # Iterate over all ASes | |
363 | for a in db.ases: | |
364 | f.write("\n") | |
365 | f.write(format % ("aut-num:", "AS%s" % a.number)) | |
366 | f.write(format % ("name:", a.name)) | |
367 | ||
368 | flags = { | |
369 | location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:", | |
370 | location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:", | |
371 | location.NETWORK_FLAG_ANYCAST : "is-anycast:", | |
372 | location.NETWORK_FLAG_DROP : "drop:", | |
373 | } | |
374 | ||
375 | # Iterate over all networks | |
376 | for n in db.networks: | |
377 | f.write("\n") | |
378 | f.write(format % ("net:", n)) | |
379 | ||
380 | if n.country_code: | |
381 | f.write(format % ("country:", n.country_code)) | |
382 | ||
383 | if n.asn: | |
384 | f.write(format % ("aut-num:", n.asn)) | |
385 | ||
386 | # Print all flags | |
387 | for flag in flags: | |
388 | if n.has_flag(flag): | |
389 | f.write(format % (flags[flag], "yes")) | |
390 | ||
391 | def handle_get_as(self, db, ns): | |
392 | """ | |
393 | Gets information about Autonomous Systems | |
394 | """ | |
395 | ret = 0 | |
396 | ||
397 | for asn in ns.asn: | |
398 | try: | |
399 | asn = int(asn) | |
400 | except ValueError: | |
401 | print(_("Invalid ASN: %s") % asn, file=sys.stderr) | |
402 | ret = 1 | |
403 | continue | |
404 | ||
405 | # Fetch AS from database | |
406 | a = db.get_as(asn) | |
407 | ||
408 | # Nothing found | |
409 | if not a: | |
410 | print(_("Could not find AS%s") % asn, file=sys.stderr) | |
411 | ret = 1 | |
412 | continue | |
413 | ||
414 | print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name }) | |
415 | ||
416 | return ret | |
417 | ||
418 | def handle_search_as(self, db, ns): | |
419 | for query in ns.query: | |
420 | # Print all matches ASes | |
421 | for a in db.search_as(query): | |
422 | print(a) | |
423 | ||
424 | def handle_update(self, db, ns): | |
425 | if ns.cron and db: | |
426 | now = time.time() | |
427 | ||
428 | if ns.cron == "daily": | |
429 | delta = datetime.timedelta(days=1) | |
430 | elif ns.cron == "weekly": | |
431 | delta = datetime.timedelta(days=7) | |
432 | elif ns.cron == "monthly": | |
433 | delta = datetime.timedelta(days=30) | |
434 | ||
435 | delta = delta.total_seconds() | |
436 | ||
437 | # Check if the database has recently been updated | |
438 | if db.created_at >= (now - delta): | |
439 | log.info( | |
440 | _("The database has been updated recently"), | |
441 | ) | |
442 | return 3 | |
443 | ||
444 | # Fetch the timestamp we need from DNS | |
445 | t = location.discover_latest_version() | |
446 | ||
447 | # Check the version of the local database | |
448 | if db and t and db.created_at >= t: | |
449 | log.info("Already on the latest version") | |
450 | return | |
451 | ||
452 | # Download the database into the correct directory | |
453 | tmpdir = os.path.dirname(ns.database) | |
454 | ||
455 | # Create a downloader | |
456 | d = location.downloader.Downloader() | |
457 | ||
458 | # Try downloading a new database | |
459 | try: | |
460 | t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir) | |
461 | ||
462 | # If no file could be downloaded, log a message | |
463 | except FileNotFoundError as e: | |
464 | log.error("Could not download a new database") | |
465 | return 1 | |
466 | ||
467 | # If we have not received a new file, there is nothing to do | |
468 | if not t: | |
469 | return 3 | |
470 | ||
471 | # Move temporary file to destination | |
472 | shutil.move(t.name, ns.database) | |
473 | ||
474 | return 0 | |
475 | ||
476 | def handle_verify(self, db, ns): | |
477 | # Verify the database | |
478 | with open(ns.public_key, "r") as f: | |
479 | if not db.verify(f): | |
480 | log.error("Could not verify database") | |
481 | return 1 | |
482 | ||
483 | # Success | |
7880c134 | 484 | log.info("Database successfully verified") |
1f2c3ccb JS |
485 | return 0 |
486 | ||
487 | def __get_output_formatter(self, ns): | |
488 | try: | |
489 | cls = location.export.formats[ns.format] | |
490 | except KeyError: | |
491 | cls = location.export.OutputFormatter | |
492 | ||
493 | return cls | |
494 | ||
495 | def handle_list_countries(self, db, ns): | |
496 | for country in db.countries: | |
497 | line = [ | |
498 | country.code, | |
499 | ] | |
500 | ||
501 | if ns.show_continent: | |
502 | line.append(country.continent_code) | |
503 | ||
504 | if ns.show_name: | |
505 | line.append(country.name) | |
506 | ||
507 | # Format the output | |
508 | line = " ".join(line) | |
509 | ||
510 | # Print the output | |
511 | print(line) | |
512 | ||
513 | def handle_list_networks_by_as(self, db, ns): | |
514 | writer = self.__get_output_formatter(ns) | |
515 | ||
516 | for asn in ns.asn: | |
517 | f = writer("AS%s" % asn, f=sys.stdout) | |
518 | ||
519 | # Print all matching networks | |
520 | for n in db.search_networks(asns=[asn], family=ns.family): | |
521 | f.write(n) | |
522 | ||
523 | f.finish() | |
524 | ||
525 | def handle_list_networks_by_cc(self, db, ns): | |
526 | writer = self.__get_output_formatter(ns) | |
527 | ||
528 | for country_code in ns.country_code: | |
529 | # Open standard output | |
530 | f = writer(country_code, f=sys.stdout) | |
531 | ||
532 | # Print all matching networks | |
533 | for n in db.search_networks(country_codes=[country_code], family=ns.family): | |
534 | f.write(n) | |
535 | ||
536 | f.finish() | |
537 | ||
538 | def handle_list_networks_by_flags(self, db, ns): | |
539 | flags = 0 | |
540 | ||
541 | if ns.anonymous_proxy: | |
542 | flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY | |
543 | ||
544 | if ns.satellite_provider: | |
545 | flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER | |
546 | ||
547 | if ns.anycast: | |
548 | flags |= location.NETWORK_FLAG_ANYCAST | |
549 | ||
550 | if ns.drop: | |
551 | flags |= location.NETWORK_FLAG_DROP | |
552 | ||
553 | if not flags: | |
554 | raise ValueError(_("You must at least pass one flag")) | |
555 | ||
556 | writer = self.__get_output_formatter(ns) | |
557 | f = writer("custom", f=sys.stdout) | |
558 | ||
559 | for n in db.search_networks(flags=flags, family=ns.family): | |
560 | f.write(n) | |
561 | ||
562 | f.finish() | |
563 | ||
564 | def handle_list_bogons(self, db, ns): | |
565 | writer = self.__get_output_formatter(ns) | |
566 | f = writer("bogons", f=sys.stdout) | |
567 | ||
568 | for n in db.list_bogons(family=ns.family): | |
569 | f.write(n) | |
570 | ||
571 | f.finish() | |
572 | ||
573 | def handle_export(self, db, ns): | |
574 | countries, asns = [], [] | |
575 | ||
576 | # Translate family | |
577 | if ns.family: | |
578 | families = [ ns.family ] | |
579 | else: | |
580 | families = [ socket.AF_INET6, socket.AF_INET ] | |
581 | ||
582 | for object in ns.objects: | |
583 | m = re.match("^AS(\d+)$", object) | |
584 | if m: | |
585 | object = int(m.group(1)) | |
586 | ||
587 | asns.append(object) | |
588 | ||
589 | elif location.country_code_is_valid(object) \ | |
590 | or object in ("A1", "A2", "A3", "XD"): | |
591 | countries.append(object) | |
592 | ||
593 | else: | |
594 | log.warning("Invalid argument: %s" % object) | |
595 | continue | |
596 | ||
597 | # Default to exporting all countries | |
598 | if not countries and not asns: | |
599 | countries = ["A1", "A2", "A3", "XD"] + [country.code for country in db.countries] | |
600 | ||
601 | # Select the output format | |
602 | writer = self.__get_output_formatter(ns) | |
603 | ||
604 | e = location.export.Exporter(db, writer) | |
605 | e.export(ns.directory, countries=countries, asns=asns, families=families) | |
606 | ||
607 | ||
608 | def format_timedelta(t): | |
609 | s = [] | |
610 | ||
611 | if t.days: | |
612 | s.append( | |
613 | _("One Day", "%(days)s Days", t.days) % { "days" : t.days, } | |
614 | ) | |
615 | ||
616 | hours = t.seconds // 3600 | |
617 | if hours: | |
618 | s.append( | |
619 | _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, } | |
620 | ) | |
621 | ||
622 | minutes = (t.seconds % 3600) // 60 | |
623 | if minutes: | |
624 | s.append( | |
625 | _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, } | |
626 | ) | |
627 | ||
628 | seconds = t.seconds % 60 | |
629 | if t.seconds: | |
630 | s.append( | |
631 | _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, } | |
632 | ) | |
633 | ||
634 | if not s: | |
635 | return _("Now") | |
636 | ||
637 | return _("%s ago") % ", ".join(s) | |
638 | ||
639 | def main(): | |
640 | # Run the command line interface | |
641 | c = CLI() | |
642 | c.run() | |
643 | ||
644 | main() |