]> git.ipfire.org Git - location/libloc.git/blob - src/python/location-importer.in
projects / location / libloc.git / blob
? search:


ea6d48527c8139760e38c6ee6c407c6cd8376009
[location/libloc.git] / src / python / location-importer.in
1 #!/usr/bin/python3
2 ###############################################################################
3 # #
4 # libloc - A library to determine the location of someone on the Internet #
5 # #
6 # Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
7 # #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
12 # #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
17 # #
18 ###############################################################################
19
20 import argparse
21 import ipaddress
22 import logging
23 import math
24 import re
25 import sys
26 import telnetlib
27
28 # Load our location module
29 import location
30 import location.database
31 import location.importer
32 from location.i18n import _
33
34 # Initialise logging
35 log = logging.getLogger("location.importer")
36 log.propagate = 1
37
38 class CLI(object):
39 def parse_cli(self):
40 parser = argparse.ArgumentParser(
41 description=_("Location Importer Command Line Interface"),
42 )
43 subparsers = parser.add_subparsers()
44
45 # Global configuration flags
46 parser.add_argument("--debug", action="store_true",
47 help=_("Enable debug output"))
48 parser.add_argument("--quiet", action="store_true",
49 help=_("Enable quiet mode"))
50
51 # version
52 parser.add_argument("--version", action="version",
53 version="%(prog)s @VERSION@")
54
55 # Database
56 parser.add_argument("--database-host", required=True,
57 help=_("Database Hostname"), metavar=_("HOST"))
58 parser.add_argument("--database-name", required=True,
59 help=_("Database Name"), metavar=_("NAME"))
60 parser.add_argument("--database-username", required=True,
61 help=_("Database Username"), metavar=_("USERNAME"))
62 parser.add_argument("--database-password", required=True,
63 help=_("Database Password"), metavar=_("PASSWORD"))
64
65 # Write Database
66 write = subparsers.add_parser("write", help=_("Write database to file"))
67 write.set_defaults(func=self.handle_write)
68 write.add_argument("file", nargs=1, help=_("Database File"))
69 write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
70 write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
71 write.add_argument("--description", nargs="?", help=_("Sets a description"))
72 write.add_argument("--license", nargs="?", help=_("Sets the license"))
73
74 # Update WHOIS
75 update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
76 update_whois.set_defaults(func=self.handle_update_whois)
77
78 # Update announcements
79 update_announcements = subparsers.add_parser("update-announcements",
80 help=_("Update BGP Annoucements"))
81 update_announcements.set_defaults(func=self.handle_update_announcements)
82 update_announcements.add_argument("server", nargs=1,
83 help=_("Route Server to connect to"), metavar=_("SERVER"))
84
85 # Update overrides
86 update_overrides = subparsers.add_parser("update-overrides",
87 help=_("Update overrides"),
88 )
89 update_overrides.add_argument(
90 "files", nargs="+", help=_("Files to import"),
91 )
92 update_overrides.set_defaults(func=self.handle_update_overrides)
93
94 args = parser.parse_args()
95
96 # Configure logging
97 if args.debug:
98 location.logger.set_level(logging.DEBUG)
99 elif args.quiet:
100 location.logger.set_level(logging.WARNING)
101
102 # Print usage if no action was given
103 if not "func" in args:
104 parser.print_usage()
105 sys.exit(2)
106
107 return args
108
109 def run(self):
110 # Parse command line arguments
111 args = self.parse_cli()
112
113 # Initialise database
114 self.db = self._setup_database(args)
115
116 # Call function
117 ret = args.func(args)
118
119 # Return with exit code
120 if ret:
121 sys.exit(ret)
122
123 # Otherwise just exit
124 sys.exit(0)
125
126 def _setup_database(self, ns):
127 """
128 Initialise the database
129 """
130 # Connect to database
131 db = location.database.Connection(
132 host=ns.database_host, database=ns.database_name,
133 user=ns.database_username, password=ns.database_password,
134 )
135
136 with db.transaction():
137 db.execute("""
138 -- announcements
139 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
140 first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
141 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
142 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
143 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
144
145 -- autnums
146 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
147 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
148
149 -- networks
150 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
151 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
152 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
153
154 -- overrides
155 CREATE TABLE IF NOT EXISTS autnum_overrides(
156 number bigint NOT NULL,
157 name text,
158 is_anonymous_proxy boolean DEFAULT FALSE,
159 is_satellite_provider boolean DEFAULT FALSE,
160 is_anycast boolean DEFAULT FALSE
161 );
162 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
163 ON autnum_overrides(number);
164
165 CREATE TABLE IF NOT EXISTS network_overrides(
166 network inet NOT NULL,
167 country text,
168 is_anonymous_proxy boolean DEFAULT FALSE,
169 is_satellite_provider boolean DEFAULT FALSE,
170 is_anycast boolean DEFAULT FALSE
171 );
172 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
173 ON network_overrides(network);
174 """)
175
176 return db
177
178 def handle_write(self, ns):
179 """
180 Compiles a database in libloc format out of what is in the database
181 """
182 # Allocate a writer
183 writer = location.Writer(ns.signing_key)
184
185 # Set all metadata
186 if ns.vendor:
187 writer.vendor = ns.vendor
188
189 if ns.description:
190 writer.description = ns.description
191
192 if ns.license:
193 writer.license = ns.license
194
195 # Add all Autonomous Systems
196 log.info("Writing Autonomous Systems...")
197
198 # Select all ASes with a name
199 rows = self.db.query("""
200 SELECT
201 autnums.number AS number,
202 COALESCE(
203 (SELECT overrides.name FROM autnum_overrides overrides
204 WHERE overrides.number = autnums.number),
205 autnums.name
206 ) AS name
207 FROM autnums
208 WHERE name <> %s ORDER BY number
209 """, "")
210
211 for row in rows:
212 a = writer.add_as(row.number)
213 a.name = row.name
214
215 # Add all networks
216 log.info("Writing networks...")
217
218 # Select all known networks
219 rows = self.db.query("""
220 SELECT
221 DISTINCT ON (announcements.network)
222 announcements.network AS network,
223 announcements.autnum AS autnum,
224 networks.country AS country,
225
226 -- Must be part of returned values for ORDER BY clause
227 masklen(networks.network) AS sort,
228
229 -- Flags
230 COALESCE(
231 (
232 SELECT is_anonymous_proxy FROM network_overrides overrides
233 WHERE announcements.network <<= overrides.network
234 ORDER BY masklen(overrides.network) DESC
235 LIMIT 1
236 ),
237 (
238 SELECT is_anonymous_proxy FROM autnum_overrides overrides
239 WHERE announcements.autnum = overrides.number
240 )
241 ) AS is_anonymous_proxy,
242 COALESCE(
243 (
244 SELECT is_satellite_provider FROM network_overrides overrides
245 WHERE announcements.network <<= overrides.network
246 ORDER BY masklen(overrides.network) DESC
247 LIMIT 1
248 ),
249 (
250 SELECT is_satellite_provider FROM autnum_overrides overrides
251 WHERE announcements.autnum = overrides.number
252 )
253 ) AS is_satellite_provider,
254 COALESCE(
255 (
256 SELECT is_anycast FROM network_overrides overrides
257 WHERE announcements.network <<= overrides.network
258 ORDER BY masklen(overrides.network) DESC
259 LIMIT 1
260 ),
261 (
262 SELECT is_anycast FROM autnum_overrides overrides
263 WHERE announcements.autnum = overrides.number
264 )
265 ) AS is_anycast
266 FROM announcements
267 LEFT JOIN networks ON announcements.network <<= networks.network
268 ORDER BY announcements.network, sort DESC
269 """)
270
271 for row in rows:
272 network = writer.add_network(row.network)
273
274 # Save AS & country
275 network.asn, network.country_code = row.autnum, row.country
276
277 # Set flags
278 if row.is_anonymous_proxy:
279 network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
280
281 if row.is_satellite_provider:
282 network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
283
284 if row.is_anycast:
285 network.set_flag(location.NETWORK_FLAG_ANYCAST)
286
287 # Write everything to file
288 log.info("Writing database to file...")
289 for file in ns.file:
290 writer.write(file)
291
292 def handle_update_whois(self, ns):
293 downloader = location.importer.Downloader()
294
295 # Download all sources
296 with self.db.transaction():
297 # Create some temporary tables to store parsed data
298 self.db.execute("""
299 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
300 ON COMMIT DROP;
301 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
302
303 CREATE TEMPORARY TABLE _organizations(handle text, name text)
304 ON COMMIT DROP;
305 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
306 """)
307
308 for source in location.importer.WHOIS_SOURCES:
309 with downloader.request(source, return_blocks=True) as f:
310 for block in f:
311 self._parse_block(block)
312
313 self.db.execute("""
314 INSERT INTO autnums(number, name)
315 SELECT _autnums.number, _organizations.name FROM _autnums
316 LEFT JOIN _organizations ON _autnums.organization = _organizations.handle
317 ON CONFLICT (number) DO UPDATE SET name = excluded.name;
318 """)
319
320 # Download all extended sources
321 for source in location.importer.EXTENDED_SOURCES:
322 with self.db.transaction():
323 # Download data
324 with downloader.request(source) as f:
325 for line in f:
326 self._parse_line(line)
327
328 def _parse_block(self, block):
329 # Get first line to find out what type of block this is
330 line = block[0]
331
332 # aut-num
333 if line.startswith("aut-num:"):
334 return self._parse_autnum_block(block)
335
336 # organisation
337 elif line.startswith("organisation:"):
338 return self._parse_org_block(block)
339
340 def _parse_autnum_block(self, block):
341 autnum = {}
342 for line in block:
343 # Split line
344 key, val = split_line(line)
345
346 if key == "aut-num":
347 m = re.match(r"^(AS|as)(\d+)", val)
348 if m:
349 autnum["asn"] = m.group(2)
350
351 elif key == "org":
352 autnum[key] = val
353
354 # Skip empty objects
355 if not autnum:
356 return
357
358 # Insert into database
359 self.db.execute("INSERT INTO _autnums(number, organization) \
360 VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
361 organization = excluded.organization",
362 autnum.get("asn"), autnum.get("org"),
363 )
364
365 def _parse_org_block(self, block):
366 org = {}
367 for line in block:
368 # Split line
369 key, val = split_line(line)
370
371 if key in ("organisation", "org-name"):
372 org[key] = val
373
374 # Skip empty objects
375 if not org:
376 return
377
378 self.db.execute("INSERT INTO _organizations(handle, name) \
379 VALUES(%s, %s) ON CONFLICT (handle) DO \
380 UPDATE SET name = excluded.name",
381 org.get("organisation"), org.get("org-name"),
382 )
383
384 def _parse_line(self, line):
385 # Skip version line
386 if line.startswith("2"):
387 return
388
389 # Skip comments
390 if line.startswith("#"):
391 return
392
393 try:
394 registry, country_code, type, line = line.split("|", 3)
395 except:
396 log.warning("Could not parse line: %s" % line)
397 return
398
399 # Skip any lines that are for stats only
400 if country_code == "*":
401 return
402
403 if type in ("ipv6", "ipv4"):
404 return self._parse_ip_line(country_code, type, line)
405
406 def _parse_ip_line(self, country, type, line):
407 try:
408 address, prefix, date, status, organization = line.split("|")
409 except ValueError:
410 organization = None
411
412 # Try parsing the line without organization
413 try:
414 address, prefix, date, status = line.split("|")
415 except ValueError:
416 log.warning("Unhandled line format: %s" % line)
417 return
418
419 # Skip anything that isn't properly assigned
420 if not status in ("assigned", "allocated"):
421 return
422
423 # Cast prefix into an integer
424 try:
425 prefix = int(prefix)
426 except:
427 log.warning("Invalid prefix: %s" % prefix)
428 return
429
430 # Fix prefix length for IPv4
431 if type == "ipv4":
432 prefix = 32 - int(math.log(prefix, 2))
433
434 # Try to parse the address
435 try:
436 network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
437 except ValueError:
438 log.warning("Invalid IP address: %s" % address)
439 return
440
441 self.db.execute("INSERT INTO networks(network, country) \
442 VALUES(%s, %s) ON CONFLICT (network) DO \
443 UPDATE SET country = excluded.country",
444 "%s" % network, country,
445 )
446
447 def handle_update_announcements(self, ns):
448 server = ns.server[0]
449
450 # Pre-compile regular expression for routes
451 #route = re.compile(b"^\*>?\s[\si]?([^\s]+)[.\s]*?(\d+)\si$", re.MULTILINE)
452 route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
453
454 with telnetlib.Telnet(server) as t:
455 # Enable debug mode
456 #if ns.debug:
457 # t.set_debuglevel(10)
458
459 # Wait for console greeting
460 greeting = t.read_until(b"> ")
461 log.debug(greeting.decode())
462
463 # Disable pagination
464 t.write(b"terminal length 0\n")
465
466 # Wait for the prompt to return
467 t.read_until(b"> ")
468
469 # Fetch the routing tables
470 with self.db.transaction():
471 for protocol in ("ipv6", "ipv4"):
472 log.info("Requesting %s routing table" % protocol)
473
474 # Request the full unicast routing table
475 t.write(b"show bgp %s unicast\n" % protocol.encode())
476
477 # Read entire header which ends with "Path"
478 t.read_until(b"Path\r\n")
479
480 while True:
481 # Try reading a full entry
482 # Those might be broken across multiple lines but ends with i
483 line = t.read_until(b"i\r\n", timeout=5)
484 if not line:
485 break
486
487 # Show line for debugging
488 #log.debug(repr(line))
489
490 # Try finding a route in here
491 m = route.match(line)
492 if m:
493 network, autnum = m.groups()
494
495 # Convert network to string
496 network = network.decode()
497
498 # Append /24 for IPv4 addresses
499 if not "/" in network and not ":" in network:
500 network = "%s/24" % network
501
502 # Convert AS number to integer
503 autnum = int(autnum)
504
505 log.info("Found announcement for %s by %s" % (network, autnum))
506
507 self.db.execute("INSERT INTO announcements(network, autnum) \
508 VALUES(%s, %s) ON CONFLICT (network) DO \
509 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
510 network, autnum,
511 )
512
513 log.info("Finished reading the %s routing table" % protocol)
514
515 # Purge anything we never want here
516 self.db.execute("""
517 -- Delete default routes
518 DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
519
520 -- Delete anything that is not global unicast address space
521 DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
522
523 -- DELETE "current network" address space
524 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
525
526 -- DELETE local loopback address space
527 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
528
529 -- DELETE RFC 1918 address space
530 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
531 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
532 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
533
534 -- DELETE test, benchmark and documentation address space
535 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
536 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
537 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
538 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
539 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
540
541 -- DELETE CGNAT address space (RFC 6598)
542 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
543
544 -- DELETE link local address space
545 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
546
547 -- DELETE IPv6 to IPv4 (6to4) address space
548 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
549
550 -- DELETE multicast and reserved address space
551 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
552 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
553
554 -- Delete networks that are too small to be in the global routing table
555 DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
556 DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
557
558 -- Delete any non-public or reserved ASNs
559 DELETE FROM announcements WHERE NOT (
560 (autnum >= 1 AND autnum <= 23455)
561 OR
562 (autnum >= 23457 AND autnum <= 64495)
563 OR
564 (autnum >= 131072 AND autnum <= 4199999999)
565 );
566
567 -- Delete everything that we have not seen for 14 days
568 DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
569 """)
570
571 def handle_update_overrides(self, ns):
572 with self.db.transaction():
573 # Drop all data that we have
574 self.db.execute("""
575 TRUNCATE TABLE autnum_overrides;
576 TRUNCATE TABLE network_overrides;
577 """)
578
579 for file in ns.files:
580 log.info("Reading %s..." % file)
581
582 with open(file, "rb") as f:
583 for type, block in location.importer.read_blocks(f):
584 if type == "net":
585 network = block.get("net")
586 # Try to parse and normalise the network
587 try:
588 network = ipaddress.ip_network(network, strict=False)
589 except ValueError as e:
590 log.warning("Invalid IP network: %s: %s" % (network, e))
591 continue
592
593 # Prevent that we overwrite all networks
594 if network.prefixlen == 0:
595 log.warning("Skipping %s: You cannot overwrite default" % network)
596 continue
597
598 self.db.execute("""
599 INSERT INTO network_overrides(
600 network,
601 country,
602 is_anonymous_proxy,
603 is_satellite_provider,
604 is_anycast
605 ) VALUES (%s, %s, %s, %s, %s)
606 ON CONFLICT (network) DO NOTHING""",
607 "%s" % network,
608 block.get("country"),
609 block.get("is-anonymous-proxy") == "yes",
610 block.get("is-satellite-provider") == "yes",
611 block.get("is-anycast") == "yes",
612 )
613
614 elif type == "aut-num":
615 autnum = block.get("aut-num")
616
617 # Check if AS number begins with "AS"
618 if not autnum.startswith("AS"):
619 log.warning("Invalid AS number: %s" % autnum)
620 continue
621
622 # Strip "AS"
623 autnum = autnum[2:]
624
625 self.db.execute("""
626 INSERT INTO autnum_overrides(
627 number,
628 name,
629 is_anonymous_proxy,
630 is_satellite_provider,
631 is_anycast
632 ) VALUES(%s, %s, %s, %s, %s)
633 ON CONFLICT DO NOTHING""",
634 autnum, block.get("name"),
635 block.get("is-anonymous-proxy") == "yes",
636 block.get("is-satellite-provider") == "yes",
637 block.get("is-anycast") == "yes",
638 )
639
640 else:
641 log.warning("Unsupport type: %s" % type)
642
643
644 def split_line(line):
645 key, colon, val = line.partition(":")
646
647 # Strip any excess space
648 key = key.strip()
649 val = val.strip()
650
651 return key, val
652
653 def main():
654 # Run the command line interface
655 c = CLI()
656 c.run()
657
658 main()
A library to determine the location of someone on the Internet