]> git.ipfire.org Git - location/libloc.git/blob - src/python/location-importer.in
projects / location / libloc.git / blob
? search:


5cfa09efe95e9c1f78425572e2e1a7039147d3bd
[location/libloc.git] / src / python / location-importer.in
1 #!/usr/bin/python3
2 ###############################################################################
3 # #
4 # libloc - A library to determine the location of someone on the Internet #
5 # #
6 # Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
7 # #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
12 # #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
17 # #
18 ###############################################################################
19
20 import argparse
21 import ipaddress
22 import logging
23 import math
24 import re
25 import sys
26 import telnetlib
27
28 # Load our location module
29 import location
30 import location.database
31 import location.importer
32 from location.i18n import _
33
34 # Initialise logging
35 log = logging.getLogger("location.importer")
36 log.propagate = 1
37
38 class CLI(object):
39 def parse_cli(self):
40 parser = argparse.ArgumentParser(
41 description=_("Location Importer Command Line Interface"),
42 )
43 subparsers = parser.add_subparsers()
44
45 # Global configuration flags
46 parser.add_argument("--debug", action="store_true",
47 help=_("Enable debug output"))
48
49 # version
50 parser.add_argument("--version", action="version",
51 version="%(prog)s @VERSION@")
52
53 # Database
54 parser.add_argument("--database-host", required=True,
55 help=_("Database Hostname"), metavar=_("HOST"))
56 parser.add_argument("--database-name", required=True,
57 help=_("Database Name"), metavar=_("NAME"))
58 parser.add_argument("--database-username", required=True,
59 help=_("Database Username"), metavar=_("USERNAME"))
60 parser.add_argument("--database-password", required=True,
61 help=_("Database Password"), metavar=_("PASSWORD"))
62
63 # Write Database
64 write = subparsers.add_parser("write", help=_("Write database to file"))
65 write.set_defaults(func=self.handle_write)
66 write.add_argument("file", nargs=1, help=_("Database File"))
67 write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
68 write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
69 write.add_argument("--description", nargs="?", help=_("Sets a description"))
70 write.add_argument("--license", nargs="?", help=_("Sets the license"))
71
72 # Update WHOIS
73 update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
74 update_whois.set_defaults(func=self.handle_update_whois)
75
76 # Update announcements
77 update_announcements = subparsers.add_parser("update-announcements",
78 help=_("Update BGP Annoucements"))
79 update_announcements.set_defaults(func=self.handle_update_announcements)
80 update_announcements.add_argument("server", nargs=1,
81 help=_("Route Server to connect to"), metavar=_("SERVER"))
82
83 # Update overrides
84 update_overrides = subparsers.add_parser("update-overrides",
85 help=_("Update overrides"),
86 )
87 update_overrides.add_argument(
88 "files", nargs="+", help=_("Files to import"),
89 )
90 update_overrides.set_defaults(func=self.handle_update_overrides)
91
92 args = parser.parse_args()
93
94 # Enable debug logging
95 if args.debug:
96 log.setLevel(logging.DEBUG)
97
98 # Print usage if no action was given
99 if not "func" in args:
100 parser.print_usage()
101 sys.exit(2)
102
103 return args
104
105 def run(self):
106 # Parse command line arguments
107 args = self.parse_cli()
108
109 # Initialise database
110 self.db = self._setup_database(args)
111
112 # Call function
113 ret = args.func(args)
114
115 # Return with exit code
116 if ret:
117 sys.exit(ret)
118
119 # Otherwise just exit
120 sys.exit(0)
121
122 def _setup_database(self, ns):
123 """
124 Initialise the database
125 """
126 # Connect to database
127 db = location.database.Connection(
128 host=ns.database_host, database=ns.database_name,
129 user=ns.database_username, password=ns.database_password,
130 )
131
132 with db.transaction():
133 db.execute("""
134 -- announcements
135 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
136 first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
137 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
138 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
139 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
140
141 -- autnums
142 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
143 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
144
145 -- networks
146 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
147 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
148 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
149
150 -- overrides
151 CREATE TABLE IF NOT EXISTS autnum_overrides(
152 number bigint NOT NULL,
153 name text,
154 is_anonymous_proxy boolean DEFAULT FALSE,
155 is_satellite_provider boolean DEFAULT FALSE,
156 is_anycast boolean DEFAULT FALSE
157 );
158 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
159 ON autnum_overrides(number);
160
161 CREATE TABLE IF NOT EXISTS network_overrides(
162 network inet NOT NULL,
163 country text,
164 is_anonymous_proxy boolean DEFAULT FALSE,
165 is_satellite_provider boolean DEFAULT FALSE,
166 is_anycast boolean DEFAULT FALSE
167 );
168 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
169 ON network_overrides(network);
170 """)
171
172 return db
173
174 def handle_write(self, ns):
175 """
176 Compiles a database in libloc format out of what is in the database
177 """
178 print(ns)
179
180 # Allocate a writer
181 writer = location.Writer(ns.signing_key)
182
183 # Set all metadata
184 if ns.vendor:
185 writer.vendor = ns.vendor
186
187 if ns.description:
188 writer.description = ns.description
189
190 if ns.license:
191 writer.license = ns.license
192
193 # Add all Autonomous Systems
194 log.info("Writing Autonomous Systems...")
195
196 # Select all ASes with a name
197 rows = self.db.query("""
198 SELECT
199 autnums.number AS number,
200 COALESCE(
201 (SELECT overrides.name FROM autnum_overrides overrides
202 WHERE overrides.number = autnums.number),
203 autnums.name
204 ) AS name
205 FROM autnums
206 WHERE name <> %s ORDER BY number
207 """, "")
208
209 for row in rows:
210 a = writer.add_as(row.number)
211 a.name = row.name
212
213 # Add all networks
214 log.info("Writing networks...")
215
216 # Select all known networks
217 rows = self.db.query("""
218 SELECT
219 announcements.network AS network,
220 announcements.autnum AS autnum,
221 (
222 SELECT networks.country FROM networks
223 WHERE announcements.network <<= networks.network
224 ORDER BY masklen(networks.network) DESC
225 LIMIT 1
226 ) AS country,
227
228 -- Flags
229 COALESCE(
230 (
231 SELECT is_anonymous_proxy FROM network_overrides overrides
232 WHERE announcements.network <<= overrides.network
233 ORDER BY masklen(overrides.network) DESC
234 LIMIT 1
235 ),
236 (
237 SELECT is_anonymous_proxy FROM autnum_overrides overrides
238 WHERE announcements.autnum = overrides.number
239 )
240 ) AS is_anonymous_proxy,
241 COALESCE(
242 (
243 SELECT is_satellite_provider FROM network_overrides overrides
244 WHERE announcements.network <<= overrides.network
245 ORDER BY masklen(overrides.network) DESC
246 LIMIT 1
247 ),
248 (
249 SELECT is_satellite_provider FROM autnum_overrides overrides
250 WHERE announcements.autnum = overrides.number
251 )
252 ) AS is_satellite_provider,
253 COALESCE(
254 (
255 SELECT is_anycast FROM network_overrides overrides
256 WHERE announcements.network <<= overrides.network
257 ORDER BY masklen(overrides.network) DESC
258 LIMIT 1
259 ),
260 (
261 SELECT is_anycast FROM autnum_overrides overrides
262 WHERE announcements.autnum = overrides.number
263 )
264 ) AS is_anycast
265 FROM announcements
266 """)
267
268 for row in rows:
269 network = writer.add_network(row.network)
270
271 # Save AS & country
272 network.asn, network.country_code = row.autnum, row.country
273
274 # Set flags
275 if row.is_anonymous_proxy:
276 network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
277
278 if row.is_satellite_provider:
279 network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
280
281 if row.is_anycast:
282 network.set_flag(location.NETWORK_FLAG_ANYCAST)
283
284 # Write everything to file
285 log.info("Writing database to file...")
286 for file in ns.file:
287 writer.write(file)
288
289 def handle_update_whois(self, ns):
290 downloader = location.importer.Downloader()
291
292 # Download all sources
293 with self.db.transaction():
294 # Create some temporary tables to store parsed data
295 self.db.execute("""
296 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
297 ON COMMIT DROP;
298 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
299
300 CREATE TEMPORARY TABLE _organizations(handle text, name text)
301 ON COMMIT DROP;
302 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
303 """)
304
305 for source in location.importer.WHOIS_SOURCES:
306 with downloader.request(source, return_blocks=True) as f:
307 for block in f:
308 self._parse_block(block)
309
310 self.db.execute("""
311 INSERT INTO autnums(number, name)
312 SELECT _autnums.number, _organizations.name FROM _autnums
313 LEFT JOIN _organizations ON _autnums.organization = _organizations.handle
314 ON CONFLICT (number) DO UPDATE SET name = excluded.name;
315 """)
316
317 # Download all extended sources
318 for source in location.importer.EXTENDED_SOURCES:
319 with self.db.transaction():
320 # Download data
321 with downloader.request(source) as f:
322 for line in f:
323 self._parse_line(line)
324
325 def _parse_block(self, block):
326 # Get first line to find out what type of block this is
327 line = block[0]
328
329 # aut-num
330 if line.startswith("aut-num:"):
331 return self._parse_autnum_block(block)
332
333 # organisation
334 elif line.startswith("organisation:"):
335 return self._parse_org_block(block)
336
337 def _parse_autnum_block(self, block):
338 autnum = {}
339 for line in block:
340 # Split line
341 key, val = split_line(line)
342
343 if key == "aut-num":
344 m = re.match(r"^(AS|as)(\d+)", val)
345 if m:
346 autnum["asn"] = m.group(2)
347
348 elif key == "org":
349 autnum[key] = val
350
351 # Skip empty objects
352 if not autnum:
353 return
354
355 # Insert into database
356 self.db.execute("INSERT INTO _autnums(number, organization) \
357 VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
358 organization = excluded.organization",
359 autnum.get("asn"), autnum.get("org"),
360 )
361
362 def _parse_org_block(self, block):
363 org = {}
364 for line in block:
365 # Split line
366 key, val = split_line(line)
367
368 if key in ("organisation", "org-name"):
369 org[key] = val
370
371 # Skip empty objects
372 if not org:
373 return
374
375 self.db.execute("INSERT INTO _organizations(handle, name) \
376 VALUES(%s, %s) ON CONFLICT (handle) DO \
377 UPDATE SET name = excluded.name",
378 org.get("organisation"), org.get("org-name"),
379 )
380
381 def _parse_line(self, line):
382 # Skip version line
383 if line.startswith("2"):
384 return
385
386 # Skip comments
387 if line.startswith("#"):
388 return
389
390 try:
391 registry, country_code, type, line = line.split("|", 3)
392 except:
393 log.warning("Could not parse line: %s" % line)
394 return
395
396 # Skip any lines that are for stats only
397 if country_code == "*":
398 return
399
400 if type in ("ipv6", "ipv4"):
401 return self._parse_ip_line(country_code, type, line)
402
403 def _parse_ip_line(self, country, type, line):
404 try:
405 address, prefix, date, status, organization = line.split("|")
406 except ValueError:
407 organization = None
408
409 # Try parsing the line without organization
410 try:
411 address, prefix, date, status = line.split("|")
412 except ValueError:
413 log.warning("Unhandled line format: %s" % line)
414 return
415
416 # Skip anything that isn't properly assigned
417 if not status in ("assigned", "allocated"):
418 return
419
420 # Cast prefix into an integer
421 try:
422 prefix = int(prefix)
423 except:
424 log.warning("Invalid prefix: %s" % prefix)
425
426 # Fix prefix length for IPv4
427 if type == "ipv4":
428 prefix = 32 - int(math.log(prefix, 2))
429
430 # Try to parse the address
431 try:
432 network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
433 except ValueError:
434 log.warning("Invalid IP address: %s" % address)
435 return
436
437 self.db.execute("INSERT INTO networks(network, country) \
438 VALUES(%s, %s) ON CONFLICT (network) DO \
439 UPDATE SET country = excluded.country",
440 "%s" % network, country,
441 )
442
443 def handle_update_announcements(self, ns):
444 server = ns.server[0]
445
446 # Pre-compile regular expression for routes
447 #route = re.compile(b"^\*>?\s[\si]?([^\s]+)[.\s]*?(\d+)\si$", re.MULTILINE)
448 route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
449
450 with telnetlib.Telnet(server) as t:
451 # Enable debug mode
452 #if ns.debug:
453 # t.set_debuglevel(10)
454
455 # Wait for console greeting
456 greeting = t.read_until(b"> ")
457 log.debug(greeting.decode())
458
459 # Disable pagination
460 t.write(b"terminal length 0\n")
461
462 # Wait for the prompt to return
463 t.read_until(b"> ")
464
465 # Fetch the routing tables
466 with self.db.transaction():
467 for protocol in ("ipv6", "ipv4"):
468 log.info("Requesting %s routing table" % protocol)
469
470 # Request the full unicast routing table
471 t.write(b"show bgp %s unicast\n" % protocol.encode())
472
473 # Read entire header which ends with "Path"
474 t.read_until(b"Path\r\n")
475
476 while True:
477 # Try reading a full entry
478 # Those might be broken across multiple lines but ends with i
479 line = t.read_until(b"i\r\n", timeout=5)
480 if not line:
481 break
482
483 # Show line for debugging
484 #log.debug(repr(line))
485
486 # Try finding a route in here
487 m = route.match(line)
488 if m:
489 network, autnum = m.groups()
490
491 # Convert network to string
492 network = network.decode()
493
494 # Append /24 for IPv4 addresses
495 if not "/" in network and not ":" in network:
496 network = "%s/24" % network
497
498 # Convert AS number to integer
499 autnum = int(autnum)
500
501 log.info("Found announcement for %s by %s" % (network, autnum))
502
503 self.db.execute("INSERT INTO announcements(network, autnum) \
504 VALUES(%s, %s) ON CONFLICT (network) DO \
505 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
506 network, autnum,
507 )
508
509 log.info("Finished reading the %s routing table" % protocol)
510
511 # Purge anything we never want here
512 self.db.execute("""
513 -- Delete default routes
514 DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
515
516 -- Delete anything that is not global unicast address space
517 DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
518
519 -- DELETE local loopback address space
520 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
521
522 -- DELETE RFC1918 address space
523 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
524 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
525 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
526
527 -- DELETE test, benchmark and documentation address space
528 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
529 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
530 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
531 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
532 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
533
534 -- DELETE CGNAT address space (RFC 6598)
535 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
536
537 -- DELETE link local address space
538 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
539
540 -- DELETE IPv6 to IPv4 (6to4) address space
541 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
542
543 -- Delete networks that are too small to be in the global routing table
544 DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
545 DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
546
547 -- Delete any non-public or reserved ASNs
548 DELETE FROM announcements WHERE NOT (
549 (autnum >= 1 AND autnum <= 23455)
550 OR
551 (autnum >= 23457 AND autnum <= 64495)
552 OR
553 (autnum >= 131072 AND autnum <= 4199999999)
554 );
555
556 -- Delete everything that we have not seen for 14 days
557 DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
558 """)
559
560 def handle_update_overrides(self, ns):
561 with self.db.transaction():
562 # Drop all data that we have
563 self.db.execute("""
564 TRUNCATE TABLE autnum_overrides;
565 TRUNCATE TABLE network_overrides;
566 """)
567
568 for file in ns.files:
569 log.info("Reading %s..." % file)
570
571 with open(file, "rb") as f:
572 for type, block in location.importer.read_blocks(f):
573 if type == "net":
574 network = block.get("net")
575 # Try to parse and normalise the network
576 try:
577 network = ipaddress.ip_network(network, strict=False)
578 except ValueError as e:
579 log.warning("Invalid IP network: %s: %s" % (network, e))
580 continue
581
582 self.db.execute("""
583 INSERT INTO network_overrides(
584 network,
585 country,
586 is_anonymous_proxy,
587 is_satellite_provider,
588 is_anycast
589 ) VALUES (%s, %s, %s, %s)
590 ON CONFLICT (network) DO NOTHING""",
591 "%s" % network,
592 block.get("country"),
593 block.get("is-anonymous-proxy") == "yes",
594 block.get("is-satellite-provider") == "yes",
595 block.get("is-anycast") == "yes",
596 )
597
598 elif type == "autnum":
599 autnum = block.get("autnum")
600
601 # Check if AS number begins with "AS"
602 if not autnum.startswith("AS"):
603 log.warning("Invalid AS number: %s" % autnum)
604 continue
605
606 # Strip "AS"
607 autnum = autnum[2:]
608
609 self.db.execute("""
610 INSERT INTO autnum_overrides(
611 number,
612 name,
613 is_anonymous_proxy,
614 is_satellite_provider,
615 is_anycast
616 ) VALUES(%s, %s, %s, %s, %s)
617 ON CONFLICT DO NOTHING""",
618 autnum, block.get("name"),
619 block.get("is-anonymous-proxy") == "yes",
620 block.get("is-satellite-provider") == "yes",
621 block.get("is-anycast") == "yes",
622 )
623
624 else:
625 log.warning("Unsupport type: %s" % type)
626
627
628 def split_line(line):
629 key, colon, val = line.partition(":")
630
631 # Strip any excess space
632 key = key.strip()
633 val = val.strip()
634
635 return key, val
636
637 def main():
638 # Run the command line interface
639 c = CLI()
640 c.run()
641
642 main()
A library to determine the location of someone on the Internet