]>
Commit | Line | Data |
---|---|---|
1 | #!/usr/bin/python3 | |
2 | ############################################################################### | |
3 | # # | |
4 | # libloc - A library to determine the location of someone on the Internet # | |
5 | # # | |
6 | # Copyright (C) 2020 IPFire Development Team <info@ipfire.org> # | |
7 | # # | |
8 | # This library is free software; you can redistribute it and/or # | |
9 | # modify it under the terms of the GNU Lesser General Public # | |
10 | # License as published by the Free Software Foundation; either # | |
11 | # version 2.1 of the License, or (at your option) any later version. # | |
12 | # # | |
13 | # This library is distributed in the hope that it will be useful, # | |
14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of # | |
15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # | |
16 | # Lesser General Public License for more details. # | |
17 | # # | |
18 | ############################################################################### | |
19 | ||
20 | import argparse | |
21 | import ipaddress | |
22 | import logging | |
23 | import math | |
24 | import re | |
25 | import socket | |
26 | import sys | |
27 | import telnetlib | |
28 | ||
29 | # Load our location module | |
30 | import location | |
31 | import location.database | |
32 | import location.importer | |
33 | from location.i18n import _ | |
34 | ||
35 | # Initialise logging | |
36 | log = logging.getLogger("location.importer") | |
37 | log.propagate = 1 | |
38 | ||
39 | class CLI(object): | |
40 | def parse_cli(self): | |
41 | parser = argparse.ArgumentParser( | |
42 | description=_("Location Importer Command Line Interface"), | |
43 | ) | |
44 | subparsers = parser.add_subparsers() | |
45 | ||
46 | # Global configuration flags | |
47 | parser.add_argument("--debug", action="store_true", | |
48 | help=_("Enable debug output")) | |
49 | parser.add_argument("--quiet", action="store_true", | |
50 | help=_("Enable quiet mode")) | |
51 | ||
52 | # version | |
53 | parser.add_argument("--version", action="version", | |
54 | version="%(prog)s @VERSION@") | |
55 | ||
56 | # Database | |
57 | parser.add_argument("--database-host", required=True, | |
58 | help=_("Database Hostname"), metavar=_("HOST")) | |
59 | parser.add_argument("--database-name", required=True, | |
60 | help=_("Database Name"), metavar=_("NAME")) | |
61 | parser.add_argument("--database-username", required=True, | |
62 | help=_("Database Username"), metavar=_("USERNAME")) | |
63 | parser.add_argument("--database-password", required=True, | |
64 | help=_("Database Password"), metavar=_("PASSWORD")) | |
65 | ||
66 | # Write Database | |
67 | write = subparsers.add_parser("write", help=_("Write database to file")) | |
68 | write.set_defaults(func=self.handle_write) | |
69 | write.add_argument("file", nargs=1, help=_("Database File")) | |
70 | write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key")) | |
71 | write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key")) | |
72 | write.add_argument("--vendor", nargs="?", help=_("Sets the vendor")) | |
73 | write.add_argument("--description", nargs="?", help=_("Sets a description")) | |
74 | write.add_argument("--license", nargs="?", help=_("Sets the license")) | |
75 | write.add_argument("--version", type=int, help=_("Database Format Version")) | |
76 | ||
77 | # Update WHOIS | |
78 | update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information")) | |
79 | update_whois.set_defaults(func=self.handle_update_whois) | |
80 | ||
81 | # Update announcements | |
82 | update_announcements = subparsers.add_parser("update-announcements", | |
83 | help=_("Update BGP Annoucements")) | |
84 | update_announcements.set_defaults(func=self.handle_update_announcements) | |
85 | update_announcements.add_argument("server", nargs=1, | |
86 | help=_("Route Server to connect to"), metavar=_("SERVER")) | |
87 | ||
88 | # Update overrides | |
89 | update_overrides = subparsers.add_parser("update-overrides", | |
90 | help=_("Update overrides"), | |
91 | ) | |
92 | update_overrides.add_argument( | |
93 | "files", nargs="+", help=_("Files to import"), | |
94 | ) | |
95 | update_overrides.set_defaults(func=self.handle_update_overrides) | |
96 | ||
97 | # Import countries | |
98 | import_countries = subparsers.add_parser("import-countries", | |
99 | help=_("Import countries"), | |
100 | ) | |
101 | import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"), | |
102 | help=_("File to import")) | |
103 | import_countries.set_defaults(func=self.handle_import_countries) | |
104 | ||
105 | args = parser.parse_args() | |
106 | ||
107 | # Configure logging | |
108 | if args.debug: | |
109 | location.logger.set_level(logging.DEBUG) | |
110 | elif args.quiet: | |
111 | location.logger.set_level(logging.WARNING) | |
112 | ||
113 | # Print usage if no action was given | |
114 | if not "func" in args: | |
115 | parser.print_usage() | |
116 | sys.exit(2) | |
117 | ||
118 | return args | |
119 | ||
120 | def run(self): | |
121 | # Parse command line arguments | |
122 | args = self.parse_cli() | |
123 | ||
124 | # Initialise database | |
125 | self.db = self._setup_database(args) | |
126 | ||
127 | # Call function | |
128 | ret = args.func(args) | |
129 | ||
130 | # Return with exit code | |
131 | if ret: | |
132 | sys.exit(ret) | |
133 | ||
134 | # Otherwise just exit | |
135 | sys.exit(0) | |
136 | ||
137 | def _setup_database(self, ns): | |
138 | """ | |
139 | Initialise the database | |
140 | """ | |
141 | # Connect to database | |
142 | db = location.database.Connection( | |
143 | host=ns.database_host, database=ns.database_name, | |
144 | user=ns.database_username, password=ns.database_password, | |
145 | ) | |
146 | ||
147 | with db.transaction(): | |
148 | db.execute(""" | |
149 | -- announcements | |
150 | CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint, | |
151 | first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP, | |
152 | last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP); | |
153 | CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network); | |
154 | CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network)); | |
155 | CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops); | |
156 | ||
157 | -- autnums | |
158 | CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL); | |
159 | CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number); | |
160 | ||
161 | -- countries | |
162 | CREATE TABLE IF NOT EXISTS countries( | |
163 | country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL); | |
164 | CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code); | |
165 | ||
166 | -- networks | |
167 | CREATE TABLE IF NOT EXISTS networks(network inet, country text); | |
168 | CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); | |
169 | CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network)); | |
170 | CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); | |
171 | ||
172 | -- overrides | |
173 | CREATE TABLE IF NOT EXISTS autnum_overrides( | |
174 | number bigint NOT NULL, | |
175 | name text, | |
176 | country text, | |
177 | is_anonymous_proxy boolean, | |
178 | is_satellite_provider boolean, | |
179 | is_anycast boolean | |
180 | ); | |
181 | CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number | |
182 | ON autnum_overrides(number); | |
183 | ||
184 | CREATE TABLE IF NOT EXISTS network_overrides( | |
185 | network inet NOT NULL, | |
186 | country text, | |
187 | is_anonymous_proxy boolean, | |
188 | is_satellite_provider boolean, | |
189 | is_anycast boolean | |
190 | ); | |
191 | CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network | |
192 | ON network_overrides(network); | |
193 | CREATE INDEX IF NOT EXISTS network_overrides_search | |
194 | ON network_overrides USING GIST(network inet_ops); | |
195 | """) | |
196 | ||
197 | return db | |
198 | ||
199 | def handle_write(self, ns): | |
200 | """ | |
201 | Compiles a database in libloc format out of what is in the database | |
202 | """ | |
203 | # Allocate a writer | |
204 | writer = location.Writer(ns.signing_key, ns.backup_signing_key) | |
205 | ||
206 | # Set all metadata | |
207 | if ns.vendor: | |
208 | writer.vendor = ns.vendor | |
209 | ||
210 | if ns.description: | |
211 | writer.description = ns.description | |
212 | ||
213 | if ns.license: | |
214 | writer.license = ns.license | |
215 | ||
216 | # Add all Autonomous Systems | |
217 | log.info("Writing Autonomous Systems...") | |
218 | ||
219 | # Select all ASes with a name | |
220 | rows = self.db.query(""" | |
221 | SELECT | |
222 | autnums.number AS number, | |
223 | COALESCE( | |
224 | (SELECT overrides.name FROM autnum_overrides overrides | |
225 | WHERE overrides.number = autnums.number), | |
226 | autnums.name | |
227 | ) AS name | |
228 | FROM autnums | |
229 | WHERE name <> %s ORDER BY number | |
230 | """, "") | |
231 | ||
232 | for row in rows: | |
233 | a = writer.add_as(row.number) | |
234 | a.name = row.name | |
235 | ||
236 | # Add all networks | |
237 | log.info("Writing networks...") | |
238 | ||
239 | # Select all known networks | |
240 | rows = self.db.query(""" | |
241 | -- Return a list of those networks enriched with all | |
242 | -- other information that we store in the database | |
243 | SELECT | |
244 | DISTINCT ON (network) | |
245 | network, | |
246 | autnum, | |
247 | ||
248 | -- Country | |
249 | COALESCE( | |
250 | ( | |
251 | SELECT country FROM network_overrides overrides | |
252 | WHERE networks.network <<= overrides.network | |
253 | ORDER BY masklen(overrides.network) DESC | |
254 | LIMIT 1 | |
255 | ), | |
256 | ( | |
257 | SELECT country FROM autnum_overrides overrides | |
258 | WHERE networks.autnum = overrides.number | |
259 | ), | |
260 | networks.country | |
261 | ) AS country, | |
262 | ||
263 | -- Flags | |
264 | COALESCE( | |
265 | ( | |
266 | SELECT is_anonymous_proxy FROM network_overrides overrides | |
267 | WHERE networks.network <<= overrides.network | |
268 | ORDER BY masklen(overrides.network) DESC | |
269 | LIMIT 1 | |
270 | ), | |
271 | ( | |
272 | SELECT is_anonymous_proxy FROM autnum_overrides overrides | |
273 | WHERE networks.autnum = overrides.number | |
274 | ), | |
275 | FALSE | |
276 | ) AS is_anonymous_proxy, | |
277 | COALESCE( | |
278 | ( | |
279 | SELECT is_satellite_provider FROM network_overrides overrides | |
280 | WHERE networks.network <<= overrides.network | |
281 | ORDER BY masklen(overrides.network) DESC | |
282 | LIMIT 1 | |
283 | ), | |
284 | ( | |
285 | SELECT is_satellite_provider FROM autnum_overrides overrides | |
286 | WHERE networks.autnum = overrides.number | |
287 | ), | |
288 | FALSE | |
289 | ) AS is_satellite_provider, | |
290 | COALESCE( | |
291 | ( | |
292 | SELECT is_anycast FROM network_overrides overrides | |
293 | WHERE networks.network <<= overrides.network | |
294 | ORDER BY masklen(overrides.network) DESC | |
295 | LIMIT 1 | |
296 | ), | |
297 | ( | |
298 | SELECT is_anycast FROM autnum_overrides overrides | |
299 | WHERE networks.autnum = overrides.number | |
300 | ), | |
301 | FALSE | |
302 | ) AS is_anycast | |
303 | FROM ( | |
304 | SELECT | |
305 | known_networks.network AS network, | |
306 | announcements.autnum AS autnum, | |
307 | networks.country AS country, | |
308 | ||
309 | -- Must be part of returned values for ORDER BY clause | |
310 | masklen(announcements.network) AS sort_a, | |
311 | masklen(networks.network) AS sort_b | |
312 | FROM ( | |
313 | SELECT network FROM announcements | |
314 | UNION ALL | |
315 | SELECT network FROM networks | |
316 | UNION ALL | |
317 | SELECT network FROM network_overrides | |
318 | ) known_networks | |
319 | LEFT JOIN | |
320 | announcements ON known_networks.network <<= announcements.network | |
321 | LEFT JOIN | |
322 | networks ON known_networks.network <<= networks.network | |
323 | ORDER BY | |
324 | known_networks.network, | |
325 | sort_a DESC, | |
326 | sort_b DESC | |
327 | ) networks | |
328 | """) | |
329 | ||
330 | for row in rows: | |
331 | network = writer.add_network(row.network) | |
332 | ||
333 | # Save country | |
334 | if row.country: | |
335 | network.country_code = row.country | |
336 | ||
337 | # Save ASN | |
338 | if row.autnum: | |
339 | network.asn = row.autnum | |
340 | ||
341 | # Set flags | |
342 | if row.is_anonymous_proxy: | |
343 | network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY) | |
344 | ||
345 | if row.is_satellite_provider: | |
346 | network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER) | |
347 | ||
348 | if row.is_anycast: | |
349 | network.set_flag(location.NETWORK_FLAG_ANYCAST) | |
350 | ||
351 | # Add all countries | |
352 | log.info("Writing countries...") | |
353 | rows = self.db.query("SELECT * FROM countries ORDER BY country_code") | |
354 | ||
355 | for row in rows: | |
356 | c = writer.add_country(row.country_code) | |
357 | c.continent_code = row.continent_code | |
358 | c.name = row.name | |
359 | ||
360 | # Write everything to file | |
361 | log.info("Writing database to file...") | |
362 | for file in ns.file: | |
363 | writer.write(file) | |
364 | ||
365 | def handle_update_whois(self, ns): | |
366 | downloader = location.importer.Downloader() | |
367 | ||
368 | # Download all sources | |
369 | with self.db.transaction(): | |
370 | # Create some temporary tables to store parsed data | |
371 | self.db.execute(""" | |
372 | CREATE TEMPORARY TABLE _autnums(number integer, organization text) | |
373 | ON COMMIT DROP; | |
374 | CREATE UNIQUE INDEX _autnums_number ON _autnums(number); | |
375 | ||
376 | CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL) | |
377 | ON COMMIT DROP; | |
378 | CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); | |
379 | ||
380 | CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL) | |
381 | ON COMMIT DROP; | |
382 | CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network)); | |
383 | CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network); | |
384 | """) | |
385 | ||
386 | # Remove all previously imported content | |
387 | self.db.execute(""" | |
388 | TRUNCATE TABLE networks; | |
389 | """) | |
390 | ||
391 | for source in location.importer.WHOIS_SOURCES: | |
392 | with downloader.request(source, return_blocks=True) as f: | |
393 | for block in f: | |
394 | self._parse_block(block) | |
395 | ||
396 | # Process all parsed networks from every RIR we happen to have access to, | |
397 | # insert the largest network chunks into the networks table immediately... | |
398 | families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)") | |
399 | ||
400 | for family in (row.family for row in families): | |
401 | smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family) | |
402 | ||
403 | self.db.execute("INSERT INTO networks(network, country) \ | |
404 | SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family) | |
405 | ||
406 | # ... determine any other prefixes for this network family, ... | |
407 | prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \ | |
408 | WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family) | |
409 | ||
410 | # ... and insert networks with this prefix in case they provide additional | |
411 | # information (i. e. subnet of a larger chunk with a different country) | |
412 | for prefix in (row.prefix for row in prefixes): | |
413 | self.db.execute(""" | |
414 | WITH candidates AS ( | |
415 | SELECT | |
416 | _rirdata.network, | |
417 | _rirdata.country | |
418 | FROM | |
419 | _rirdata | |
420 | WHERE | |
421 | family(_rirdata.network) = %s | |
422 | AND | |
423 | masklen(_rirdata.network) = %s | |
424 | ), | |
425 | filtered AS ( | |
426 | SELECT | |
427 | DISTINCT ON (c.network) | |
428 | c.network, | |
429 | c.country, | |
430 | masklen(networks.network), | |
431 | networks.country AS parent_country | |
432 | FROM | |
433 | candidates c | |
434 | LEFT JOIN | |
435 | networks | |
436 | ON | |
437 | c.network << networks.network | |
438 | ORDER BY | |
439 | c.network, | |
440 | masklen(networks.network) DESC NULLS LAST | |
441 | ) | |
442 | INSERT INTO | |
443 | networks(network, country) | |
444 | SELECT | |
445 | network, | |
446 | country | |
447 | FROM | |
448 | filtered | |
449 | WHERE | |
450 | parent_country IS NULL | |
451 | OR | |
452 | country <> parent_country | |
453 | ON CONFLICT DO NOTHING""", | |
454 | family, prefix, | |
455 | ) | |
456 | ||
457 | self.db.execute(""" | |
458 | INSERT INTO autnums(number, name) | |
459 | SELECT _autnums.number, _organizations.name FROM _autnums | |
460 | JOIN _organizations ON _autnums.organization = _organizations.handle | |
461 | ON CONFLICT (number) DO UPDATE SET name = excluded.name; | |
462 | """) | |
463 | ||
464 | # Download all extended sources | |
465 | for source in location.importer.EXTENDED_SOURCES: | |
466 | with self.db.transaction(): | |
467 | # Download data | |
468 | with downloader.request(source) as f: | |
469 | for line in f: | |
470 | self._parse_line(line) | |
471 | ||
472 | def _check_parsed_network(self, network): | |
473 | """ | |
474 | Assistive function to detect and subsequently sort out parsed | |
475 | networks from RIR data (both Whois and so-called "extended sources"), | |
476 | which are or have... | |
477 | ||
478 | (a) not globally routable (RFC 1918 space, et al.) | |
479 | (b) covering a too large chunk of the IP address space (prefix length | |
480 | is < 7 for IPv4 networks, and < 10 for IPv6) | |
481 | (c) "0.0.0.0" or "::" as a network address | |
482 | (d) are too small for being publicly announced (we have decided not to | |
483 | process them at the moment, as they significantly enlarge our | |
484 | database without providing very helpful additional information) | |
485 | ||
486 | This unfortunately is necessary due to brain-dead clutter across | |
487 | various RIR databases, causing mismatches and eventually disruptions. | |
488 | ||
489 | We will return False in case a network is not suitable for adding | |
490 | it to our database, and True otherwise. | |
491 | """ | |
492 | ||
493 | if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)): | |
494 | return False | |
495 | ||
496 | if not network.is_global: | |
497 | log.warning("Skipping non-globally routable network: %s" % network) | |
498 | return False | |
499 | ||
500 | if network.version == 4: | |
501 | if network.prefixlen < 7: | |
502 | log.warning("Skipping too big IP chunk: %s" % network) | |
503 | return False | |
504 | ||
505 | if network.prefixlen > 24: | |
506 | log.debug("Skipping network too small to be publicly announced: %s" % network) | |
507 | return False | |
508 | ||
509 | if str(network.network_address) == "0.0.0.0": | |
510 | log.warning("Skipping network based on 0.0.0.0: %s" % network) | |
511 | return False | |
512 | ||
513 | elif network.version == 6: | |
514 | if network.prefixlen < 10: | |
515 | log.warning("Skipping too big IP chunk: %s" % network) | |
516 | return False | |
517 | ||
518 | if network.prefixlen > 48: | |
519 | log.debug("Skipping network too small to be publicly announced: %s" % network) | |
520 | return False | |
521 | ||
522 | if str(network.network_address) == "::": | |
523 | log.warning("Skipping network based on '::': %s" % network) | |
524 | return False | |
525 | ||
526 | else: | |
527 | # This should not happen... | |
528 | log.warning("Skipping network of unknown family, this should not happen: %s" % network) | |
529 | return False | |
530 | ||
531 | # In case we have made it here, the network is considered to | |
532 | # be suitable for libloc consumption... | |
533 | return True | |
534 | ||
535 | def _parse_block(self, block): | |
536 | # Get first line to find out what type of block this is | |
537 | line = block[0] | |
538 | ||
539 | # aut-num | |
540 | if line.startswith("aut-num:"): | |
541 | return self._parse_autnum_block(block) | |
542 | ||
543 | # inetnum | |
544 | if line.startswith("inet6num:") or line.startswith("inetnum:"): | |
545 | return self._parse_inetnum_block(block) | |
546 | ||
547 | # organisation | |
548 | elif line.startswith("organisation:"): | |
549 | return self._parse_org_block(block) | |
550 | ||
551 | def _parse_autnum_block(self, block): | |
552 | autnum = {} | |
553 | for line in block: | |
554 | # Split line | |
555 | key, val = split_line(line) | |
556 | ||
557 | if key == "aut-num": | |
558 | m = re.match(r"^(AS|as)(\d+)", val) | |
559 | if m: | |
560 | autnum["asn"] = m.group(2) | |
561 | ||
562 | elif key == "org": | |
563 | autnum[key] = val.upper() | |
564 | ||
565 | # Skip empty objects | |
566 | if not autnum: | |
567 | return | |
568 | ||
569 | # Insert into database | |
570 | self.db.execute("INSERT INTO _autnums(number, organization) \ | |
571 | VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \ | |
572 | organization = excluded.organization", | |
573 | autnum.get("asn"), autnum.get("org"), | |
574 | ) | |
575 | ||
576 | def _parse_inetnum_block(self, block): | |
577 | log.debug("Parsing inetnum block:") | |
578 | ||
579 | inetnum = {} | |
580 | for line in block: | |
581 | log.debug(line) | |
582 | ||
583 | # Split line | |
584 | key, val = split_line(line) | |
585 | ||
586 | # Filter any inetnum records which are only referring to IP space | |
587 | # not managed by that specific RIR... | |
588 | if key == "netname": | |
589 | if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()): | |
590 | log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip()) | |
591 | return | |
592 | ||
593 | if key == "inetnum": | |
594 | start_address, delim, end_address = val.partition("-") | |
595 | ||
596 | # Strip any excess space | |
597 | start_address, end_address = start_address.rstrip(), end_address.strip() | |
598 | ||
599 | # Convert to IP address | |
600 | try: | |
601 | start_address = ipaddress.ip_address(start_address) | |
602 | end_address = ipaddress.ip_address(end_address) | |
603 | except ValueError: | |
604 | log.warning("Could not parse line: %s" % line) | |
605 | return | |
606 | ||
607 | # Set prefix to default | |
608 | prefix = 32 | |
609 | ||
610 | # Count number of addresses in this subnet | |
611 | num_addresses = int(end_address) - int(start_address) | |
612 | if num_addresses: | |
613 | prefix -= math.log(num_addresses, 2) | |
614 | ||
615 | inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix) | |
616 | ||
617 | elif key == "inet6num": | |
618 | inetnum[key] = val | |
619 | ||
620 | elif key == "country": | |
621 | inetnum[key] = val.upper() | |
622 | ||
623 | # Skip empty objects | |
624 | if not inetnum or not "country" in inetnum: | |
625 | return | |
626 | ||
627 | # Skip objects with bogus country code 'ZZ' | |
628 | if inetnum.get("country") == "ZZ": | |
629 | log.warning("Skipping network with bogus country 'ZZ': %s" % \ | |
630 | (inetnum.get("inet6num") or inetnum.get("inetnum"))) | |
631 | return | |
632 | ||
633 | network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) | |
634 | ||
635 | if not self._check_parsed_network(network): | |
636 | return | |
637 | ||
638 | self.db.execute("INSERT INTO _rirdata(network, country) \ | |
639 | VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country", | |
640 | "%s" % network, inetnum.get("country"), | |
641 | ) | |
642 | ||
643 | def _parse_org_block(self, block): | |
644 | org = {} | |
645 | for line in block: | |
646 | # Split line | |
647 | key, val = split_line(line) | |
648 | ||
649 | if key == "organisation": | |
650 | org[key] = val.upper() | |
651 | elif key == "org-name": | |
652 | org[key] = val | |
653 | ||
654 | # Skip empty objects | |
655 | if not org: | |
656 | return | |
657 | ||
658 | self.db.execute("INSERT INTO _organizations(handle, name) \ | |
659 | VALUES(%s, %s) ON CONFLICT (handle) DO \ | |
660 | UPDATE SET name = excluded.name", | |
661 | org.get("organisation"), org.get("org-name"), | |
662 | ) | |
663 | ||
664 | def _parse_line(self, line): | |
665 | # Skip version line | |
666 | if line.startswith("2"): | |
667 | return | |
668 | ||
669 | # Skip comments | |
670 | if line.startswith("#"): | |
671 | return | |
672 | ||
673 | try: | |
674 | registry, country_code, type, line = line.split("|", 3) | |
675 | except: | |
676 | log.warning("Could not parse line: %s" % line) | |
677 | return | |
678 | ||
679 | # Skip any lines that are for stats only | |
680 | if country_code == "*": | |
681 | return | |
682 | ||
683 | if type in ("ipv6", "ipv4"): | |
684 | return self._parse_ip_line(country_code, type, line) | |
685 | ||
686 | def _parse_ip_line(self, country, type, line): | |
687 | try: | |
688 | address, prefix, date, status, organization = line.split("|") | |
689 | except ValueError: | |
690 | organization = None | |
691 | ||
692 | # Try parsing the line without organization | |
693 | try: | |
694 | address, prefix, date, status = line.split("|") | |
695 | except ValueError: | |
696 | log.warning("Unhandled line format: %s" % line) | |
697 | return | |
698 | ||
699 | # Skip anything that isn't properly assigned | |
700 | if not status in ("assigned", "allocated"): | |
701 | return | |
702 | ||
703 | # Cast prefix into an integer | |
704 | try: | |
705 | prefix = int(prefix) | |
706 | except: | |
707 | log.warning("Invalid prefix: %s" % prefix) | |
708 | return | |
709 | ||
710 | # Fix prefix length for IPv4 | |
711 | if type == "ipv4": | |
712 | prefix = 32 - int(math.log(prefix, 2)) | |
713 | ||
714 | # Try to parse the address | |
715 | try: | |
716 | network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False) | |
717 | except ValueError: | |
718 | log.warning("Invalid IP address: %s" % address) | |
719 | return | |
720 | ||
721 | if not self._check_parsed_network(network): | |
722 | return | |
723 | ||
724 | self.db.execute("INSERT INTO networks(network, country) \ | |
725 | VALUES(%s, %s) ON CONFLICT (network) DO \ | |
726 | UPDATE SET country = excluded.country", | |
727 | "%s" % network, country, | |
728 | ) | |
729 | ||
730 | def handle_update_announcements(self, ns): | |
731 | server = ns.server[0] | |
732 | ||
733 | with self.db.transaction(): | |
734 | if server.startswith("/"): | |
735 | self._handle_update_announcements_from_bird(server) | |
736 | else: | |
737 | self._handle_update_announcements_from_telnet(server) | |
738 | ||
739 | # Purge anything we never want here | |
740 | self.db.execute(""" | |
741 | -- Delete default routes | |
742 | DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0'; | |
743 | ||
744 | -- Delete anything that is not global unicast address space | |
745 | DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3'; | |
746 | ||
747 | -- DELETE "current network" address space | |
748 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8'; | |
749 | ||
750 | -- DELETE local loopback address space | |
751 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8'; | |
752 | ||
753 | -- DELETE RFC 1918 address space | |
754 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8'; | |
755 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12'; | |
756 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16'; | |
757 | ||
758 | -- DELETE test, benchmark and documentation address space | |
759 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24'; | |
760 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24'; | |
761 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15'; | |
762 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24'; | |
763 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24'; | |
764 | ||
765 | -- DELETE CGNAT address space (RFC 6598) | |
766 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10'; | |
767 | ||
768 | -- DELETE link local address space | |
769 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16'; | |
770 | ||
771 | -- DELETE IPv6 to IPv4 (6to4) address space | |
772 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24'; | |
773 | ||
774 | -- DELETE multicast and reserved address space | |
775 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4'; | |
776 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4'; | |
777 | ||
778 | -- Delete networks that are too small to be in the global routing table | |
779 | DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48; | |
780 | DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24; | |
781 | ||
782 | -- Delete any non-public or reserved ASNs | |
783 | DELETE FROM announcements WHERE NOT ( | |
784 | (autnum >= 1 AND autnum <= 23455) | |
785 | OR | |
786 | (autnum >= 23457 AND autnum <= 64495) | |
787 | OR | |
788 | (autnum >= 131072 AND autnum <= 4199999999) | |
789 | ); | |
790 | ||
791 | -- Delete everything that we have not seen for 14 days | |
792 | DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days'; | |
793 | """) | |
794 | ||
795 | def _handle_update_announcements_from_bird(self, server): | |
796 | # Pre-compile the regular expression for faster searching | |
797 | route = re.compile(b"^\s(.+?)\s+.+?\[AS(.*?).\]$") | |
798 | ||
799 | log.info("Requesting routing table from Bird (%s)" % server) | |
800 | ||
801 | # Send command to list all routes | |
802 | for line in self._bird_cmd(server, "show route"): | |
803 | m = route.match(line) | |
804 | if not m: | |
805 | log.debug("Could not parse line: %s" % line.decode()) | |
806 | continue | |
807 | ||
808 | # Fetch the extracted network and ASN | |
809 | network, autnum = m.groups() | |
810 | ||
811 | # Insert it into the database | |
812 | self.db.execute("INSERT INTO announcements(network, autnum) \ | |
813 | VALUES(%s, %s) ON CONFLICT (network) DO \ | |
814 | UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP", | |
815 | network.decode(), autnum.decode(), | |
816 | ) | |
817 | ||
818 | def _handle_update_announcements_from_telnet(self, server): | |
819 | # Pre-compile regular expression for routes | |
820 | route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL) | |
821 | ||
822 | with telnetlib.Telnet(server) as t: | |
823 | # Enable debug mode | |
824 | #if ns.debug: | |
825 | # t.set_debuglevel(10) | |
826 | ||
827 | # Wait for console greeting | |
828 | greeting = t.read_until(b"> ", timeout=30) | |
829 | if not greeting: | |
830 | log.error("Could not get a console prompt") | |
831 | return 1 | |
832 | ||
833 | # Disable pagination | |
834 | t.write(b"terminal length 0\n") | |
835 | ||
836 | # Wait for the prompt to return | |
837 | t.read_until(b"> ") | |
838 | ||
839 | # Fetch the routing tables | |
840 | for protocol in ("ipv6", "ipv4"): | |
841 | log.info("Requesting %s routing table" % protocol) | |
842 | ||
843 | # Request the full unicast routing table | |
844 | t.write(b"show bgp %s unicast\n" % protocol.encode()) | |
845 | ||
846 | # Read entire header which ends with "Path" | |
847 | t.read_until(b"Path\r\n") | |
848 | ||
849 | while True: | |
850 | # Try reading a full entry | |
851 | # Those might be broken across multiple lines but ends with i | |
852 | line = t.read_until(b"i\r\n", timeout=5) | |
853 | if not line: | |
854 | break | |
855 | ||
856 | # Show line for debugging | |
857 | #log.debug(repr(line)) | |
858 | ||
859 | # Try finding a route in here | |
860 | m = route.match(line) | |
861 | if m: | |
862 | network, autnum = m.groups() | |
863 | ||
864 | # Convert network to string | |
865 | network = network.decode() | |
866 | ||
867 | # Append /24 for IPv4 addresses | |
868 | if not "/" in network and not ":" in network: | |
869 | network = "%s/24" % network | |
870 | ||
871 | # Convert AS number to integer | |
872 | autnum = int(autnum) | |
873 | ||
874 | log.info("Found announcement for %s by %s" % (network, autnum)) | |
875 | ||
876 | self.db.execute("INSERT INTO announcements(network, autnum) \ | |
877 | VALUES(%s, %s) ON CONFLICT (network) DO \ | |
878 | UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP", | |
879 | network, autnum, | |
880 | ) | |
881 | ||
882 | log.info("Finished reading the %s routing table" % protocol) | |
883 | ||
884 | def _bird_cmd(self, socket_path, command): | |
885 | # Connect to the socket | |
886 | s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) | |
887 | s.connect(socket_path) | |
888 | ||
889 | # Allocate some buffer | |
890 | buffer = b"" | |
891 | ||
892 | # Send the command | |
893 | s.send(b"%s\n" % command.encode()) | |
894 | ||
895 | while True: | |
896 | # Fill up the buffer | |
897 | buffer += s.recv(4096) | |
898 | ||
899 | while True: | |
900 | # Search for the next newline | |
901 | pos = buffer.find(b"\n") | |
902 | ||
903 | # If we cannot find one, we go back and read more data | |
904 | if pos <= 0: | |
905 | break | |
906 | ||
907 | # Cut after the newline character | |
908 | pos += 1 | |
909 | ||
910 | # Split the line we want and keep the rest in buffer | |
911 | line, buffer = buffer[:pos], buffer[pos:] | |
912 | ||
913 | # Look for the end-of-output indicator | |
914 | if line == b"0000 \n": | |
915 | return | |
916 | ||
917 | # Otherwise return the line | |
918 | yield line | |
919 | ||
920 | def handle_update_overrides(self, ns): | |
921 | with self.db.transaction(): | |
922 | # Drop all data that we have | |
923 | self.db.execute(""" | |
924 | TRUNCATE TABLE autnum_overrides; | |
925 | TRUNCATE TABLE network_overrides; | |
926 | """) | |
927 | ||
928 | for file in ns.files: | |
929 | log.info("Reading %s..." % file) | |
930 | ||
931 | with open(file, "rb") as f: | |
932 | for type, block in location.importer.read_blocks(f): | |
933 | if type == "net": | |
934 | network = block.get("net") | |
935 | # Try to parse and normalise the network | |
936 | try: | |
937 | network = ipaddress.ip_network(network, strict=False) | |
938 | except ValueError as e: | |
939 | log.warning("Invalid IP network: %s: %s" % (network, e)) | |
940 | continue | |
941 | ||
942 | # Prevent that we overwrite all networks | |
943 | if network.prefixlen == 0: | |
944 | log.warning("Skipping %s: You cannot overwrite default" % network) | |
945 | continue | |
946 | ||
947 | self.db.execute(""" | |
948 | INSERT INTO network_overrides( | |
949 | network, | |
950 | country, | |
951 | is_anonymous_proxy, | |
952 | is_satellite_provider, | |
953 | is_anycast | |
954 | ) VALUES (%s, %s, %s, %s, %s) | |
955 | ON CONFLICT (network) DO NOTHING""", | |
956 | "%s" % network, | |
957 | block.get("country"), | |
958 | self._parse_bool(block, "is-anonymous-proxy"), | |
959 | self._parse_bool(block, "is-satellite-provider"), | |
960 | self._parse_bool(block, "is-anycast"), | |
961 | ) | |
962 | ||
963 | elif type == "aut-num": | |
964 | autnum = block.get("aut-num") | |
965 | ||
966 | # Check if AS number begins with "AS" | |
967 | if not autnum.startswith("AS"): | |
968 | log.warning("Invalid AS number: %s" % autnum) | |
969 | continue | |
970 | ||
971 | # Strip "AS" | |
972 | autnum = autnum[2:] | |
973 | ||
974 | self.db.execute(""" | |
975 | INSERT INTO autnum_overrides( | |
976 | number, | |
977 | name, | |
978 | country, | |
979 | is_anonymous_proxy, | |
980 | is_satellite_provider, | |
981 | is_anycast | |
982 | ) VALUES(%s, %s, %s, %s, %s, %s) | |
983 | ON CONFLICT DO NOTHING""", | |
984 | autnum, | |
985 | block.get("name"), | |
986 | block.get("country"), | |
987 | self._parse_bool(block, "is-anonymous-proxy"), | |
988 | self._parse_bool(block, "is-satellite-provider"), | |
989 | self._parse_bool(block, "is-anycast"), | |
990 | ) | |
991 | ||
992 | else: | |
993 | log.warning("Unsupport type: %s" % type) | |
994 | ||
995 | @staticmethod | |
996 | def _parse_bool(block, key): | |
997 | val = block.get(key) | |
998 | ||
999 | # There is no point to proceed when we got None | |
1000 | if val is None: | |
1001 | return | |
1002 | ||
1003 | # Convert to lowercase | |
1004 | val = val.lower() | |
1005 | ||
1006 | # True | |
1007 | if val in ("yes", "1"): | |
1008 | return True | |
1009 | ||
1010 | # False | |
1011 | if val in ("no", "0"): | |
1012 | return False | |
1013 | ||
1014 | # Default to None | |
1015 | return None | |
1016 | ||
1017 | def handle_import_countries(self, ns): | |
1018 | with self.db.transaction(): | |
1019 | # Drop all data that we have | |
1020 | self.db.execute("TRUNCATE TABLE countries") | |
1021 | ||
1022 | for file in ns.file: | |
1023 | for line in file: | |
1024 | line = line.rstrip() | |
1025 | ||
1026 | # Ignore any comments | |
1027 | if line.startswith("#"): | |
1028 | continue | |
1029 | ||
1030 | try: | |
1031 | country_code, continent_code, name = line.split(maxsplit=2) | |
1032 | except: | |
1033 | log.warning("Could not parse line: %s" % line) | |
1034 | continue | |
1035 | ||
1036 | self.db.execute("INSERT INTO countries(country_code, name, continent_code) \ | |
1037 | VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code) | |
1038 | ||
1039 | ||
1040 | def split_line(line): | |
1041 | key, colon, val = line.partition(":") | |
1042 | ||
1043 | # Strip any excess space | |
1044 | key = key.strip() | |
1045 | val = val.strip() | |
1046 | ||
1047 | return key, val | |
1048 | ||
1049 | def main(): | |
1050 | # Run the command line interface | |
1051 | c = CLI() | |
1052 | c.run() | |
1053 | ||
1054 | main() |