]> git.ipfire.org Git - people/ms/libloc.git/blob - src/python/location-importer.in
projects / people / ms / libloc.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Revert "location-importer.in: only import relevant data from AFRINIC, APNIC and RIPE"
[people/ms/libloc.git] / src / python / location-importer.in
1 #!/usr/bin/python3
2 ###############################################################################
3 # #
4 # libloc - A library to determine the location of someone on the Internet #
5 # #
6 # Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
7 # #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
12 # #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
17 # #
18 ###############################################################################
19
20 import argparse
21 import ipaddress
22 import logging
23 import math
24 import re
25 import socket
26 import sys
27 import telnetlib
28
29 # Load our location module
30 import location
31 import location.database
32 import location.importer
33 from location.i18n import _
34
35 # Initialise logging
36 log = logging.getLogger("location.importer")
37 log.propagate = 1
38
39 class CLI(object):
40 def parse_cli(self):
41 parser = argparse.ArgumentParser(
42 description=_("Location Importer Command Line Interface"),
43 )
44 subparsers = parser.add_subparsers()
45
46 # Global configuration flags
47 parser.add_argument("--debug", action="store_true",
48 help=_("Enable debug output"))
49 parser.add_argument("--quiet", action="store_true",
50 help=_("Enable quiet mode"))
51
52 # version
53 parser.add_argument("--version", action="version",
54 version="%(prog)s @VERSION@")
55
56 # Database
57 parser.add_argument("--database-host", required=True,
58 help=_("Database Hostname"), metavar=_("HOST"))
59 parser.add_argument("--database-name", required=True,
60 help=_("Database Name"), metavar=_("NAME"))
61 parser.add_argument("--database-username", required=True,
62 help=_("Database Username"), metavar=_("USERNAME"))
63 parser.add_argument("--database-password", required=True,
64 help=_("Database Password"), metavar=_("PASSWORD"))
65
66 # Write Database
67 write = subparsers.add_parser("write", help=_("Write database to file"))
68 write.set_defaults(func=self.handle_write)
69 write.add_argument("file", nargs=1, help=_("Database File"))
70 write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
71 write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key"))
72 write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
73 write.add_argument("--description", nargs="?", help=_("Sets a description"))
74 write.add_argument("--license", nargs="?", help=_("Sets the license"))
75 write.add_argument("--version", type=int, help=_("Database Format Version"))
76
77 # Update WHOIS
78 update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
79 update_whois.set_defaults(func=self.handle_update_whois)
80
81 # Update announcements
82 update_announcements = subparsers.add_parser("update-announcements",
83 help=_("Update BGP Annoucements"))
84 update_announcements.set_defaults(func=self.handle_update_announcements)
85 update_announcements.add_argument("server", nargs=1,
86 help=_("Route Server to connect to"), metavar=_("SERVER"))
87
88 # Update overrides
89 update_overrides = subparsers.add_parser("update-overrides",
90 help=_("Update overrides"),
91 )
92 update_overrides.add_argument(
93 "files", nargs="+", help=_("Files to import"),
94 )
95 update_overrides.set_defaults(func=self.handle_update_overrides)
96
97 # Import countries
98 import_countries = subparsers.add_parser("import-countries",
99 help=_("Import countries"),
100 )
101 import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"),
102 help=_("File to import"))
103 import_countries.set_defaults(func=self.handle_import_countries)
104
105 args = parser.parse_args()
106
107 # Configure logging
108 if args.debug:
109 location.logger.set_level(logging.DEBUG)
110 elif args.quiet:
111 location.logger.set_level(logging.WARNING)
112
113 # Print usage if no action was given
114 if not "func" in args:
115 parser.print_usage()
116 sys.exit(2)
117
118 return args
119
120 def run(self):
121 # Parse command line arguments
122 args = self.parse_cli()
123
124 # Initialise database
125 self.db = self._setup_database(args)
126
127 # Call function
128 ret = args.func(args)
129
130 # Return with exit code
131 if ret:
132 sys.exit(ret)
133
134 # Otherwise just exit
135 sys.exit(0)
136
137 def _setup_database(self, ns):
138 """
139 Initialise the database
140 """
141 # Connect to database
142 db = location.database.Connection(
143 host=ns.database_host, database=ns.database_name,
144 user=ns.database_username, password=ns.database_password,
145 )
146
147 with db.transaction():
148 db.execute("""
149 -- announcements
150 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
151 first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
152 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
153 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
154 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
155
156 -- autnums
157 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
158 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
159
160 -- countries
161 CREATE TABLE IF NOT EXISTS countries(
162 country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
163 CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
164
165 -- networks
166 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
167 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
168 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
169
170 -- overrides
171 CREATE TABLE IF NOT EXISTS autnum_overrides(
172 number bigint NOT NULL,
173 name text,
174 country text,
175 is_anonymous_proxy boolean,
176 is_satellite_provider boolean,
177 is_anycast boolean
178 );
179 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
180 ON autnum_overrides(number);
181
182 CREATE TABLE IF NOT EXISTS network_overrides(
183 network inet NOT NULL,
184 country text,
185 is_anonymous_proxy boolean,
186 is_satellite_provider boolean,
187 is_anycast boolean
188 );
189 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
190 ON network_overrides(network);
191 """)
192
193 return db
194
195 def handle_write(self, ns):
196 """
197 Compiles a database in libloc format out of what is in the database
198 """
199 # Allocate a writer
200 writer = location.Writer(ns.signing_key, ns.backup_signing_key)
201
202 # Set all metadata
203 if ns.vendor:
204 writer.vendor = ns.vendor
205
206 if ns.description:
207 writer.description = ns.description
208
209 if ns.license:
210 writer.license = ns.license
211
212 # Add all Autonomous Systems
213 log.info("Writing Autonomous Systems...")
214
215 # Select all ASes with a name
216 rows = self.db.query("""
217 SELECT
218 autnums.number AS number,
219 COALESCE(
220 (SELECT overrides.name FROM autnum_overrides overrides
221 WHERE overrides.number = autnums.number),
222 autnums.name
223 ) AS name
224 FROM autnums
225 WHERE name <> %s ORDER BY number
226 """, "")
227
228 for row in rows:
229 a = writer.add_as(row.number)
230 a.name = row.name
231
232 # Add all networks
233 log.info("Writing networks...")
234
235 # Select all known networks
236 rows = self.db.query("""
237 -- Get a (sorted) list of all known networks
238 WITH known_networks AS (
239 SELECT network FROM announcements
240 UNION
241 SELECT network FROM networks
242 UNION
243 SELECT network FROM network_overrides
244 ORDER BY network
245 )
246
247 -- Return a list of those networks enriched with all
248 -- other information that we store in the database
249 SELECT
250 DISTINCT ON (known_networks.network)
251 known_networks.network AS network,
252 announcements.autnum AS autnum,
253
254 -- Country
255 COALESCE(
256 (
257 SELECT country FROM network_overrides overrides
258 WHERE announcements.network <<= overrides.network
259 ORDER BY masklen(overrides.network) DESC
260 LIMIT 1
261 ),
262 (
263 SELECT country FROM autnum_overrides overrides
264 WHERE announcements.autnum = overrides.number
265 ),
266 networks.country
267 ) AS country,
268
269 -- Flags
270 COALESCE(
271 (
272 SELECT is_anonymous_proxy FROM network_overrides overrides
273 WHERE announcements.network <<= overrides.network
274 ORDER BY masklen(overrides.network) DESC
275 LIMIT 1
276 ),
277 (
278 SELECT is_anonymous_proxy FROM autnum_overrides overrides
279 WHERE announcements.autnum = overrides.number
280 ),
281 FALSE
282 ) AS is_anonymous_proxy,
283 COALESCE(
284 (
285 SELECT is_satellite_provider FROM network_overrides overrides
286 WHERE announcements.network <<= overrides.network
287 ORDER BY masklen(overrides.network) DESC
288 LIMIT 1
289 ),
290 (
291 SELECT is_satellite_provider FROM autnum_overrides overrides
292 WHERE announcements.autnum = overrides.number
293 ),
294 FALSE
295 ) AS is_satellite_provider,
296 COALESCE(
297 (
298 SELECT is_anycast FROM network_overrides overrides
299 WHERE announcements.network <<= overrides.network
300 ORDER BY masklen(overrides.network) DESC
301 LIMIT 1
302 ),
303 (
304 SELECT is_anycast FROM autnum_overrides overrides
305 WHERE announcements.autnum = overrides.number
306 ),
307 FALSE
308 ) AS is_anycast,
309
310 -- Must be part of returned values for ORDER BY clause
311 masklen(announcements.network) AS sort_a,
312 masklen(networks.network) AS sort_b
313 FROM known_networks
314 LEFT JOIN announcements ON known_networks.network <<= announcements.network
315 LEFT JOIN networks ON known_networks.network <<= networks.network
316 ORDER BY known_networks.network, sort_a DESC, sort_b DESC
317 """)
318
319 for row in rows:
320 network = writer.add_network(row.network)
321
322 # Save country
323 if row.country:
324 network.country_code = row.country
325
326 # Save ASN
327 if row.autnum:
328 network.asn = row.autnum
329
330 # Set flags
331 if row.is_anonymous_proxy:
332 network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
333
334 if row.is_satellite_provider:
335 network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
336
337 if row.is_anycast:
338 network.set_flag(location.NETWORK_FLAG_ANYCAST)
339
340 # Add all countries
341 log.info("Writing countries...")
342 rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
343
344 for row in rows:
345 c = writer.add_country(row.country_code)
346 c.continent_code = row.continent_code
347 c.name = row.name
348
349 # Write everything to file
350 log.info("Writing database to file...")
351 for file in ns.file:
352 writer.write(file)
353
354 def handle_update_whois(self, ns):
355 downloader = location.importer.Downloader()
356
357 # Download all sources
358 with self.db.transaction():
359 # Create some temporary tables to store parsed data
360 self.db.execute("""
361 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
362 ON COMMIT DROP;
363 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
364
365 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
366 ON COMMIT DROP;
367 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
368 """)
369
370 for source in location.importer.WHOIS_SOURCES:
371 with downloader.request(source, return_blocks=True) as f:
372 for block in f:
373 self._parse_block(block)
374
375 self.db.execute("""
376 INSERT INTO autnums(number, name)
377 SELECT _autnums.number, _organizations.name FROM _autnums
378 JOIN _organizations ON _autnums.organization = _organizations.handle
379 ON CONFLICT (number) DO UPDATE SET name = excluded.name;
380 """)
381
382 # Download all extended sources
383 for source in location.importer.EXTENDED_SOURCES:
384 with self.db.transaction():
385 # Download data
386 with downloader.request(source) as f:
387 for line in f:
388 self._parse_line(line)
389
390 def _parse_block(self, block):
391 # Get first line to find out what type of block this is
392 line = block[0]
393
394 # aut-num
395 if line.startswith("aut-num:"):
396 return self._parse_autnum_block(block)
397
398 # inetnum
399 if line.startswith("inet6num:") or line.startswith("inetnum:"):
400 return self._parse_inetnum_block(block)
401
402 # organisation
403 elif line.startswith("organisation:"):
404 return self._parse_org_block(block)
405
406 def _parse_autnum_block(self, block):
407 autnum = {}
408 for line in block:
409 # Split line
410 key, val = split_line(line)
411
412 if key == "aut-num":
413 m = re.match(r"^(AS|as)(\d+)", val)
414 if m:
415 autnum["asn"] = m.group(2)
416
417 elif key == "org":
418 autnum[key] = val
419
420 # Skip empty objects
421 if not autnum:
422 return
423
424 # Insert into database
425 self.db.execute("INSERT INTO _autnums(number, organization) \
426 VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
427 organization = excluded.organization",
428 autnum.get("asn"), autnum.get("org"),
429 )
430
431 def _parse_inetnum_block(self, block):
432 logging.debug("Parsing inetnum block:")
433
434 inetnum = {}
435 for line in block:
436 logging.debug(line)
437
438 # Split line
439 key, val = split_line(line)
440
441 if key == "inetnum":
442 start_address, delim, end_address = val.partition("-")
443
444 # Strip any excess space
445 start_address, end_address = start_address.rstrip(), end_address.strip()
446
447 # Convert to IP address
448 try:
449 start_address = ipaddress.ip_address(start_address)
450 end_address = ipaddress.ip_address(end_address)
451 except ValueError:
452 logging.warning("Could not parse line: %s" % line)
453 return
454
455 # Set prefix to default
456 prefix = 32
457
458 # Count number of addresses in this subnet
459 num_addresses = int(end_address) - int(start_address)
460 if num_addresses:
461 prefix -= math.log(num_addresses, 2)
462
463 inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
464
465 elif key == "inet6num":
466 inetnum[key] = val
467
468 elif key == "country":
469 if val == "UNITED STATES":
470 val = "US"
471
472 inetnum[key] = val.upper()
473
474 # Skip empty objects
475 if not inetnum:
476 return
477
478 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
479
480 # Bail out in case we have processed a non-public IP network
481 if network.is_private:
482 logging.warning("Skipping non-globally routable network: %s" % network)
483 return
484
485 self.db.execute("INSERT INTO networks(network, country) \
486 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
487 "%s" % network, inetnum.get("country"),
488 )
489
490 def _parse_org_block(self, block):
491 org = {}
492 for line in block:
493 # Split line
494 key, val = split_line(line)
495
496 if key in ("organisation", "org-name"):
497 org[key] = val
498
499 # Skip empty objects
500 if not org:
501 return
502
503 self.db.execute("INSERT INTO _organizations(handle, name) \
504 VALUES(%s, %s) ON CONFLICT (handle) DO \
505 UPDATE SET name = excluded.name",
506 org.get("organisation"), org.get("org-name"),
507 )
508
509 def _parse_line(self, line):
510 # Skip version line
511 if line.startswith("2"):
512 return
513
514 # Skip comments
515 if line.startswith("#"):
516 return
517
518 try:
519 registry, country_code, type, line = line.split("|", 3)
520 except:
521 log.warning("Could not parse line: %s" % line)
522 return
523
524 # Skip any lines that are for stats only
525 if country_code == "*":
526 return
527
528 if type in ("ipv6", "ipv4"):
529 return self._parse_ip_line(country_code, type, line)
530
531 def _parse_ip_line(self, country, type, line):
532 try:
533 address, prefix, date, status, organization = line.split("|")
534 except ValueError:
535 organization = None
536
537 # Try parsing the line without organization
538 try:
539 address, prefix, date, status = line.split("|")
540 except ValueError:
541 log.warning("Unhandled line format: %s" % line)
542 return
543
544 # Skip anything that isn't properly assigned
545 if not status in ("assigned", "allocated"):
546 return
547
548 # Cast prefix into an integer
549 try:
550 prefix = int(prefix)
551 except:
552 log.warning("Invalid prefix: %s" % prefix)
553 return
554
555 # Fix prefix length for IPv4
556 if type == "ipv4":
557 prefix = 32 - int(math.log(prefix, 2))
558
559 # Try to parse the address
560 try:
561 network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
562 except ValueError:
563 log.warning("Invalid IP address: %s" % address)
564 return
565
566 self.db.execute("INSERT INTO networks(network, country) \
567 VALUES(%s, %s) ON CONFLICT (network) DO \
568 UPDATE SET country = excluded.country",
569 "%s" % network, country,
570 )
571
572 def handle_update_announcements(self, ns):
573 server = ns.server[0]
574
575 with self.db.transaction():
576 if server.startswith("/"):
577 self._handle_update_announcements_from_bird(server)
578 else:
579 self._handle_update_announcements_from_telnet(server)
580
581 # Purge anything we never want here
582 self.db.execute("""
583 -- Delete default routes
584 DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
585
586 -- Delete anything that is not global unicast address space
587 DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
588
589 -- DELETE "current network" address space
590 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
591
592 -- DELETE local loopback address space
593 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
594
595 -- DELETE RFC 1918 address space
596 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
597 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
598 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
599
600 -- DELETE test, benchmark and documentation address space
601 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
602 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
603 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
604 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
605 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
606
607 -- DELETE CGNAT address space (RFC 6598)
608 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
609
610 -- DELETE link local address space
611 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
612
613 -- DELETE IPv6 to IPv4 (6to4) address space
614 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
615
616 -- DELETE multicast and reserved address space
617 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
618 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
619
620 -- Delete networks that are too small to be in the global routing table
621 DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
622 DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
623
624 -- Delete any non-public or reserved ASNs
625 DELETE FROM announcements WHERE NOT (
626 (autnum >= 1 AND autnum <= 23455)
627 OR
628 (autnum >= 23457 AND autnum <= 64495)
629 OR
630 (autnum >= 131072 AND autnum <= 4199999999)
631 );
632
633 -- Delete everything that we have not seen for 14 days
634 DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
635 """)
636
637 def _handle_update_announcements_from_bird(self, server):
638 # Pre-compile the regular expression for faster searching
639 route = re.compile(b"^\s(.+?)\s+.+?\[AS(.*?).\]$")
640
641 log.info("Requesting routing table from Bird (%s)" % server)
642
643 # Send command to list all routes
644 for line in self._bird_cmd(server, "show route"):
645 m = route.match(line)
646 if not m:
647 log.debug("Could not parse line: %s" % line.decode())
648 continue
649
650 # Fetch the extracted network and ASN
651 network, autnum = m.groups()
652
653 # Insert it into the database
654 self.db.execute("INSERT INTO announcements(network, autnum) \
655 VALUES(%s, %s) ON CONFLICT (network) DO \
656 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
657 network.decode(), autnum.decode(),
658 )
659
660 def _handle_update_announcements_from_telnet(self, server):
661 # Pre-compile regular expression for routes
662 route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
663
664 with telnetlib.Telnet(server) as t:
665 # Enable debug mode
666 #if ns.debug:
667 # t.set_debuglevel(10)
668
669 # Wait for console greeting
670 greeting = t.read_until(b"> ", timeout=30)
671 if not greeting:
672 log.error("Could not get a console prompt")
673 return 1
674
675 # Disable pagination
676 t.write(b"terminal length 0\n")
677
678 # Wait for the prompt to return
679 t.read_until(b"> ")
680
681 # Fetch the routing tables
682 for protocol in ("ipv6", "ipv4"):
683 log.info("Requesting %s routing table" % protocol)
684
685 # Request the full unicast routing table
686 t.write(b"show bgp %s unicast\n" % protocol.encode())
687
688 # Read entire header which ends with "Path"
689 t.read_until(b"Path\r\n")
690
691 while True:
692 # Try reading a full entry
693 # Those might be broken across multiple lines but ends with i
694 line = t.read_until(b"i\r\n", timeout=5)
695 if not line:
696 break
697
698 # Show line for debugging
699 #log.debug(repr(line))
700
701 # Try finding a route in here
702 m = route.match(line)
703 if m:
704 network, autnum = m.groups()
705
706 # Convert network to string
707 network = network.decode()
708
709 # Append /24 for IPv4 addresses
710 if not "/" in network and not ":" in network:
711 network = "%s/24" % network
712
713 # Convert AS number to integer
714 autnum = int(autnum)
715
716 log.info("Found announcement for %s by %s" % (network, autnum))
717
718 self.db.execute("INSERT INTO announcements(network, autnum) \
719 VALUES(%s, %s) ON CONFLICT (network) DO \
720 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
721 network, autnum,
722 )
723
724 log.info("Finished reading the %s routing table" % protocol)
725
726 def _bird_cmd(self, socket_path, command):
727 # Connect to the socket
728 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
729 s.connect(socket_path)
730
731 # Allocate some buffer
732 buffer = b""
733
734 # Send the command
735 s.send(b"%s\n" % command.encode())
736
737 while True:
738 # Fill up the buffer
739 buffer += s.recv(4096)
740
741 while True:
742 # Search for the next newline
743 pos = buffer.find(b"\n")
744
745 # If we cannot find one, we go back and read more data
746 if pos <= 0:
747 break
748
749 # Cut after the newline character
750 pos += 1
751
752 # Split the line we want and keep the rest in buffer
753 line, buffer = buffer[:pos], buffer[pos:]
754
755 # Look for the end-of-output indicator
756 if line == b"0000 \n":
757 return
758
759 # Otherwise return the line
760 yield line
761
762 def handle_update_overrides(self, ns):
763 with self.db.transaction():
764 # Drop all data that we have
765 self.db.execute("""
766 TRUNCATE TABLE autnum_overrides;
767 TRUNCATE TABLE network_overrides;
768 """)
769
770 for file in ns.files:
771 log.info("Reading %s..." % file)
772
773 with open(file, "rb") as f:
774 for type, block in location.importer.read_blocks(f):
775 if type == "net":
776 network = block.get("net")
777 # Try to parse and normalise the network
778 try:
779 network = ipaddress.ip_network(network, strict=False)
780 except ValueError as e:
781 log.warning("Invalid IP network: %s: %s" % (network, e))
782 continue
783
784 # Prevent that we overwrite all networks
785 if network.prefixlen == 0:
786 log.warning("Skipping %s: You cannot overwrite default" % network)
787 continue
788
789 self.db.execute("""
790 INSERT INTO network_overrides(
791 network,
792 country,
793 is_anonymous_proxy,
794 is_satellite_provider,
795 is_anycast
796 ) VALUES (%s, %s, %s, %s, %s)
797 ON CONFLICT (network) DO NOTHING""",
798 "%s" % network,
799 block.get("country"),
800 self._parse_bool(block, "is-anonymous-proxy"),
801 self._parse_bool(block, "is-satellite-provider"),
802 self._parse_bool(block, "is-anycast"),
803 )
804
805 elif type == "aut-num":
806 autnum = block.get("aut-num")
807
808 # Check if AS number begins with "AS"
809 if not autnum.startswith("AS"):
810 log.warning("Invalid AS number: %s" % autnum)
811 continue
812
813 # Strip "AS"
814 autnum = autnum[2:]
815
816 self.db.execute("""
817 INSERT INTO autnum_overrides(
818 number,
819 name,
820 country,
821 is_anonymous_proxy,
822 is_satellite_provider,
823 is_anycast
824 ) VALUES(%s, %s, %s, %s, %s, %s)
825 ON CONFLICT DO NOTHING""",
826 autnum,
827 block.get("name"),
828 block.get("country"),
829 self._parse_bool(block, "is-anonymous-proxy"),
830 self._parse_bool(block, "is-satellite-provider"),
831 self._parse_bool(block, "is-anycast"),
832 )
833
834 else:
835 log.warning("Unsupport type: %s" % type)
836
837 @staticmethod
838 def _parse_bool(block, key):
839 val = block.get(key)
840
841 # There is no point to proceed when we got None
842 if val is None:
843 return
844
845 # Convert to lowercase
846 val = val.lower()
847
848 # True
849 if val in ("yes", "1"):
850 return True
851
852 # False
853 if val in ("no", "0"):
854 return False
855
856 # Default to None
857 return None
858
859 def handle_import_countries(self, ns):
860 with self.db.transaction():
861 # Drop all data that we have
862 self.db.execute("TRUNCATE TABLE countries")
863
864 for file in ns.file:
865 for line in file:
866 line = line.rstrip()
867
868 # Ignore any comments
869 if line.startswith("#"):
870 continue
871
872 try:
873 country_code, continent_code, name = line.split(maxsplit=2)
874 except:
875 log.warning("Could not parse line: %s" % line)
876 continue
877
878 self.db.execute("INSERT INTO countries(country_code, name, continent_code) \
879 VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code)
880
881
882 def split_line(line):
883 key, colon, val = line.partition(":")
884
885 # Strip any excess space
886 key = key.strip()
887 val = val.strip()
888
889 return key, val
890
891 def main():
892 # Run the command line interface
893 c = CLI()
894 c.run()
895
896 main()
A library to determine the location of someone on the Internet