]>
Commit | Line | Data |
---|---|---|
78ff0cf2 MT |
1 | #!/usr/bin/python3 |
2 | ############################################################################### | |
3 | # # | |
4 | # libloc - A library to determine the location of someone on the Internet # | |
5 | # # | |
6 | # Copyright (C) 2020 IPFire Development Team <info@ipfire.org> # | |
7 | # # | |
8 | # This library is free software; you can redistribute it and/or # | |
9 | # modify it under the terms of the GNU Lesser General Public # | |
10 | # License as published by the Free Software Foundation; either # | |
11 | # version 2.1 of the License, or (at your option) any later version. # | |
12 | # # | |
13 | # This library is distributed in the hope that it will be useful, # | |
14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of # | |
15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # | |
16 | # Lesser General Public License for more details. # | |
17 | # # | |
18 | ############################################################################### | |
19 | ||
20 | import argparse | |
6ffd06b5 | 21 | import ipaddress |
78ff0cf2 | 22 | import logging |
6ffd06b5 MT |
23 | import math |
24 | import re | |
78ff0cf2 | 25 | import sys |
83d61c46 | 26 | import telnetlib |
78ff0cf2 MT |
27 | |
28 | # Load our location module | |
29 | import location | |
29c6fa22 | 30 | import location.database |
3192b66c | 31 | import location.importer |
78ff0cf2 MT |
32 | from location.i18n import _ |
33 | ||
34 | # Initialise logging | |
35 | log = logging.getLogger("location.importer") | |
36 | log.propagate = 1 | |
37 | ||
38 | class CLI(object): | |
39 | def parse_cli(self): | |
40 | parser = argparse.ArgumentParser( | |
41 | description=_("Location Importer Command Line Interface"), | |
42 | ) | |
6ffd06b5 | 43 | subparsers = parser.add_subparsers() |
78ff0cf2 MT |
44 | |
45 | # Global configuration flags | |
46 | parser.add_argument("--debug", action="store_true", | |
47 | help=_("Enable debug output")) | |
48 | ||
49 | # version | |
50 | parser.add_argument("--version", action="version", | |
51 | version="%(prog)s @VERSION@") | |
52 | ||
29c6fa22 MT |
53 | # Database |
54 | parser.add_argument("--database-host", required=True, | |
55 | help=_("Database Hostname"), metavar=_("HOST")) | |
56 | parser.add_argument("--database-name", required=True, | |
57 | help=_("Database Name"), metavar=_("NAME")) | |
58 | parser.add_argument("--database-username", required=True, | |
59 | help=_("Database Username"), metavar=_("USERNAME")) | |
60 | parser.add_argument("--database-password", required=True, | |
61 | help=_("Database Password"), metavar=_("PASSWORD")) | |
62 | ||
6ffd06b5 MT |
63 | # Update WHOIS |
64 | update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information")) | |
65 | update_whois.set_defaults(func=self.handle_update_whois) | |
66 | ||
83d61c46 MT |
67 | # Update announcements |
68 | update_announcements = subparsers.add_parser("update-announcements", | |
69 | help=_("Update BGP Annoucements")) | |
70 | update_announcements.set_defaults(func=self.handle_update_announcements) | |
71 | update_announcements.add_argument("server", nargs=1, | |
72 | help=_("Route Server to connect to"), metavar=_("SERVER")) | |
73 | ||
d7fc3057 MT |
74 | # Update overrides |
75 | update_overrides = subparsers.add_parser("update-overrides", | |
76 | help=_("Update overrides"), | |
77 | ) | |
78 | update_overrides.add_argument( | |
79 | "files", nargs="+", help=_("Files to import"), | |
80 | ) | |
81 | update_overrides.set_defaults(func=self.handle_update_overrides) | |
82 | ||
78ff0cf2 MT |
83 | args = parser.parse_args() |
84 | ||
85 | # Enable debug logging | |
86 | if args.debug: | |
87 | log.setLevel(logging.DEBUG) | |
88 | ||
6ffd06b5 MT |
89 | # Print usage if no action was given |
90 | if not "func" in args: | |
91 | parser.print_usage() | |
92 | sys.exit(2) | |
93 | ||
78ff0cf2 MT |
94 | return args |
95 | ||
96 | def run(self): | |
97 | # Parse command line arguments | |
98 | args = self.parse_cli() | |
99 | ||
29c6fa22 | 100 | # Initialise database |
6ffd06b5 | 101 | self.db = self._setup_database(args) |
29c6fa22 | 102 | |
78ff0cf2 | 103 | # Call function |
6ffd06b5 | 104 | ret = args.func(args) |
78ff0cf2 MT |
105 | |
106 | # Return with exit code | |
107 | if ret: | |
108 | sys.exit(ret) | |
109 | ||
110 | # Otherwise just exit | |
111 | sys.exit(0) | |
112 | ||
29c6fa22 MT |
113 | def _setup_database(self, ns): |
114 | """ | |
115 | Initialise the database | |
116 | """ | |
117 | # Connect to database | |
118 | db = location.database.Connection( | |
119 | host=ns.database_host, database=ns.database_name, | |
120 | user=ns.database_username, password=ns.database_password, | |
121 | ) | |
122 | ||
123 | with db.transaction(): | |
124 | db.execute(""" | |
83d61c46 MT |
125 | -- announcements |
126 | CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint, | |
127 | first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP, | |
128 | last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP); | |
129 | CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network); | |
130 | CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network)); | |
131 | ||
6ffd06b5 | 132 | -- autnums |
83d61c46 | 133 | CREATE TABLE IF NOT EXISTS autnums(number bigint, name text); |
6ffd06b5 MT |
134 | CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number); |
135 | ||
429a43d1 | 136 | -- networks |
83d61c46 | 137 | CREATE TABLE IF NOT EXISTS networks(network inet, country text); |
429a43d1 | 138 | CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network); |
83d61c46 | 139 | CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops); |
d7fc3057 MT |
140 | |
141 | -- overrides | |
142 | CREATE TABLE IF NOT EXISTS autnum_overrides( | |
143 | number bigint NOT NULL, | |
144 | name text, | |
145 | is_anonymous_proxy boolean DEFAULT FALSE, | |
146 | is_satellite_provider boolean DEFAULT FALSE, | |
147 | is_anycast boolean DEFAULT FALSE | |
148 | ); | |
149 | CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number | |
150 | ON autnum_overrides(number); | |
151 | ||
152 | CREATE TABLE IF NOT EXISTS network_overrides( | |
153 | network inet NOT NULL, | |
154 | country text, | |
155 | is_anonymous_proxy boolean DEFAULT FALSE, | |
156 | is_satellite_provider boolean DEFAULT FALSE, | |
157 | is_anycast boolean DEFAULT FALSE | |
158 | ); | |
159 | CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network | |
160 | ON network_overrides(network); | |
29c6fa22 MT |
161 | """) |
162 | ||
163 | return db | |
164 | ||
6ffd06b5 MT |
165 | def handle_update_whois(self, ns): |
166 | downloader = location.importer.Downloader() | |
167 | ||
168 | # Download all sources | |
0365119d MT |
169 | with self.db.transaction(): |
170 | # Create some temporary tables to store parsed data | |
171 | self.db.execute(""" | |
172 | CREATE TEMPORARY TABLE _autnums(number integer, organization text) | |
173 | ON COMMIT DROP; | |
174 | CREATE UNIQUE INDEX _autnums_number ON _autnums(number); | |
175 | ||
176 | CREATE TEMPORARY TABLE _organizations(handle text, name text) | |
177 | ON COMMIT DROP; | |
178 | CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle); | |
179 | """) | |
180 | ||
181 | for source in location.importer.WHOIS_SOURCES: | |
6ffd06b5 MT |
182 | with downloader.request(source, return_blocks=True) as f: |
183 | for block in f: | |
184 | self._parse_block(block) | |
185 | ||
0365119d MT |
186 | self.db.execute(""" |
187 | INSERT INTO autnums(number, name) | |
188 | SELECT _autnums.number, _organizations.name FROM _autnums | |
189 | LEFT JOIN _organizations ON _autnums.organization = _organizations.handle | |
190 | ON CONFLICT (number) DO UPDATE SET name = excluded.name; | |
191 | """) | |
192 | ||
429a43d1 MT |
193 | # Download all extended sources |
194 | for source in location.importer.EXTENDED_SOURCES: | |
195 | with self.db.transaction(): | |
429a43d1 MT |
196 | # Download data |
197 | with downloader.request(source) as f: | |
198 | for line in f: | |
199 | self._parse_line(line) | |
200 | ||
6ffd06b5 MT |
201 | def _parse_block(self, block): |
202 | # Get first line to find out what type of block this is | |
203 | line = block[0] | |
204 | ||
6ffd06b5 | 205 | # aut-num |
429a43d1 | 206 | if line.startswith("aut-num:"): |
6ffd06b5 MT |
207 | return self._parse_autnum_block(block) |
208 | ||
209 | # organisation | |
210 | elif line.startswith("organisation:"): | |
211 | return self._parse_org_block(block) | |
212 | ||
6ffd06b5 | 213 | def _parse_autnum_block(self, block): |
6ffd06b5 MT |
214 | autnum = {} |
215 | for line in block: | |
216 | # Split line | |
217 | key, val = split_line(line) | |
218 | ||
219 | if key == "aut-num": | |
220 | m = re.match(r"^(AS|as)(\d+)", val) | |
221 | if m: | |
222 | autnum["asn"] = m.group(2) | |
223 | ||
0365119d | 224 | elif key == "org": |
6ffd06b5 MT |
225 | autnum[key] = val |
226 | ||
227 | # Skip empty objects | |
228 | if not autnum: | |
229 | return | |
230 | ||
231 | # Insert into database | |
0365119d MT |
232 | self.db.execute("INSERT INTO _autnums(number, organization) \ |
233 | VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \ | |
234 | organization = excluded.organization", | |
235 | autnum.get("asn"), autnum.get("org"), | |
6ffd06b5 MT |
236 | ) |
237 | ||
6ffd06b5 MT |
238 | def _parse_org_block(self, block): |
239 | org = {} | |
240 | for line in block: | |
241 | # Split line | |
242 | key, val = split_line(line) | |
243 | ||
0365119d | 244 | if key in ("organisation", "org-name"): |
6ffd06b5 MT |
245 | org[key] = val |
246 | ||
247 | # Skip empty objects | |
248 | if not org: | |
249 | return | |
250 | ||
0365119d MT |
251 | self.db.execute("INSERT INTO _organizations(handle, name) \ |
252 | VALUES(%s, %s) ON CONFLICT (handle) DO \ | |
253 | UPDATE SET name = excluded.name", | |
254 | org.get("organisation"), org.get("org-name"), | |
6ffd06b5 MT |
255 | ) |
256 | ||
429a43d1 MT |
257 | def _parse_line(self, line): |
258 | # Skip version line | |
259 | if line.startswith("2"): | |
260 | return | |
6ffd06b5 | 261 | |
429a43d1 MT |
262 | # Skip comments |
263 | if line.startswith("#"): | |
264 | return | |
6ffd06b5 | 265 | |
429a43d1 MT |
266 | try: |
267 | registry, country_code, type, line = line.split("|", 3) | |
268 | except: | |
269 | log.warning("Could not parse line: %s" % line) | |
270 | return | |
6ffd06b5 | 271 | |
429a43d1 MT |
272 | # Skip any lines that are for stats only |
273 | if country_code == "*": | |
6ffd06b5 MT |
274 | return |
275 | ||
429a43d1 MT |
276 | if type in ("ipv6", "ipv4"): |
277 | return self._parse_ip_line(country_code, type, line) | |
278 | ||
429a43d1 MT |
279 | def _parse_ip_line(self, country, type, line): |
280 | try: | |
281 | address, prefix, date, status, organization = line.split("|") | |
282 | except ValueError: | |
283 | organization = None | |
284 | ||
285 | # Try parsing the line without organization | |
286 | try: | |
287 | address, prefix, date, status = line.split("|") | |
288 | except ValueError: | |
289 | log.warning("Unhandled line format: %s" % line) | |
290 | return | |
291 | ||
292 | # Skip anything that isn't properly assigned | |
293 | if not status in ("assigned", "allocated"): | |
294 | return | |
295 | ||
296 | # Cast prefix into an integer | |
297 | try: | |
298 | prefix = int(prefix) | |
299 | except: | |
300 | log.warning("Invalid prefix: %s" % prefix) | |
301 | ||
302 | # Fix prefix length for IPv4 | |
303 | if type == "ipv4": | |
304 | prefix = 32 - int(math.log(prefix, 2)) | |
305 | ||
306 | # Try to parse the address | |
307 | try: | |
308 | network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False) | |
309 | except ValueError: | |
310 | log.warning("Invalid IP address: %s" % address) | |
311 | return | |
312 | ||
87b3e102 MT |
313 | self.db.execute("INSERT INTO networks(network, country) \ |
314 | VALUES(%s, %s) ON CONFLICT (network) DO \ | |
315 | UPDATE SET country = excluded.country", | |
316 | "%s" % network, country, | |
6ffd06b5 MT |
317 | ) |
318 | ||
83d61c46 MT |
319 | def handle_update_announcements(self, ns): |
320 | server = ns.server[0] | |
321 | ||
322 | # Pre-compile regular expression for routes | |
323 | #route = re.compile(b"^\*>?\s[\si]?([^\s]+)[.\s]*?(\d+)\si$", re.MULTILINE) | |
324 | route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL) | |
325 | ||
326 | with telnetlib.Telnet(server) as t: | |
327 | # Enable debug mode | |
328 | #if ns.debug: | |
329 | # t.set_debuglevel(10) | |
330 | ||
331 | # Wait for console greeting | |
332 | greeting = t.read_until(b"> ") | |
333 | log.debug(greeting.decode()) | |
334 | ||
335 | # Disable pagination | |
336 | t.write(b"terminal length 0\n") | |
337 | ||
338 | # Wait for the prompt to return | |
339 | t.read_until(b"> ") | |
340 | ||
341 | # Fetch the routing tables | |
342 | with self.db.transaction(): | |
343 | for protocol in ("ipv6", "ipv4"): | |
344 | log.info("Requesting %s routing table" % protocol) | |
345 | ||
346 | # Request the full unicast routing table | |
347 | t.write(b"show bgp %s unicast\n" % protocol.encode()) | |
348 | ||
349 | # Read entire header which ends with "Path" | |
350 | t.read_until(b"Path\r\n") | |
351 | ||
352 | while True: | |
353 | # Try reading a full entry | |
354 | # Those might be broken across multiple lines but ends with i | |
355 | line = t.read_until(b"i\r\n", timeout=5) | |
356 | if not line: | |
357 | break | |
358 | ||
359 | # Show line for debugging | |
360 | #log.debug(repr(line)) | |
361 | ||
362 | # Try finding a route in here | |
363 | m = route.match(line) | |
364 | if m: | |
365 | network, autnum = m.groups() | |
366 | ||
367 | # Convert network to string | |
368 | network = network.decode() | |
369 | ||
370 | # Convert AS number to integer | |
371 | autnum = int(autnum) | |
372 | ||
373 | log.info("Found announcement for %s by %s" % (network, autnum)) | |
374 | ||
375 | self.db.execute("INSERT INTO announcements(network, autnum) \ | |
376 | VALUES(%s, %s) ON CONFLICT (network) DO \ | |
377 | UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP", | |
378 | network, autnum, | |
379 | ) | |
380 | ||
381 | log.info("Finished reading the %s routing table" % protocol) | |
382 | ||
383 | # Purge anything we never want here | |
384 | self.db.execute(""" | |
385 | -- Delete default routes | |
386 | DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0'; | |
387 | ||
388 | -- Delete anything that is not global unicast address space | |
389 | DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3'; | |
390 | ||
391 | -- DELETE RFC1918 address space | |
392 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8'; | |
393 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12'; | |
394 | DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16'; | |
395 | ||
396 | -- Delete networks that are too small to be in the global routing table | |
397 | DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48; | |
398 | DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24; | |
399 | ||
400 | -- Delete any non-public or reserved ASNs | |
401 | DELETE FROM announcements WHERE NOT ( | |
402 | (autnum >= 1 AND autnum <= 23455) | |
403 | OR | |
404 | (autnum >= 23457 AND autnum <= 64495) | |
405 | OR | |
406 | (autnum >= 131072 AND autnum <= 4199999999) | |
407 | ); | |
408 | ||
409 | -- Delete everything that we have not seen for 14 days | |
410 | DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days'; | |
411 | """) | |
412 | ||
d7fc3057 MT |
413 | def handle_update_overrides(self, ns): |
414 | with self.db.transaction(): | |
415 | # Drop all data that we have | |
416 | self.db.execute(""" | |
417 | TRUNCATE TABLE autnum_overrides; | |
418 | TRUNCATE TABLE network_overrides; | |
419 | """) | |
420 | ||
421 | for file in ns.files: | |
422 | log.info("Reading %s..." % file) | |
423 | ||
424 | with open(file, "rb") as f: | |
425 | for type, block in location.importer.read_blocks(f): | |
426 | if type == "net": | |
427 | network = block.get("net") | |
428 | # Try to parse and normalise the network | |
429 | try: | |
430 | network = ipaddress.ip_network(network, strict=False) | |
431 | except ValueError as e: | |
432 | log.warning("Invalid IP network: %s: %s" % (network, e)) | |
433 | continue | |
434 | ||
435 | self.db.execute(""" | |
436 | INSERT INTO network_overrides( | |
437 | network, | |
438 | country, | |
439 | is_anonymous_proxy, | |
440 | is_satellite_provider, | |
441 | is_anycast | |
442 | ) VALUES (%s, %s, %s, %s) | |
443 | ON CONFLICT (network) DO NOTHING""", | |
444 | "%s" % network, | |
445 | block.get("country"), | |
446 | block.get("is-anonymous-proxy") == "yes", | |
447 | block.get("is-satellite-provider") == "yes", | |
448 | block.get("is-anycast") == "yes", | |
449 | ) | |
450 | ||
451 | elif type == "autnum": | |
452 | autnum = block.get("autnum") | |
453 | ||
454 | # Check if AS number begins with "AS" | |
455 | if not autnum.startswith("AS"): | |
456 | log.warning("Invalid AS number: %s" % autnum) | |
457 | continue | |
458 | ||
459 | # Strip "AS" | |
460 | autnum = autnum[2:] | |
461 | ||
462 | self.db.execute(""" | |
463 | INSERT INTO autnum_overrides( | |
464 | number, | |
465 | name, | |
466 | is_anonymous_proxy, | |
467 | is_satellite_provider, | |
468 | is_anycast | |
469 | ) VALUES(%s, %s, %s, %s, %s) | |
470 | ON CONFLICT DO NOTHING""", | |
471 | autnum, block.get("name"), | |
472 | block.get("is-anonymous-proxy") == "yes", | |
473 | block.get("is-satellite-provider") == "yes", | |
474 | block.get("is-anycast") == "yes", | |
475 | ) | |
476 | ||
477 | else: | |
478 | log.warning("Unsupport type: %s" % type) | |
479 | ||
6ffd06b5 MT |
480 | |
481 | def split_line(line): | |
482 | key, colon, val = line.partition(":") | |
483 | ||
484 | # Strip any excess space | |
485 | key = key.strip() | |
486 | val = val.strip() | |
78ff0cf2 | 487 | |
6ffd06b5 | 488 | return key, val |
78ff0cf2 MT |
489 | |
490 | def main(): | |
491 | # Run the command line interface | |
492 | c = CLI() | |
493 | c.run() | |
494 | ||
495 | main() |