]>
Commit | Line | Data |
---|---|---|
244a3b61 MT |
1 | #!/usr/bin/python3 |
2 | ############################################################################### | |
3 | # # | |
4 | # libloc - A library to determine the location of someone on the Internet # | |
5 | # # | |
6 | # Copyright (C) 2019 IPFire Development Team <info@ipfire.org> # | |
7 | # # | |
8 | # This library is free software; you can redistribute it and/or # | |
9 | # modify it under the terms of the GNU Lesser General Public # | |
10 | # License as published by the Free Software Foundation; either # | |
11 | # version 2.1 of the License, or (at your option) any later version. # | |
12 | # # | |
13 | # This library is distributed in the hope that it will be useful, # | |
14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of # | |
15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # | |
16 | # Lesser General Public License for more details. # | |
17 | # # | |
18 | ############################################################################### | |
19 | ||
20 | import argparse | |
f4fef543 | 21 | import datetime |
5a9b4c77 | 22 | import logging |
244a3b61 MT |
23 | import lzma |
24 | import os | |
25 | import random | |
26 | import shutil | |
679e5ae2 | 27 | import stat |
244a3b61 MT |
28 | import sys |
29 | import tempfile | |
30 | import time | |
31 | import urllib.error | |
32 | import urllib.parse | |
33 | import urllib.request | |
34 | ||
35 | # Load our location module | |
36 | import location | |
7dccb767 | 37 | from location.i18n import _ |
244a3b61 | 38 | |
8881d5f4 | 39 | DATABASE_FILENAME = "location.db.xz" |
244a3b61 MT |
40 | MIRRORS = ( |
41 | "https://location.ipfire.org/databases/", | |
244a3b61 MT |
42 | ) |
43 | ||
5a9b4c77 | 44 | # Initialise logging |
e44b30f4 MT |
45 | log = logging.getLogger("location.downloader") |
46 | log.propagate = 1 | |
5a9b4c77 | 47 | |
244a3b61 | 48 | class Downloader(object): |
3ee36b5e MT |
49 | def __init__(self, version, mirrors): |
50 | self.version = version | |
244a3b61 MT |
51 | self.mirrors = list(mirrors) |
52 | ||
53 | # Randomize mirrors | |
54 | random.shuffle(self.mirrors) | |
55 | ||
56 | # Get proxies from environment | |
57 | self.proxies = self._get_proxies() | |
58 | ||
59 | def _get_proxies(self): | |
60 | proxies = {} | |
61 | ||
62 | for protocol in ("https", "http"): | |
63 | proxy = os.environ.get("%s_proxy" % protocol, None) | |
64 | ||
65 | if proxy: | |
66 | proxies[protocol] = proxy | |
67 | ||
68 | return proxies | |
69 | ||
70 | def _make_request(self, url, baseurl=None, headers={}): | |
71 | if baseurl: | |
72 | url = urllib.parse.urljoin(baseurl, url) | |
73 | ||
74 | req = urllib.request.Request(url, method="GET") | |
75 | ||
76 | # Update headers | |
77 | headers.update({ | |
d2714e4a | 78 | "User-Agent" : "location-downloader/@VERSION@", |
244a3b61 MT |
79 | }) |
80 | ||
81 | # Set headers | |
82 | for header in headers: | |
83 | req.add_header(header, headers[header]) | |
84 | ||
85 | # Set proxies | |
86 | for protocol in self.proxies: | |
87 | req.set_proxy(self.proxies[protocol], protocol) | |
88 | ||
89 | return req | |
90 | ||
91 | def _send_request(self, req, **kwargs): | |
92 | # Log request headers | |
5a9b4c77 MT |
93 | log.debug("HTTP %s Request to %s" % (req.method, req.host)) |
94 | log.debug(" URL: %s" % req.full_url) | |
95 | log.debug(" Headers:") | |
244a3b61 | 96 | for k, v in req.header_items(): |
5a9b4c77 | 97 | log.debug(" %s: %s" % (k, v)) |
244a3b61 MT |
98 | |
99 | try: | |
100 | res = urllib.request.urlopen(req, **kwargs) | |
101 | ||
102 | except urllib.error.HTTPError as e: | |
103 | # Log response headers | |
5a9b4c77 MT |
104 | log.debug("HTTP Response: %s" % e.code) |
105 | log.debug(" Headers:") | |
244a3b61 | 106 | for header in e.headers: |
5a9b4c77 | 107 | log.debug(" %s: %s" % (header, e.headers[header])) |
244a3b61 | 108 | |
244a3b61 MT |
109 | # Raise all other errors |
110 | raise e | |
111 | ||
112 | # Log response headers | |
5a9b4c77 MT |
113 | log.debug("HTTP Response: %s" % res.code) |
114 | log.debug(" Headers:") | |
244a3b61 | 115 | for k, v in res.getheaders(): |
5a9b4c77 | 116 | log.debug(" %s: %s" % (k, v)) |
244a3b61 MT |
117 | |
118 | return res | |
119 | ||
679e5ae2 | 120 | def download(self, url, public_key, timestamp=None, tmpdir=None, **kwargs): |
244a3b61 MT |
121 | headers = {} |
122 | ||
f4fef543 MT |
123 | if timestamp: |
124 | headers["If-Modified-Since"] = timestamp.strftime( | |
125 | "%a, %d %b %Y %H:%M:%S GMT", | |
244a3b61 MT |
126 | ) |
127 | ||
679e5ae2 | 128 | t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) |
244a3b61 MT |
129 | with t: |
130 | # Try all mirrors | |
131 | for mirror in self.mirrors: | |
132 | # Prepare HTTP request | |
133 | req = self._make_request(url, baseurl=mirror, headers=headers) | |
134 | ||
135 | try: | |
136 | with self._send_request(req) as res: | |
137 | decompressor = lzma.LZMADecompressor() | |
138 | ||
139 | # Read all data | |
140 | while True: | |
141 | buf = res.read(1024) | |
142 | if not buf: | |
143 | break | |
144 | ||
145 | # Decompress data | |
146 | buf = decompressor.decompress(buf) | |
147 | if buf: | |
148 | t.write(buf) | |
149 | ||
f4fef543 MT |
150 | # Write all data to disk |
151 | t.flush() | |
244a3b61 MT |
152 | |
153 | # Catch decompression errors | |
154 | except lzma.LZMAError as e: | |
5a9b4c77 | 155 | log.warning("Could not decompress downloaded file: %s" % e) |
244a3b61 MT |
156 | continue |
157 | ||
244a3b61 | 158 | except urllib.error.HTTPError as e: |
f4fef543 MT |
159 | # The file on the server was too old |
160 | if e.code == 304: | |
161 | log.warning("%s is serving an outdated database. Trying next mirror..." % mirror) | |
244a3b61 | 162 | |
f4fef543 MT |
163 | # Log any other HTTP errors |
164 | else: | |
165 | log.warning("%s reported: %s" % (mirror, e)) | |
166 | ||
167 | # Throw away any downloaded content and try again | |
168 | t.truncate() | |
244a3b61 | 169 | |
f4fef543 MT |
170 | else: |
171 | # Check if the downloaded database is recent | |
116b1352 | 172 | if not self._check_database(t, public_key, timestamp): |
f4fef543 | 173 | log.warning("Downloaded database is outdated. Trying next mirror...") |
244a3b61 | 174 | |
f4fef543 MT |
175 | # Throw away the data and try again |
176 | t.truncate() | |
177 | continue | |
178 | ||
679e5ae2 MT |
179 | # Make the file readable for everyone |
180 | os.chmod(t.name, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH) | |
181 | ||
f4fef543 MT |
182 | # Return temporary file |
183 | return t | |
244a3b61 MT |
184 | |
185 | raise FileNotFoundError(url) | |
186 | ||
116b1352 | 187 | def _check_database(self, f, public_key, timestamp=None): |
f4fef543 MT |
188 | """ |
189 | Checks the downloaded database if it can be opened, | |
190 | verified and if it is recent enough | |
191 | """ | |
192 | log.debug("Opening downloaded database at %s" % f.name) | |
193 | ||
194 | db = location.Database(f.name) | |
195 | ||
196 | # Database is not recent | |
197 | if timestamp and db.created_at < timestamp.timestamp(): | |
198 | return False | |
199 | ||
200 | log.info("Downloaded new database from %s" % (time.strftime( | |
201 | "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at), | |
202 | ))) | |
203 | ||
116b1352 MT |
204 | # Verify the database |
205 | with open(public_key, "r") as f: | |
206 | if not db.verify(f): | |
207 | log.error("Could not verify database") | |
208 | return False | |
209 | ||
f4fef543 MT |
210 | return True |
211 | ||
244a3b61 MT |
212 | |
213 | class CLI(object): | |
214 | def __init__(self): | |
3ee36b5e MT |
215 | # Which version are we downloading? |
216 | self.version = location.DATABASE_VERSION_LATEST | |
217 | ||
218 | self.downloader = Downloader(version=self.version, mirrors=MIRRORS) | |
244a3b61 MT |
219 | |
220 | def parse_cli(self): | |
221 | parser = argparse.ArgumentParser( | |
222 | description=_("Location Downloader Command Line Interface"), | |
223 | ) | |
224 | subparsers = parser.add_subparsers() | |
225 | ||
226 | # Global configuration flags | |
227 | parser.add_argument("--debug", action="store_true", | |
228 | help=_("Enable debug output")) | |
bc1f5f53 MT |
229 | parser.add_argument("--quiet", action="store_true", |
230 | help=_("Enable quiet mode")) | |
244a3b61 MT |
231 | |
232 | # version | |
233 | parser.add_argument("--version", action="version", | |
d2714e4a | 234 | version="%(prog)s @VERSION@") |
244a3b61 MT |
235 | |
236 | # database | |
237 | parser.add_argument("--database", "-d", | |
238 | default="@databasedir@/database.db", help=_("Path to database"), | |
239 | ) | |
240 | ||
116b1352 MT |
241 | # public key |
242 | parser.add_argument("--public-key", "-k", | |
243 | default="@databasedir@/signing-key.pem", help=_("Public Signing Key"), | |
244 | ) | |
245 | ||
244a3b61 MT |
246 | # Update |
247 | update = subparsers.add_parser("update", help=_("Update database")) | |
248 | update.set_defaults(func=self.handle_update) | |
249 | ||
8e753500 MT |
250 | # Verify |
251 | verify = subparsers.add_parser("verify", | |
252 | help=_("Verify the downloaded database")) | |
253 | verify.set_defaults(func=self.handle_verify) | |
254 | ||
244a3b61 MT |
255 | args = parser.parse_args() |
256 | ||
bc1f5f53 | 257 | # Configure logging |
244a3b61 | 258 | if args.debug: |
f9de5e61 | 259 | location.logger.set_level(logging.DEBUG) |
bc1f5f53 MT |
260 | elif args.quiet: |
261 | location.logger.set_level(logging.WARNING) | |
244a3b61 MT |
262 | |
263 | # Print usage if no action was given | |
264 | if not "func" in args: | |
265 | parser.print_usage() | |
266 | sys.exit(2) | |
267 | ||
268 | return args | |
269 | ||
270 | def run(self): | |
271 | # Parse command line arguments | |
272 | args = self.parse_cli() | |
273 | ||
274 | # Call function | |
275 | ret = args.func(args) | |
276 | ||
277 | # Return with exit code | |
278 | if ret: | |
279 | sys.exit(ret) | |
280 | ||
281 | # Otherwise just exit | |
282 | sys.exit(0) | |
283 | ||
284 | def handle_update(self, ns): | |
3ee36b5e MT |
285 | # Fetch the timestamp we need from DNS |
286 | t = location.discover_latest_version(self.version) | |
f4fef543 MT |
287 | |
288 | # Parse timestamp into datetime format | |
3ee36b5e | 289 | timestamp = datetime.datetime.fromtimestamp(t) if t else None |
244a3b61 MT |
290 | |
291 | # Open database | |
292 | try: | |
293 | db = location.Database(ns.database) | |
294 | ||
f4fef543 | 295 | # Check if we are already on the latest version |
3ee36b5e | 296 | if timestamp and db.created_at >= timestamp.timestamp(): |
f4fef543 MT |
297 | log.info("Already on the latest version") |
298 | return | |
299 | ||
244a3b61 MT |
300 | except FileNotFoundError as e: |
301 | db = None | |
302 | ||
679e5ae2 MT |
303 | # Download the database into the correct directory |
304 | tmpdir = os.path.dirname(ns.database) | |
305 | ||
244a3b61 MT |
306 | # Try downloading a new database |
307 | try: | |
3ee36b5e | 308 | t = self.downloader.download("%s/%s" % (self.version, DATABASE_FILENAME), |
679e5ae2 | 309 | public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir) |
244a3b61 MT |
310 | |
311 | # If no file could be downloaded, log a message | |
312 | except FileNotFoundError as e: | |
5a9b4c77 | 313 | log.error("Could not download a new database") |
244a3b61 MT |
314 | return 1 |
315 | ||
316 | # If we have not received a new file, there is nothing to do | |
317 | if not t: | |
f47a500f | 318 | return 3 |
244a3b61 | 319 | |
679e5ae2 MT |
320 | # Move temporary file to destination |
321 | shutil.move(t.name, ns.database) | |
244a3b61 | 322 | |
8e753500 MT |
323 | return 0 |
324 | ||
325 | def handle_verify(self, ns): | |
326 | try: | |
327 | db = location.Database(ns.database) | |
328 | except FileNotFoundError as e: | |
329 | log.error("%s: %s" % (ns.database, e)) | |
330 | return 127 | |
331 | ||
332 | # Verify the database | |
333 | with open(ns.public_key, "r") as f: | |
334 | if not db.verify(f): | |
335 | log.error("Could not verify database") | |
336 | return 1 | |
337 | ||
338 | # Success | |
339 | log.debug("Database successfully verified") | |
340 | return 0 | |
f47a500f | 341 | |
244a3b61 MT |
342 | |
343 | def main(): | |
344 | # Run the command line interface | |
345 | c = CLI() | |
346 | c.run() | |
347 | ||
348 | main() |