]>
git.ipfire.org Git - people/ms/libloc.git/blob - src/python/location-downloader.in
961c5dffdaf631344588ebf04df8ebf1fcb51c9c
2 ###############################################################################
4 # libloc - A library to determine the location of someone on the Internet #
6 # Copyright (C) 2019 IPFire Development Team <info@ipfire.org> #
8 # This library is free software; you can redistribute it and/or #
9 # modify it under the terms of the GNU Lesser General Public #
10 # License as published by the Free Software Foundation; either #
11 # version 2.1 of the License, or (at your option) any later version. #
13 # This library is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
16 # Lesser General Public License for more details. #
18 ###############################################################################
23 import logging
.handlers
35 # Load our location module
38 DATABASE_FILENAME
= "test.db.xz"
40 "https://location.ipfire.org/databases/",
41 "https://people.ipfire.org/~ms/location/",
44 def setup_logging(level
=logging
.INFO
):
45 l
= logging
.getLogger("location-downloader")
49 h
= logging
.StreamHandler()
50 h
.setLevel(logging
.DEBUG
)
54 h
= logging
.handlers
.SysLogHandler(address
="/dev/log",
55 facility
=logging
.handlers
.SysLogHandler
.LOG_DAEMON
)
56 h
.setLevel(logging
.INFO
)
59 # Format syslog messages
60 formatter
= logging
.Formatter("location-downloader[%(process)d]: %(message)s")
61 h
.setFormatter(formatter
)
69 def _(singular
, plural
=None, n
=None):
71 return gettext
.dngettext("libloc", singular
, plural
, n
)
73 return gettext
.dgettext("libloc", singular
)
75 class NotModifiedError(Exception):
77 Raised when the file has not been modified on the server
82 class Downloader(object):
83 def __init__(self
, mirrors
):
84 self
.mirrors
= list(mirrors
)
87 random
.shuffle(self
.mirrors
)
89 # Get proxies from environment
90 self
.proxies
= self
._get
_proxies
()
92 def _get_proxies(self
):
95 for protocol
in ("https", "http"):
96 proxy
= os
.environ
.get("%s_proxy" % protocol
, None)
99 proxies
[protocol
] = proxy
103 def _make_request(self
, url
, baseurl
=None, headers
={}):
105 url
= urllib
.parse
.urljoin(baseurl
, url
)
107 req
= urllib
.request
.Request(url
, method
="GET")
111 "User-Agent" : "location-downloader/%s" % location
.__version
__,
115 for header
in headers
:
116 req
.add_header(header
, headers
[header
])
119 for protocol
in self
.proxies
:
120 req
.set_proxy(self
.proxies
[protocol
], protocol
)
124 def _send_request(self
, req
, **kwargs
):
125 # Log request headers
126 log
.debug("HTTP %s Request to %s" % (req
.method
, req
.host
))
127 log
.debug(" URL: %s" % req
.full_url
)
128 log
.debug(" Headers:")
129 for k
, v
in req
.header_items():
130 log
.debug(" %s: %s" % (k
, v
))
133 res
= urllib
.request
.urlopen(req
, **kwargs
)
135 except urllib
.error
.HTTPError
as e
:
136 # Log response headers
137 log
.debug("HTTP Response: %s" % e
.code
)
138 log
.debug(" Headers:")
139 for header
in e
.headers
:
140 log
.debug(" %s: %s" % (header
, e
.headers
[header
]))
144 raise NotModifiedError() from e
146 # Raise all other errors
149 # Log response headers
150 log
.debug("HTTP Response: %s" % res
.code
)
151 log
.debug(" Headers:")
152 for k
, v
in res
.getheaders():
153 log
.debug(" %s: %s" % (k
, v
))
157 def download(self
, url
, mtime
=None, **kwargs
):
161 headers
["If-Modified-Since"] = time
.strftime(
162 "%a, %d %b %Y %H:%M:%S GMT", time
.gmtime(mtime
),
165 t
= tempfile
.NamedTemporaryFile(delete
=False)
168 for mirror
in self
.mirrors
:
169 # Prepare HTTP request
170 req
= self
._make
_request
(url
, baseurl
=mirror
, headers
=headers
)
173 with self
._send
_request
(req
) as res
:
174 decompressor
= lzma
.LZMADecompressor()
183 buf
= decompressor
.decompress(buf
)
187 # Write all data to disk
190 # Nothing to do when the database on the server is up to date
191 except NotModifiedError
:
192 log
.info("Local database is up to date")
195 # Catch decompression errors
196 except lzma
.LZMAError
as e
:
197 log
.warning("Could not decompress downloaded file: %s" % e
)
200 # XXX what do we catch here?
201 except urllib
.error
.HTTPError
as e
:
205 # Truncate the target file and drop downloaded content
213 # Return temporary file
216 raise FileNotFoundError(url
)
221 self
.downloader
= Downloader(mirrors
=MIRRORS
)
224 parser
= argparse
.ArgumentParser(
225 description
=_("Location Downloader Command Line Interface"),
227 subparsers
= parser
.add_subparsers()
229 # Global configuration flags
230 parser
.add_argument("--debug", action
="store_true",
231 help=_("Enable debug output"))
234 parser
.add_argument("--version", action
="version",
235 version
="%%(prog)s %s" % location
.__version
__)
238 parser
.add_argument("--database", "-d",
239 default
="@databasedir@/database.db", help=_("Path to database"),
243 update
= subparsers
.add_parser("update", help=_("Update database"))
244 update
.set_defaults(func
=self
.handle_update
)
246 args
= parser
.parse_args()
248 # Enable debug logging
250 log
.setLevel(logging
.DEBUG
)
252 # Print usage if no action was given
253 if not "func" in args
:
260 # Parse command line arguments
261 args
= self
.parse_cli()
264 ret
= args
.func(args
)
266 # Return with exit code
270 # Otherwise just exit
273 def handle_update(self
, ns
):
278 db
= location
.Database(ns
.database
)
280 # Get mtime of the old file
281 mtime
= os
.path
.getmtime(ns
.database
)
282 except FileNotFoundError
as e
:
285 # Try downloading a new database
287 t
= self
.downloader
.download(DATABASE_FILENAME
, mtime
=mtime
)
289 # If no file could be downloaded, log a message
290 except FileNotFoundError
as e
:
291 log
.error("Could not download a new database")
294 # If we have not received a new file, there is nothing to do
298 # Save old database creation time
299 created_at
= db
.created_at
if db
else 0
301 # Try opening the downloaded file
303 db
= location
.Database(t
.name
)
304 except Exception as e
:
307 # Check if the downloaded file is newer
308 if db
.created_at
<= created_at
:
309 log
.warning("Downloaded database is older than the current version")
312 log
.info("Downloaded new database from %s" % (time
.strftime(
313 "%a, %d %b %Y %H:%M:%S GMT", time
.gmtime(db
.created_at
),
316 # Write temporary file to destination
317 shutil
.copyfile(t
.name
, ns
.database
)
319 # Remove temporary file
324 # Run the command line interface