]>
Commit | Line | Data |
---|---|---|
940227cb MT |
1 | #!/usr/bin/python |
2 | ||
3 | import logging | |
54af860e | 4 | import os.path |
940227cb MT |
5 | import socket |
6 | import time | |
7 | import tornado.httpclient | |
8 | ||
9 | from databases import Databases | |
10 | from geoip import GeoIP | |
11 | from misc import Singleton | |
12 | ||
13 | class Mirrors(object): | |
14 | __metaclass__ = Singleton | |
15 | ||
16 | @property | |
17 | def db(self): | |
18 | return Databases().webapp | |
19 | ||
20 | def list(self): | |
21 | return [Mirror(m.id) for m in self.db.query("SELECT id FROM mirrors ORDER BY state")] | |
22 | ||
23 | def check_all(self): | |
24 | for mirror in self.list(): | |
25 | mirror.check() | |
26 | ||
27 | def get(self, id): | |
54af860e | 28 | return Mirror(id) |
940227cb MT |
29 | |
30 | def get_by_hostname(self, hostname): | |
31 | mirror = self.db.get("SELECT id FROM mirrors WHERE hostname=%s", hostname) | |
32 | ||
33 | return Mirror(mirror.id) | |
34 | ||
54af860e MT |
35 | def get_with_file(self, filename, country=None): |
36 | # XXX quick and dirty solution - needs a performance boost | |
37 | mirror_ids = [m.mirror for m in self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)] | |
38 | ||
39 | #if country: | |
40 | # # Sort out all mirrors that are not preferred to the given country | |
41 | # for mirror in self.get_for_country(country): | |
42 | # if not mirror.id in mirror_ids: | |
43 | # mirror_ids.remove(mirror.id) | |
44 | ||
45 | mirrors = [] | |
46 | for mirror_id in mirror_ids: | |
47 | mirror = self.get(mirror_id) | |
48 | if not mirror.state == "UP": | |
49 | continue | |
50 | mirrors.append(mirror) | |
51 | ||
52 | logging.debug("%s" % mirrors) | |
53 | ||
54 | return mirrors | |
55 | ||
56 | def get_for_country(self, country): | |
57 | # XXX need option for random order | |
58 | mirrors = self.db.query("SELECT id FROM mirrors WHERE prefer_for_countries LIKE %s", country) | |
59 | ||
60 | for mirror in mirrors: | |
61 | yield self.get(mirror.id) | |
940227cb MT |
62 | |
63 | ||
64 | class Mirror(object): | |
65 | def __init__(self, id): | |
66 | self.id = id | |
67 | ||
68 | self.reload() | |
69 | ||
54af860e MT |
70 | def __repr__(self): |
71 | return "<%s %s>" % (self.__class__.__name__, self.url) | |
72 | ||
73 | def __cmp__(self, other): | |
74 | return cmp(self.id, other.id) | |
75 | ||
940227cb MT |
76 | @property |
77 | def db(self): | |
78 | return Databases().webapp | |
79 | ||
80 | def reload(self): | |
81 | self._info = self.db.get("SELECT * FROM mirrors WHERE id=%s", self.id) | |
82 | self._info["url"] = self.generate_url() | |
83 | ||
84 | def generate_url(self): | |
85 | url = "http://%s" % self.hostname | |
86 | if not self.path.startswith("/"): | |
87 | url += "/" | |
88 | url += "%s" % self.path | |
89 | if not self.path.endswith("/"): | |
90 | url += "/" | |
91 | return url | |
92 | ||
93 | def __getattr__(self, key): | |
94 | try: | |
95 | return self._info[key] | |
96 | except KeyError: | |
97 | raise AttributeError(key) | |
98 | ||
99 | @property | |
100 | def address(self): | |
101 | return socket.gethostbyname(self.hostname) | |
102 | ||
103 | @property | |
104 | def country_code(self): | |
105 | return GeoIP().get_country(self.address).lower() or "unknown" | |
106 | ||
107 | @property | |
108 | def filelist(self): | |
109 | filelist = self.db.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self.id) | |
110 | return [f.filename for f in filelist] | |
111 | ||
54af860e MT |
112 | @property |
113 | def prefix(self): | |
114 | if self.type.startswith("pakfire"): | |
115 | return self.type | |
116 | ||
117 | return "" | |
118 | ||
940227cb MT |
119 | def set_state(self, state): |
120 | logging.info("Setting state of %s to %s" % (self.hostname, state)) | |
121 | ||
122 | if self.state == state: | |
123 | return | |
124 | ||
125 | self.db.execute("UPDATE mirrors SET state=%s WHERE id=%s", | |
126 | state, self.id) | |
127 | ||
128 | # Reload changed settings | |
129 | self.reload() | |
130 | ||
131 | def check(self): | |
132 | logging.info("Running check for mirror %s" % self.hostname) | |
133 | ||
134 | self.check_timestamp() | |
135 | self.check_filelist() | |
136 | ||
137 | def check_state(self): | |
138 | logging.debug("Checking state of mirror %s" % self.id) | |
139 | ||
140 | if self.disabled == "Y": | |
141 | self.set_state("DOWN") | |
142 | ||
143 | time_diff = time.time() - self.last_update | |
144 | if time_diff > 3*24*60*60: # XXX get this into Settings | |
145 | self.set_state("DOWN") | |
146 | elif time_diff > 6*60*60: | |
147 | self.set_state("OUTOFSYNC") | |
148 | else: | |
149 | self.set_state("UP") | |
150 | ||
151 | def check_timestamp(self): | |
152 | if self.releases == "N": | |
153 | return | |
154 | ||
155 | http = tornado.httpclient.AsyncHTTPClient() | |
156 | ||
157 | http.fetch(self.url + ".timestamp", | |
54af860e | 158 | headers={ "Pragma" : "no-cache" }, |
940227cb MT |
159 | callback=self.__check_timestamp_response) |
160 | ||
161 | def __check_timestamp_response(self, response): | |
162 | if response.error: | |
163 | logging.debug("Error getting timestamp from %s" % self.hostname) | |
164 | return | |
165 | ||
166 | try: | |
167 | timestamp = int(response.body.strip()) | |
168 | except ValueError: | |
169 | timestamp = 0 | |
170 | ||
171 | self.db.execute("UPDATE mirrors SET last_update=%s WHERE id=%s", | |
172 | timestamp, self.id) | |
173 | ||
174 | # Reload changed settings | |
175 | self.reload() | |
176 | ||
177 | self.check_state() | |
178 | ||
179 | logging.info("Successfully updated timestamp from %s" % self.hostname) | |
180 | ||
181 | def check_filelist(self): | |
54af860e MT |
182 | # XXX need to remove data from disabled mirrors |
183 | if self.releases == "N" or self.disabled == "Y" or self.type != "full": | |
940227cb MT |
184 | return |
185 | ||
186 | http = tornado.httpclient.AsyncHTTPClient() | |
187 | ||
188 | http.fetch(self.url + ".filelist", | |
54af860e | 189 | headers={ "Pragma" : "no-cache" }, |
940227cb MT |
190 | callback=self.__check_filelist_response) |
191 | ||
192 | def __check_filelist_response(self, response): | |
193 | if response.error: | |
194 | logging.debug("Error getting timestamp from %s" % self.hostname) | |
195 | return | |
196 | ||
197 | self.db.execute("DELETE FROM mirror_files WHERE mirror=%s", self.id) | |
198 | ||
199 | for file in response.body.splitlines(): | |
200 | self.db.execute("INSERT INTO mirror_files(mirror, filename) VALUES(%s, %s)", | |
54af860e | 201 | self.id, os.path.join(self.prefix, file)) |
940227cb MT |
202 | |
203 | logging.info("Successfully updated mirror filelist from %s" % self.hostname) | |
204 | ||
54af860e MT |
205 | @property |
206 | def prefer_for_countries(self): | |
207 | return self._info.get("prefer_for_countries", "").split() | |
208 | ||
209 | ||
940227cb MT |
210 | |
211 | if __name__ == "__main__": | |
212 | m = Mirrors() | |
213 | ||
214 | for mirror in m.list(): | |
215 | print mirror.hostname, mirror.country_code |