]>
Commit | Line | Data |
---|---|---|
9137135a MT |
1 | #!/usr/bin/python |
2 | ||
3 | import datetime | |
4 | import logging | |
5 | import os | |
13b9276e MT |
6 | import pakfire |
7 | import pakfire.config | |
8 | import shutil | |
9137135a | 9 | import subprocess |
13b9276e | 10 | import tempfile |
9137135a | 11 | |
2c909128 | 12 | from . import base |
13b9276e | 13 | from . import builds |
2c909128 | 14 | from . import database |
7ad79344 | 15 | from . import git |
9137135a | 16 | |
78366294 MT |
17 | from .decorators import * |
18 | ||
9137135a | 19 | class Sources(base.Object): |
78366294 MT |
20 | def _get_source(self, query, *args): |
21 | res = self.db.get(query, *args) | |
22 | ||
23 | if res: | |
24 | return Source(self.backend, res.id, data=res) | |
25 | ||
26 | def _get_sources(self, query, *args): | |
27 | res = self.db.query(query, *args) | |
28 | ||
29 | for row in res: | |
30 | yield Source(self.backend, row.id, data=row) | |
31 | ||
32 | def _get_commit(self, query, *args): | |
33 | res = self.db.get(query, *args) | |
34 | ||
35 | if res: | |
36 | return Commit(self.backend, res.id, data=res) | |
37 | ||
38 | def _get_commits(self, query, *args): | |
39 | res = self.db.query(query, *args) | |
9137135a | 40 | |
78366294 MT |
41 | for row in res: |
42 | yield Commit(self.backend, row.id, data=row) | |
43 | ||
44 | def __iter__(self): | |
45 | return self._get_sources("SELECT * FROM sources") | |
9137135a MT |
46 | |
47 | def get_by_id(self, id): | |
78366294 MT |
48 | return self._get_source("SELECT * FROM sources \ |
49 | WHERE id = %s", id) | |
9137135a MT |
50 | |
51 | def get_by_distro(self, distro): | |
78366294 MT |
52 | return self._get_sources("SELECT * FROM sources \ |
53 | WHERE distro_id = %s", distro.id) | |
9137135a MT |
54 | |
55 | def update_revision(self, source_id, revision): | |
56 | query = "UPDATE sources SET revision = %s WHERE id = %s" | |
57 | ||
58 | return self.db.execute(query, revision, source_id) | |
59 | ||
f6e6ff79 MT |
60 | def get_pending_commits(self, limit=None): |
61 | query = "SELECT id FROM sources_commits WHERE state = 'pending' ORDER BY id ASC" | |
62 | args = [] | |
9137135a | 63 | |
f6e6ff79 MT |
64 | if limit: |
65 | query += " LIMIT %s" | |
66 | args.append(limit) | |
67 | ||
68 | rows = self.db.query(query, *args) | |
69 | ||
70 | commits = [] | |
71 | for row in rows: | |
72 | commit = Commit(self.pakfire, row.id) | |
73 | commits.append(commit) | |
74 | ||
75 | return commits | |
76 | ||
77 | def get_commit_by_id(self, commit_id): | |
78 | commit = self.db.get("SELECT id FROM sources_commits WHERE id = %s", commit_id) | |
79 | ||
80 | if commit: | |
81 | return Commit(self.pakfire, commit.id) | |
82 | ||
7ad79344 | 83 | def pull(self): |
78366294 MT |
84 | for source in self: |
85 | repo = git.Repo(self.backend, source, mode="mirror") | |
7ad79344 MT |
86 | |
87 | # If the repository is not yet cloned, we need to make a local | |
88 | # clone to work with. | |
89 | if not repo.cloned: | |
90 | repo.clone() | |
91 | ||
92 | # Otherwise we just fetch updates. | |
93 | else: | |
94 | repo.fetch() | |
95 | ||
96 | # Import all new revisions. | |
97 | repo.import_revisions() | |
98 | ||
13b9276e MT |
99 | def dist(self): |
100 | self._init_repos() | |
101 | ||
102 | for commit in self.get_pending_commits(): | |
103 | commit.state = "running" | |
104 | ||
105 | logging.debug("Processing commit %s: %s" % (commit.revision, commit.subject)) | |
106 | ||
107 | # Get the repository of this commit. | |
108 | repo = git.Repo(self.pakfire, commit.source) | |
109 | ||
110 | # Make sure, it is checked out. | |
111 | if not repo.cloned: | |
112 | repo.clone() | |
113 | ||
114 | # Navigate to the right revision. | |
115 | repo.checkout(commit.revision) | |
116 | ||
117 | # Get all changed makefiles. | |
118 | deleted_files = [] | |
119 | updated_files = [] | |
120 | ||
121 | for file in repo.changed_files(commit.revision): | |
122 | # Don't care about files that are not a makefile. | |
123 | if not file.endswith(".%s" % MAKEFILE_EXTENSION): | |
124 | continue | |
125 | ||
126 | if os.path.exists(file): | |
127 | updated_files.append(file) | |
128 | else: | |
129 | deleted_files.append(file) | |
130 | ||
131 | if updated_files: | |
132 | # Create a temporary directory where to put all the files | |
133 | # that are generated here. | |
134 | pkg_dir = tempfile.mkdtemp() | |
135 | ||
136 | try: | |
137 | config = pakfire.config.Config(["general.conf",]) | |
138 | config.parse(source.distro.get_config()) | |
139 | ||
140 | p = pakfire.PakfireServer(config=config) | |
141 | ||
142 | pkgs = [] | |
143 | for file in updated_files: | |
144 | try: | |
145 | pkg_file = p.dist(file, pkg_dir) | |
146 | pkgs.append(pkg_file) | |
147 | except: | |
148 | raise | |
149 | ||
150 | # Import all packages in one swoop. | |
151 | for pkg in pkgs: | |
152 | # Import the package file and create a build out of it. | |
153 | builds.import_from_package(_pakfire, pkg, | |
154 | distro=source.distro, commit=commit, type="release") | |
155 | ||
156 | except: | |
157 | if commit: | |
158 | commit.state = "failed" | |
159 | ||
160 | raise | |
161 | ||
162 | finally: | |
163 | if os.path.exists(pkg_dir): | |
164 | shutil.rmtree(pkg_dir) | |
165 | ||
166 | for file in deleted_files: | |
167 | # Determine the name of the package. | |
168 | name = os.path.basename(file) | |
169 | name = name[:len(MAKEFILE_EXTENSION) + 1] | |
170 | ||
171 | source.distro.delete_package(name) | |
172 | ||
173 | if commit: | |
174 | commit.state = "finished" | |
175 | ||
176 | def _init_repos(self): | |
177 | """ | |
178 | Initialize all repositories. | |
179 | """ | |
180 | for source in self.get_all(): | |
181 | # Skip those which already have a revision. | |
182 | if source.revision: | |
183 | continue | |
184 | ||
185 | # Initialize the repository or and clone it if necessary. | |
186 | repo = git.Repo(self.pakfire, source) | |
187 | if not repo.cloned: | |
188 | repo.clone() | |
189 | ||
f6e6ff79 | 190 | |
78366294 MT |
191 | class Commit(base.DataObject): |
192 | table = "sources_commits" | |
9137135a | 193 | |
f6e6ff79 MT |
194 | @property |
195 | def revision(self): | |
196 | return self.data.revision | |
197 | ||
78366294 | 198 | @lazy_property |
f6e6ff79 | 199 | def source(self): |
78366294 | 200 | return self.backend.sources.get_by_id(self.data.source_id) |
f6e6ff79 MT |
201 | |
202 | @property | |
203 | def distro(self): | |
204 | """ | |
205 | A shortcut to the distribution this commit | |
206 | belongs to. | |
207 | """ | |
208 | return self.source.distro | |
9137135a | 209 | |
f6e6ff79 | 210 | def set_state(self, state): |
78366294 | 211 | self._set_attribute("state", state) |
9137135a | 212 | |
78366294 | 213 | state = property(lambda s: s.data.state, set_state) |
9137135a | 214 | |
f6e6ff79 MT |
215 | @property |
216 | def author(self): | |
217 | return self.data.author | |
218 | ||
219 | @property | |
220 | def committer(self): | |
221 | return self.data.committer | |
222 | ||
223 | @property | |
224 | def subject(self): | |
b9d096e0 | 225 | return self.data.subject.strip() |
f6e6ff79 MT |
226 | |
227 | @property | |
228 | def message(self): | |
229 | return self.data.body.strip() | |
230 | ||
4b1e87c4 MT |
231 | @property |
232 | def message_full(self): | |
233 | msg = [self.subject, ""] + self.message.splitlines() | |
234 | ||
235 | return "\n".join(msg) | |
236 | ||
f6e6ff79 MT |
237 | @property |
238 | def date(self): | |
239 | return self.data.date | |
240 | ||
78366294 | 241 | @lazy_property |
f6e6ff79 | 242 | def packages(self): |
78366294 MT |
243 | return self.backend.packages._get_packages("SELECT * FROM packages \ |
244 | WHERE commit_id = %s", self.id) | |
f6e6ff79 MT |
245 | |
246 | def reset(self): | |
9137135a | 247 | """ |
f6e6ff79 MT |
248 | Removes all packages that have been created by this commit and |
249 | resets the state so it will be processed again. | |
9137135a | 250 | """ |
f6e6ff79 MT |
251 | # Remove all packages and corresponding builds. |
252 | for pkg in self.packages: | |
253 | # Check if there is a build associated with the package. | |
254 | # If so, the whole build will be deleted. | |
255 | if pkg.build: | |
256 | pkg.build.delete() | |
9137135a | 257 | |
f6e6ff79 MT |
258 | else: |
259 | # Delete the package. | |
260 | pkg.delete() | |
261 | ||
262 | # Clear the cache. | |
78366294 | 263 | del self.packages |
9137135a | 264 | |
f6e6ff79 MT |
265 | # Reset the state to 'pending'. |
266 | self.state = "pending" | |
9137135a | 267 | |
9137135a | 268 | |
78366294 MT |
269 | class Source(base.DataObject): |
270 | table = "sources" | |
f6e6ff79 | 271 | |
78366294 MT |
272 | def __eq__(self, other): |
273 | return self.id == other.id | |
f6e6ff79 | 274 | |
78366294 MT |
275 | def __len__(self): |
276 | ret = self.db.get("SELECT COUNT(*) AS len FROM sources_commits \ | |
277 | WHERE source_id = %s", self.id) | |
f6e6ff79 | 278 | |
78366294 | 279 | return ret.len |
9137135a | 280 | |
78366294 MT |
281 | def create_commit(self, revision, author, committer, subject, body, date): |
282 | commit = self.backend.sources._get_commit("INSERT INTO sources_commits(source_id, \ | |
86d8598e MT |
283 | revision, author, committer, subject, body, date) VALUES(%s, %s, %s, %s, %s, %s, %s) \ |
284 | RETURNING *", self.id, revision, author, committer, subject, body, date) | |
9137135a | 285 | |
78366294 MT |
286 | # Commit |
287 | commit.source = self | |
9137135a | 288 | |
78366294 | 289 | return commit |
9137135a | 290 | |
f6e6ff79 MT |
291 | @property |
292 | def info(self): | |
293 | return { | |
294 | "id" : self.id, | |
295 | "name" : self.name, | |
296 | "url" : self.url, | |
297 | "path" : self.path, | |
298 | "targetpath" : self.targetpath, | |
299 | "revision" : self.revision, | |
300 | "branch" : self.branch, | |
301 | } | |
9137135a | 302 | |
9137135a MT |
303 | @property |
304 | def name(self): | |
f6e6ff79 | 305 | return self.data.name |
9137135a MT |
306 | |
307 | @property | |
f6e6ff79 MT |
308 | def identifier(self): |
309 | return self.data.identifier | |
9137135a MT |
310 | |
311 | @property | |
f6e6ff79 MT |
312 | def url(self): |
313 | return self.data.url | |
9137135a MT |
314 | |
315 | @property | |
f6e6ff79 MT |
316 | def gitweb(self): |
317 | return self.data.gitweb | |
9137135a MT |
318 | |
319 | @property | |
320 | def revision(self): | |
f6e6ff79 | 321 | return self.data.revision |
9137135a MT |
322 | |
323 | @property | |
324 | def branch(self): | |
f6e6ff79 | 325 | return self.data.branch |
9137135a MT |
326 | |
327 | @property | |
328 | def builds(self): | |
329 | return self.pakfire.builds.get_by_source(self.id) | |
330 | ||
78366294 | 331 | @lazy_property |
9137135a | 332 | def distro(self): |
f6e6ff79 MT |
333 | return self.pakfire.distros.get_by_id(self.data.distro_id) |
334 | ||
335 | @property | |
336 | def start_revision(self): | |
337 | return self.data.revision | |
338 | ||
78366294 | 339 | @lazy_property |
f6e6ff79 | 340 | def head_revision(self): |
a7d966df | 341 | return self.backend.sources._get_commit("SELECT * FROM sources_commits \ |
78366294 | 342 | WHERE source_id = %s ORDER BY id DESC LIMIT 1", self.id) |
f6e6ff79 MT |
343 | |
344 | def get_commits(self, limit=None, offset=None): | |
a7d966df | 345 | return self.backend.sources._get_commits("SELECT * FROM sources_commits \ |
78366294 | 346 | WHERE source_id = %s ORDER BY id DESC LIMIT %s OFFSET %s", limit, offset) |
f6e6ff79 MT |
347 | |
348 | def get_commit(self, revision): | |
a7d966df | 349 | commit = self.backend.sources._get_commit("SELECT * FROM sources_commits \ |
78366294 | 350 | WHERE source_id = %s AND revision = %s", self.id, revision) |
f6e6ff79 | 351 | |
78366294 MT |
352 | if commit: |
353 | commit.source = self | |
354 | return commit |