]> git.ipfire.org Git - people/jschlag/pbs.git/blob - src/hub/handlers.py
jobs: Remove deps to type field
[people/jschlag/pbs.git] / src / hub / handlers.py
1 #!/usr/bin/python
2
3 import base64
4 import hashlib
5 import json
6 import logging
7 import time
8 import tornado.web
9
10 from .. import builds
11 from .. import builders
12 from .. import uploads
13 from .. import users
14
15 class LongPollMixin(object):
16 def initialize(self):
17 self._start_time = time.time()
18
19 def add_timeout(self, timeout, callback):
20 deadline = time.time() + timeout
21
22 return self.application.ioloop.add_timeout(deadline, callback)
23
24 def on_connection_close(self):
25 logging.debug("Connection closed unexpectedly")
26
27 def connection_closed(self):
28 return self.request.connection.stream.closed()
29
30 @property
31 def runtime(self):
32 return time.time() - self._start_time
33
34
35 class BaseHandler(LongPollMixin, tornado.web.RequestHandler):
36 @property
37 def backend(self):
38 """
39 Shortcut handler to pakfire instance
40 """
41 return self.application.backend
42
43 def get_basic_auth_credentials(self):
44 """
45 This handles HTTP Basic authentication.
46 """
47 auth_header = self.request.headers.get("Authorization", None)
48
49 # If no authentication information was provided, we stop here.
50 if not auth_header:
51 return None, None
52
53 # No basic auth? We cannot handle that.
54 if not auth_header.startswith("Basic "):
55 raise tornado.web.HTTPError(400, "Can only handle Basic auth.")
56
57 try:
58 # Decode the authentication information.
59 auth_header = base64.decodestring(auth_header[6:])
60
61 name, password = auth_header.split(":", 1)
62 except:
63 raise tornado.web.HTTPError(400, "Authorization data was malformed")
64
65 return name, password
66
67 def get_current_user(self):
68 name, password = self.get_basic_auth_credentials()
69 if name is None:
70 return
71
72 builder = self.backend.builders.auth(name, password)
73 if builder:
74 return builder
75
76 user = self.backend.users.auth(name, password)
77 if user:
78 return user
79
80 @property
81 def builder(self):
82 if isinstance(self.current_user, builders.Builder):
83 return self.current_user
84
85 @property
86 def user(self):
87 if isinstance(self.current_user, users.User):
88 return self.current_user
89
90 def get_argument_int(self, *args, **kwargs):
91 arg = self.get_argument(*args, **kwargs)
92
93 try:
94 return int(arg)
95 except (TypeError, ValueError):
96 return None
97
98 def get_argument_float(self, *args, **kwargs):
99 arg = self.get_argument(*args, **kwargs)
100
101 try:
102 return float(arg)
103 except (TypeError, ValueError):
104 return None
105
106 def get_argument_json(self, *args, **kwargs):
107 arg = self.get_argument(*args, **kwargs)
108
109 if arg:
110 return json.loads(arg)
111
112
113 class NoopHandler(BaseHandler):
114 def get(self):
115 if self.builder:
116 self.write("Welcome to the Pakfire hub, %s!" % self.builder.hostname)
117 elif self.user:
118 self.write("Welcome to the Pakfire hub, %s!" % self.user.name)
119 else:
120 self.write("Welcome to the Pakfire hub!")
121
122
123 class ErrorTestHandler(BaseHandler):
124 def get(self, error_code=200):
125 """
126 For testing a client.
127
128 This just returns a HTTP response with the given code.
129 """
130 try:
131 error_code = int(error_code)
132 except ValueError:
133 error_code = 200
134
135 raise tornado.web.HTTPError(error_code)
136
137
138 # Uploads
139
140 class UploadsCreateHandler(BaseHandler):
141 """
142 Create a new upload object in the database and return a unique ID
143 to the uploader.
144 """
145
146 @tornado.web.authenticated
147 def get(self):
148 # XXX Check permissions
149
150 filename = self.get_argument("filename")
151 filesize = self.get_argument_int("filesize")
152 filehash = self.get_argument("hash")
153
154 upload = uploads.Upload.create(self.backend, filename, filesize,
155 filehash, user=self.user, builder=self.builder)
156
157 self.finish(upload.uuid)
158
159
160 class UploadsSendChunkHandler(BaseHandler):
161 @tornado.web.authenticated
162 def post(self, upload_id):
163 upload = self.backend.uploads.get_by_uuid(upload_id)
164 if not upload:
165 raise tornado.web.HTTPError(404, "Invalid upload id.")
166
167 if not upload.builder == self.builder:
168 raise tornado.web.HTTPError(403, "Uploading an other host's file.")
169
170 chksum = self.get_argument("chksum")
171 data = self.get_argument("data")
172
173 # Decode data.
174 data = base64.b64decode(data)
175
176 # Calculate hash and compare.
177 h = hashlib.new("sha512")
178 h.update(data)
179
180 if not chksum == h.hexdigest():
181 raise tornado.web.HTTPError(400, "Checksum mismatch")
182
183 # Append the data to file.
184 upload.append(data)
185
186
187 class UploadsFinishedHandler(BaseHandler):
188 @tornado.web.authenticated
189 def get(self, upload_id):
190 upload = self.backend.uploads.get_by_uuid(upload_id)
191 if not upload:
192 raise tornado.web.HTTPError(404, "Invalid upload id.")
193
194 if not upload.builder == self.builder:
195 raise tornado.web.HTTPError(403, "Uploading an other host's file.")
196
197 # Validate the uploaded data to its hash.
198 ret = upload.validate()
199
200 # If the validation was successfull, we mark the upload
201 # as finished and send True to the client.
202 if ret:
203 upload.finished()
204 self.finish("OK")
205
206 return
207
208 # In case the download was corrupted or incomplete, we delete it
209 # and tell the client to start over.
210 upload.remove()
211
212 self.finish("ERROR: CORRUPTED OR INCOMPLETE FILE")
213
214
215 class UploadsDestroyHandler(BaseHandler):
216 @tornado.web.authenticated
217 def get(self, upload_id):
218 upload = self.backend.uploads.get_by_uuid(upload_id)
219 if not upload:
220 raise tornado.web.HTTPError(404, "Invalid upload id.")
221
222 if not upload.builder == self.builder:
223 raise tornado.web.HTTPError(403, "Removing an other host's file.")
224
225 # Remove the upload from the database and trash the data.
226 upload.remove()
227
228
229 # Builds
230
231 class BuildsCreateHandler(BaseHandler):
232 @tornado.web.authenticated
233 def get(self):
234 # Get the upload ID of the package file.
235 upload_id = self.get_argument("upload_id")
236
237 # Get the identifier of the distribution we build for.
238 distro_ident = self.get_argument("distro")
239
240 # Get a list of arches to build for.
241 arches = self.get_argument("arches", None)
242 if arches == "":
243 arches = None
244
245 # Process build type.
246 build_type = self.get_argument("build_type")
247 if build_type == "release":
248 check_for_duplicates = True
249 elif build_type == "scratch":
250 check_for_duplicates = False
251 else:
252 raise tornado.web.HTTPError(400, "Invalid build type")
253
254 ## Check if the user has permission to create a build.
255 # Users only have the permission to create scratch builds.
256 if self.user and not build_type == "scratch":
257 raise tornado.web.HTTPError(403, "Users are only allowed to upload scratch builds")
258
259 # Get previously uploaded file to create this build from.
260 upload = self.backend.uploads.get_by_uuid(upload_id)
261 if not upload:
262 raise tornado.web.HTTPError(400, "Upload does not exist: %s" % upload_id)
263
264 # Check if the uploaded file belongs to this user/builder.
265 if self.user and not upload.user == self.user:
266 raise tornado.web.HTTPError(400, "Upload does not belong to this user.")
267
268 elif self.builder and not upload.builder == self.builder:
269 raise tornado.web.HTTPError(400, "Upload does not belong to this builder.")
270
271 # Get distribution this package should be built for.
272 distro = self.backend.distros.get_by_ident(distro_ident)
273 if not distro:
274 distro = self.backend.distros.get_default()
275
276 # Open the package that was uploaded earlier and add it to
277 # the database. Create a new build object from the uploaded package.
278 args = {
279 "arches" : arches,
280 "check_for_duplicates" : check_for_duplicates,
281 "distro" : distro,
282 "type" : build_type,
283 }
284 if self.user:
285 args["owner"] = self.user
286
287 try:
288 pkg, build = builds.import_from_package(self.backend, upload.path, **args)
289
290 except:
291 # Raise any exception.
292 raise
293
294 else:
295 # Creating the build will move the file to the build directory,
296 # so we can safely remove the uploaded file.
297 upload.remove()
298
299 # Send the build ID back to the user.
300 self.finish(build.uuid)
301
302
303 class BuildsGetHandler(BaseHandler):
304 def get(self, build_uuid):
305 build = self.backend.builds.get_by_uuid(build_uuid)
306 if not build:
307 raise tornado.web.HTTPError(404, "Could not find build: %s" % build_uuid)
308
309 ret = {
310 "distro" : build.distro.identifier,
311 "jobs" : [j.uuid for j in build.jobs],
312 "name" : build.name,
313 "package" : build.pkg.uuid,
314 "priority" : build.priority,
315 "score" : build.credits,
316 "severity" : build.severity,
317 "state" : build.state,
318 "sup_arches" : build.supported_arches,
319 "time_created" : build.created.isoformat(),
320 "type" : build.type,
321 "uuid" : build.uuid,
322 }
323
324 # If the build is in a repository, update that bit.
325 if build.repo:
326 ret["repo"] = build.repo.identifier
327
328 self.finish(ret)
329
330
331 # Jobs
332
333 class JobsBaseHandler(BaseHandler):
334 def job2json(self, job):
335 ret = {
336 "arch" : job.arch,
337 "build" : job.build.uuid,
338 "duration" : job.duration,
339 "name" : job.name,
340 "packages" : [p.uuid for p in job.packages],
341 "state" : job.state,
342 "time_created" : job.time_created.isoformat(),
343 "type" : "test" if job.test else "release",
344 "uuid" : job.uuid,
345 }
346
347 if job.builder:
348 ret["builder"] = job.builder.hostname
349
350 if job.time_started:
351 ret["time_started"] = job.time_started.isoformat()
352
353 if job.time_finished:
354 ret["time_finished"] = job.time_finished.isoformat()
355
356 return ret
357
358
359 class JobsGetActiveHandler(JobsBaseHandler):
360 def get(self):
361 # Get list of all active jobs.
362 jobs = self.backend.jobs.get_active()
363
364 args = {
365 "jobs" : [self.job2json(j) for j in jobs],
366 }
367
368 self.finish(args)
369
370
371 class JobsGetLatestHandler(JobsBaseHandler):
372 def get(self):
373 limit = self.get_argument_int("limit", 5)
374
375 # Get the latest jobs.
376 jobs = self.backend.jobs.get_latest(age="24 HOUR", limit=limit)
377
378 args = {
379 "jobs" : [self.job2json(j) for j in jobs],
380 }
381
382 self.finish(args)
383
384
385 class JobsGetQueueHandler(JobsBaseHandler):
386 def get(self):
387 limit = self.get_argument_int("limit", 5)
388
389 # Get the job queue.
390 jobs = []
391 for job in self.backend.jobqueue:
392 jobs.append(job)
393
394 limit -= 1
395 if not limit: break
396
397 args = {
398 "jobs" : [self.job2json(j) for j in jobs],
399 }
400
401 self.finish(args)
402
403
404 class JobsGetHandler(JobsBaseHandler):
405 def get(self, job_uuid):
406 job = self.backend.jobs.get_by_uuid(job_uuid)
407 if not job:
408 raise tornado.web.HTTPError(404, "Could not find job: %s" % job_uuid)
409
410 # Check if user is allowed to view this job.
411 if job.build.public == False:
412 if not self.user:
413 raise tornado.web.HTTPError(401)
414
415 # Check if an authenticated user has permission to see this build.
416 if not job.build.has_perm(self.user):
417 raise tornado.web.HTTPError(403)
418
419 ret = self.job2json(job)
420 self.finish(ret)
421
422
423 # Packages
424
425 class PackagesGetHandler(BaseHandler):
426 def get(self, package_uuid):
427 pkg = self.backend.packages.get_by_uuid(package_uuid)
428 if not pkg:
429 raise tornado.web.HTTPError(404, "Could not find package: %s" % package_uuid)
430
431 ret = {
432 "arch" : pkg.arch,
433 "build_id" : pkg.build_id,
434 "build_host" : pkg.build_host,
435 "build_time" : pkg.build_time.isoformat(),
436 "description" : pkg.description,
437 "epoch" : pkg.epoch,
438 "filesize" : pkg.filesize,
439 "friendly_name" : pkg.friendly_name,
440 "friendly_version" : pkg.friendly_version,
441 "groups" : pkg.groups,
442 "hash_sha512" : pkg.hash_sha512,
443 "license" : pkg.license,
444 "name" : pkg.name,
445 "release" : pkg.release,
446 "size" : pkg.size,
447 "summary" : pkg.summary,
448 "type" : pkg.type,
449 "url" : pkg.url,
450 "uuid" : pkg.uuid,
451 "version" : pkg.version,
452
453 # Dependencies.
454 "prerequires" : pkg.prerequires,
455 "requires" : pkg.requires,
456 "provides" : pkg.provides,
457 "obsoletes" : pkg.obsoletes,
458 "conflicts" : pkg.conflicts,
459 }
460
461 if pkg.type == "source":
462 ret["supported_arches"] = pkg.supported_arches
463
464 if isinstance(pkg.maintainer, users.User):
465 ret["maintainer"] = "%s <%s>" % (pkg.maintainer.realname, pkg.maintainer.email)
466 elif pkg.maintainer:
467 ret["maintainer"] = pkg.maintainer
468
469 if pkg.distro:
470 ret["distro"] = pkg.distro.identifier
471
472 self.finish(ret)
473
474
475 # Builders
476
477 class BuildersBaseHandler(BaseHandler):
478 def prepare(self):
479 # The request must come from an authenticated buider.
480 if not self.builder:
481 raise tornado.web.HTTPError(403)
482
483
484 class BuildersInfoHandler(BuildersBaseHandler):
485 @tornado.web.authenticated
486 def post(self):
487 args = {
488 # CPU info
489 "cpu_model" : self.get_argument("cpu_model", None),
490 "cpu_count" : self.get_argument("cpu_count", None),
491 "cpu_arch" : self.get_argument("cpu_arch", None),
492 "cpu_bogomips" : self.get_argument("cpu_bogomips", None),
493
494 # Pakfire
495 "pakfire_version" : self.get_argument("pakfire_version", None),
496 "host_key" : self.get_argument("host_key", None),
497
498 # OS
499 "os_name" : self.get_argument("os_name", None),
500 }
501 self.builder.update_info(**args)
502
503
504 class BuildersKeepaliveHandler(BuildersBaseHandler):
505 @tornado.web.authenticated
506 def post(self):
507 args = {
508 # Load average
509 "loadavg1" : self.get_argument_float("loadavg1", None),
510 "loadavg5" : self.get_argument_float("loadavg5", None),
511 "loadavg15" : self.get_argument_float("loadavg15", None),
512
513 # Memory
514 "mem_total" : self.get_argument_int("mem_total", None),
515 "mem_free" : self.get_argument_int("mem_free", None),
516
517 # swap
518 "swap_total" : self.get_argument_int("swap_total", None),
519 "swap_free" : self.get_argument_int("swap_free", None),
520
521 # Disk space
522 "space_free" : self.get_argument_int("space_free", None),
523 }
524 self.builder.update_keepalive(**args)
525
526 self.finish("OK")
527
528
529 class BuildersJobsQueueHandler(BuildersBaseHandler):
530 @tornado.web.asynchronous
531 @tornado.web.authenticated
532 def get(self):
533 self.callback()
534
535 def callback(self):
536 # Break if the connection has been closed in the mean time.
537 if self.connection_closed():
538 logging.warning("Connection closed")
539 return
540
541 # Check if there is a job for us.
542 job = self.builder.get_next_job()
543
544 # Got no job, wait and try again.
545 if not job:
546 # Check if we have been running for too long.
547 if self.runtime >= self.max_runtime:
548 logging.debug("Exceeded max. runtime. Finishing request.")
549 return self.finish()
550
551 # Try again in a jiffy.
552 self.add_timeout(self.heartbeat, self.callback)
553 return
554
555 try:
556 # Set job to dispatching state.
557 job.state = "dispatching"
558
559 # Set our build host.
560 job.builder = self.builder
561
562 ret = {
563 "id" : job.uuid,
564 "arch" : job.arch,
565 "source_url" : job.build.source_download,
566 "source_hash_sha512" : job.build.source_hash_sha512,
567 "type" : "test" if job.test else "release",
568 "config" : job.get_config(),
569 }
570
571 # Send build information to the builder.
572 self.finish(ret)
573 except:
574 # If anything went wrong, we reset the state.
575 job.state = "pending"
576 raise
577
578 @property
579 def heartbeat(self):
580 return 15 # 15 seconds
581
582 @property
583 def max_runtime(self):
584 timeout = self.get_argument_int("timeout", None)
585 if timeout:
586 return timeout - self.heartbeat
587
588 return 300 # 5 min
589
590
591 class BuildersJobsStateHandler(BuildersBaseHandler):
592 @tornado.web.authenticated
593 def post(self, job_uuid, state):
594 job = self.backend.jobs.get_by_uuid(job_uuid)
595 if not job:
596 raise tornado.web.HTTPError(404, "Invalid job id.")
597
598 if not job.builder == self.builder:
599 raise tornado.web.HTTPError(403, "Altering another builder's build.")
600
601 # Save information to database.
602 job.state = state
603
604 message = self.get_argument("message", None)
605 job.update_message(message)
606
607 self.finish("OK")
608
609
610 class BuildersJobsBuildrootHandler(BuildersBaseHandler):
611 @tornado.web.authenticated
612 def post(self, job_uuid):
613 job = self.backend.jobs.get_by_uuid(job_uuid)
614 if not job:
615 raise tornado.web.HTTPError(404, "Invalid job id.")
616
617 if not job.builder == self.builder:
618 raise tornado.web.HTTPError(403, "Altering another builder's build.")
619
620 # Get buildroot.
621 buildroot = self.get_argument_json("buildroot", None)
622 if buildroot:
623 job.save_buildroot(buildroot)
624
625 self.finish("OK")
626
627
628 class BuildersJobsAddFileHandler(BuildersBaseHandler):
629 @tornado.web.authenticated
630 def post(self, job_uuid, upload_id):
631 type = self.get_argument("type")
632 assert type in ("package", "log")
633
634 # Fetch job we are working on and check if it is actually ours.
635 job = self.backend.jobs.get_by_uuid(job_uuid)
636 if not job:
637 raise tornado.web.HTTPError(404, "Invalid job id.")
638
639 if not job.builder == self.builder:
640 raise tornado.web.HTTPError(403, "Altering another builder's job.")
641
642 # Fetch uploaded file object and check we uploaded it ourself.
643 upload = self.backend.uploads.get_by_uuid(upload_id)
644 if not upload:
645 raise tornado.web.HTTPError(404, "Invalid upload id.")
646
647 if not upload.builder == self.builder:
648 raise tornado.web.HTTPError(403, "Using an other host's file.")
649
650 # Remove all files that have to be deleted, first.
651 self.backend.cleanup_files()
652
653 try:
654 job.add_file(upload.path)
655
656 finally:
657 # Finally, remove the uploaded file.
658 upload.remove()
659
660 self.finish("OK")