]>
Commit | Line | Data |
---|---|---|
1 | #!/usr/bin/python3 | |
2 | ||
3 | import difflib | |
4 | import logging | |
5 | import os.path | |
6 | import re | |
7 | import tornado.gen | |
8 | import urllib.parse | |
9 | ||
10 | from . import misc | |
11 | from . import util | |
12 | from .decorators import * | |
13 | ||
14 | INTERWIKIS = { | |
15 | "google" : ("https://www.google.com/search?q=%(url)s", None, "fab fa-google"), | |
16 | "rfc" : ("https://tools.ietf.org/html/rfc%(name)s", "RFC %s", None), | |
17 | "wp" : ("https://en.wikipedia.org/wiki/%(name)s", None, "fab fa-wikipedia-w"), | |
18 | } | |
19 | ||
20 | class Wiki(misc.Object): | |
21 | def _get_pages(self, query, *args): | |
22 | res = self.db.query(query, *args) | |
23 | ||
24 | for row in res: | |
25 | yield Page(self.backend, row.id, data=row) | |
26 | ||
27 | def _get_page(self, query, *args): | |
28 | res = self.db.get(query, *args) | |
29 | ||
30 | if res: | |
31 | return Page(self.backend, res.id, data=res) | |
32 | ||
33 | def make_path(self, page, path): | |
34 | # Nothing to do for absolute links | |
35 | if path.startswith("/"): | |
36 | pass | |
37 | ||
38 | # Relative links (one-level down) | |
39 | elif path.startswith("./"): | |
40 | path = os.path.join(page, path) | |
41 | ||
42 | # All other relative links | |
43 | else: | |
44 | p = os.path.dirname(page) | |
45 | path = os.path.join(p, path) | |
46 | ||
47 | # Normalise links | |
48 | return os.path.normpath(path) | |
49 | ||
50 | def get_page_title(self, page, default=None): | |
51 | # Try to retrieve title from cache | |
52 | title = self.memcache.get("wiki:title:%s" % page) | |
53 | if title: | |
54 | return title | |
55 | ||
56 | # If the title has not been in the cache, we will | |
57 | # have to look it up | |
58 | doc = self.get_page(page) | |
59 | if doc: | |
60 | title = doc.title | |
61 | else: | |
62 | title = os.path.basename(page) | |
63 | ||
64 | # Save in cache for forever | |
65 | self.memcache.set("wiki:title:%s" % page, title) | |
66 | ||
67 | return title | |
68 | ||
69 | def get_page(self, page, revision=None): | |
70 | page = Page.sanitise_page_name(page) | |
71 | assert page | |
72 | ||
73 | if revision: | |
74 | return self._get_page("SELECT * FROM wiki WHERE page = %s \ | |
75 | AND timestamp = %s", page, revision) | |
76 | else: | |
77 | return self._get_page("SELECT * FROM wiki WHERE page = %s \ | |
78 | ORDER BY timestamp DESC LIMIT 1", page) | |
79 | ||
80 | def get_recent_changes(self, account, limit=None): | |
81 | pages = self._get_pages("SELECT * FROM wiki \ | |
82 | ORDER BY timestamp DESC") | |
83 | ||
84 | for page in pages: | |
85 | if not page.check_acl(account): | |
86 | continue | |
87 | ||
88 | yield page | |
89 | ||
90 | limit -= 1 | |
91 | if not limit: | |
92 | break | |
93 | ||
94 | def create_page(self, page, author, content, changes=None, address=None): | |
95 | page = Page.sanitise_page_name(page) | |
96 | ||
97 | # Write page to the database | |
98 | page = self._get_page("INSERT INTO wiki(page, author_uid, markdown, changes, address) \ | |
99 | VALUES(%s, %s, %s, %s, %s) RETURNING *", page, author.uid, content or None, changes, address) | |
100 | ||
101 | # Update cache | |
102 | self.memcache.set("wiki:title:%s" % page.page, page.title) | |
103 | ||
104 | # Send email to all watchers | |
105 | page._send_watcher_emails(excludes=[author]) | |
106 | ||
107 | return page | |
108 | ||
109 | def delete_page(self, page, author, **kwargs): | |
110 | # Do nothing if the page does not exist | |
111 | if not self.get_page(page): | |
112 | return | |
113 | ||
114 | # Just creates a blank last version of the page | |
115 | self.create_page(page, author=author, content=None, **kwargs) | |
116 | ||
117 | def make_breadcrumbs(self, url): | |
118 | # Split and strip all empty elements (double slashes) | |
119 | parts = list(e for e in url.split("/") if e) | |
120 | ||
121 | ret = [] | |
122 | for part in ("/".join(parts[:i]) for i in range(1, len(parts))): | |
123 | ret.append(("/%s" % part, self.get_page_title(part, os.path.basename(part)))) | |
124 | ||
125 | return ret | |
126 | ||
127 | def search(self, query, account=None, limit=None): | |
128 | query = util.parse_search_query(query) | |
129 | ||
130 | res = self._get_pages("SELECT wiki.* FROM wiki_search_index search_index \ | |
131 | LEFT JOIN wiki ON search_index.wiki_id = wiki.id \ | |
132 | WHERE search_index.document @@ to_tsquery('english', %s) \ | |
133 | ORDER BY ts_rank(search_index.document, to_tsquery('english', %s)) DESC", | |
134 | query, query) | |
135 | ||
136 | pages = [] | |
137 | for page in res: | |
138 | # Skip any pages the user doesn't have permission for | |
139 | if not page.check_acl(account): | |
140 | continue | |
141 | ||
142 | # Return any other pages | |
143 | pages.append(page) | |
144 | ||
145 | # Break when we have found enough pages | |
146 | if limit and len(pages) >= limit: | |
147 | break | |
148 | ||
149 | return pages | |
150 | ||
151 | def refresh(self): | |
152 | """ | |
153 | Needs to be called after a page has been changed | |
154 | """ | |
155 | self.db.execute("REFRESH MATERIALIZED VIEW wiki_search_index") | |
156 | ||
157 | def get_watchlist(self, account): | |
158 | pages = self._get_pages( | |
159 | "WITH pages AS (SELECT * FROM wiki_current \ | |
160 | LEFT JOIN wiki ON wiki_current.id = wiki.id) \ | |
161 | SELECT * FROM wiki_watchlist watchlist \ | |
162 | LEFT JOIN pages ON watchlist.page = pages.page \ | |
163 | WHERE watchlist.uid = %s", | |
164 | account.uid, | |
165 | ) | |
166 | ||
167 | return sorted(pages) | |
168 | ||
169 | # ACL | |
170 | ||
171 | def check_acl(self, page, account): | |
172 | res = self.db.query("SELECT * FROM wiki_acls \ | |
173 | WHERE %s ILIKE (path || '%%') ORDER BY LENGTH(path) DESC LIMIT 1", page) | |
174 | ||
175 | for row in res: | |
176 | # Access not permitted when user is not logged in | |
177 | if not account: | |
178 | return False | |
179 | ||
180 | # If user is in a matching group, we grant permission | |
181 | for group in row.groups: | |
182 | if group in account.groups: | |
183 | return True | |
184 | ||
185 | # Otherwise access is not permitted | |
186 | return False | |
187 | ||
188 | # If no ACLs are found, we permit access | |
189 | return True | |
190 | ||
191 | # Files | |
192 | ||
193 | def _get_files(self, query, *args): | |
194 | res = self.db.query(query, *args) | |
195 | ||
196 | for row in res: | |
197 | yield File(self.backend, row.id, data=row) | |
198 | ||
199 | def _get_file(self, query, *args): | |
200 | res = self.db.get(query, *args) | |
201 | ||
202 | if res: | |
203 | return File(self.backend, res.id, data=res) | |
204 | ||
205 | def get_files(self, path): | |
206 | files = self._get_files("SELECT * FROM wiki_files \ | |
207 | WHERE path = %s AND deleted_at IS NULL ORDER BY filename", path) | |
208 | ||
209 | return list(files) | |
210 | ||
211 | def get_file_by_path(self, path): | |
212 | path, filename = os.path.dirname(path), os.path.basename(path) | |
213 | ||
214 | return self._get_file("SELECT * FROM wiki_files \ | |
215 | WHERE path = %s AND filename = %s AND deleted_at IS NULL", path, filename) | |
216 | ||
217 | def upload(self, path, filename, data, mimetype, author, address): | |
218 | # Upload the blob first | |
219 | blob = self.db.get("INSERT INTO wiki_blobs(data) VALUES(%s) RETURNING id", data) | |
220 | ||
221 | # Create entry for file | |
222 | return self._get_file("INSERT INTO wiki_files(path, filename, author_uid, address, \ | |
223 | mimetype, blob_id, size) VALUES(%s, %s, %s, %s, %s, %s, %s) RETURNING *", path, | |
224 | filename, author.uid, address, mimetype, blob.id, len(data)) | |
225 | ||
226 | def render(self, path, text): | |
227 | r = WikiRenderer(self.backend, path) | |
228 | ||
229 | return r.render(text) | |
230 | ||
231 | ||
232 | class Page(misc.Object): | |
233 | def init(self, id, data=None): | |
234 | self.id = id | |
235 | self.data = data | |
236 | ||
237 | def __repr__(self): | |
238 | return "<%s %s %s>" % (self.__class__.__name__, self.page, self.timestamp) | |
239 | ||
240 | def __eq__(self, other): | |
241 | if isinstance(other, self.__class__): | |
242 | return self.id == other.id | |
243 | ||
244 | def __lt__(self, other): | |
245 | if isinstance(other, self.__class__): | |
246 | if self.page == other.page: | |
247 | return self.timestamp < other.timestamp | |
248 | ||
249 | return self.page < other.page | |
250 | ||
251 | @staticmethod | |
252 | def sanitise_page_name(page): | |
253 | if not page: | |
254 | return "/" | |
255 | ||
256 | # Make sure that the page name does NOT end with a / | |
257 | if page.endswith("/"): | |
258 | page = page[:-1] | |
259 | ||
260 | # Make sure the page name starts with a / | |
261 | if not page.startswith("/"): | |
262 | page = "/%s" % page | |
263 | ||
264 | # Remove any double slashes | |
265 | page = page.replace("//", "/") | |
266 | ||
267 | return page | |
268 | ||
269 | @property | |
270 | def url(self): | |
271 | return self.page | |
272 | ||
273 | @property | |
274 | def full_url(self): | |
275 | return "https://wiki.ipfire.org%s" % self.url | |
276 | ||
277 | @property | |
278 | def page(self): | |
279 | return self.data.page | |
280 | ||
281 | @property | |
282 | def title(self): | |
283 | return self._title or os.path.basename(self.page[1:]) | |
284 | ||
285 | @property | |
286 | def _title(self): | |
287 | if not self.markdown: | |
288 | return | |
289 | ||
290 | # Find first H1 headline in markdown | |
291 | markdown = self.markdown.splitlines() | |
292 | ||
293 | m = re.match(r"^# (.*)( #)?$", markdown[0]) | |
294 | if m: | |
295 | return m.group(1) | |
296 | ||
297 | @lazy_property | |
298 | def author(self): | |
299 | if self.data.author_uid: | |
300 | return self.backend.accounts.get_by_uid(self.data.author_uid) | |
301 | ||
302 | @property | |
303 | def markdown(self): | |
304 | return self.data.markdown or "" | |
305 | ||
306 | @property | |
307 | def html(self): | |
308 | return self.backend.wiki.render(self.page, self.markdown) | |
309 | ||
310 | @property | |
311 | def timestamp(self): | |
312 | return self.data.timestamp | |
313 | ||
314 | def was_deleted(self): | |
315 | return self.markdown is None | |
316 | ||
317 | @lazy_property | |
318 | def breadcrumbs(self): | |
319 | return self.backend.wiki.make_breadcrumbs(self.page) | |
320 | ||
321 | def get_latest_revision(self): | |
322 | revisions = self.get_revisions() | |
323 | ||
324 | # Return first object | |
325 | for rev in revisions: | |
326 | return rev | |
327 | ||
328 | def get_revisions(self): | |
329 | return self.backend.wiki._get_pages("SELECT * FROM wiki \ | |
330 | WHERE page = %s ORDER BY timestamp DESC", self.page) | |
331 | ||
332 | @lazy_property | |
333 | def previous_revision(self): | |
334 | return self.backend.wiki._get_page("SELECT * FROM wiki \ | |
335 | WHERE page = %s AND timestamp < %s ORDER BY timestamp DESC \ | |
336 | LIMIT 1", self.page, self.timestamp) | |
337 | ||
338 | @property | |
339 | def changes(self): | |
340 | return self.data.changes | |
341 | ||
342 | # ACL | |
343 | ||
344 | def check_acl(self, account): | |
345 | return self.backend.wiki.check_acl(self.page, account) | |
346 | ||
347 | # Sidebar | |
348 | ||
349 | @lazy_property | |
350 | def sidebar(self): | |
351 | parts = self.page.split("/") | |
352 | ||
353 | while parts: | |
354 | sidebar = self.backend.wiki.get_page("%s/sidebar" % os.path.join(*parts)) | |
355 | if sidebar: | |
356 | return sidebar | |
357 | ||
358 | parts.pop() | |
359 | ||
360 | # Watchers | |
361 | ||
362 | @lazy_property | |
363 | def diff(self): | |
364 | if self.previous_revision: | |
365 | diff = difflib.unified_diff( | |
366 | self.previous_revision.markdown.splitlines(), | |
367 | self.markdown.splitlines(), | |
368 | ) | |
369 | ||
370 | return "\n".join(diff) | |
371 | ||
372 | @property | |
373 | def watchers(self): | |
374 | res = self.db.query("SELECT uid FROM wiki_watchlist \ | |
375 | WHERE page = %s", self.page) | |
376 | ||
377 | for row in res: | |
378 | # Search for account by UID and skip if none was found | |
379 | account = self.backend.accounts.get_by_uid(row.uid) | |
380 | if not account: | |
381 | continue | |
382 | ||
383 | # Return the account | |
384 | yield account | |
385 | ||
386 | def is_watched_by(self, account): | |
387 | res = self.db.get("SELECT 1 FROM wiki_watchlist \ | |
388 | WHERE page = %s AND uid = %s", self.page, account.uid) | |
389 | ||
390 | if res: | |
391 | return True | |
392 | ||
393 | return False | |
394 | ||
395 | def add_watcher(self, account): | |
396 | if self.is_watched_by(account): | |
397 | return | |
398 | ||
399 | self.db.execute("INSERT INTO wiki_watchlist(page, uid) \ | |
400 | VALUES(%s, %s)", self.page, account.uid) | |
401 | ||
402 | def remove_watcher(self, account): | |
403 | self.db.execute("DELETE FROM wiki_watchlist \ | |
404 | WHERE page = %s AND uid = %s", self.page, account.uid) | |
405 | ||
406 | def _send_watcher_emails(self, excludes=[]): | |
407 | # Nothing to do if there was no previous revision | |
408 | if not self.previous_revision: | |
409 | return | |
410 | ||
411 | for watcher in self.watchers: | |
412 | # Skip everyone who is excluded | |
413 | if watcher in excludes: | |
414 | logging.debug("Excluding %s" % watcher) | |
415 | continue | |
416 | ||
417 | # Check permissions | |
418 | if not self.backend.wiki.check_acl(self.page, watcher): | |
419 | logging.debug("Watcher %s does not have permissions" % watcher) | |
420 | continue | |
421 | ||
422 | logging.debug("Sending watcher email to %s" % watcher) | |
423 | ||
424 | # Compose message | |
425 | self.backend.messages.send_template("wiki/messages/page-changed", | |
426 | recipients=[watcher], page=self, priority=-10) | |
427 | ||
428 | ||
429 | class File(misc.Object): | |
430 | def init(self, id, data): | |
431 | self.id = id | |
432 | self.data = data | |
433 | ||
434 | @property | |
435 | def url(self): | |
436 | return os.path.join(self.path, self.filename) | |
437 | ||
438 | @property | |
439 | def path(self): | |
440 | return self.data.path | |
441 | ||
442 | @property | |
443 | def filename(self): | |
444 | return self.data.filename | |
445 | ||
446 | @property | |
447 | def mimetype(self): | |
448 | return self.data.mimetype | |
449 | ||
450 | @property | |
451 | def size(self): | |
452 | return self.data.size | |
453 | ||
454 | @lazy_property | |
455 | def author(self): | |
456 | if self.data.author_uid: | |
457 | return self.backend.accounts.get_by_uid(self.data.author_uid) | |
458 | ||
459 | @property | |
460 | def created_at(self): | |
461 | return self.data.created_at | |
462 | ||
463 | def is_pdf(self): | |
464 | return self.mimetype in ("application/pdf", "application/x-pdf") | |
465 | ||
466 | def is_image(self): | |
467 | return self.mimetype.startswith("image/") | |
468 | ||
469 | @lazy_property | |
470 | def blob(self): | |
471 | res = self.db.get("SELECT data FROM wiki_blobs \ | |
472 | WHERE id = %s", self.data.blob_id) | |
473 | ||
474 | if res: | |
475 | return bytes(res.data) | |
476 | ||
477 | def get_thumbnail(self, size): | |
478 | cache_key = "-".join((self.path, util.normalize(self.filename), self.created_at.isoformat(), "%spx" % size)) | |
479 | ||
480 | # Try to fetch the data from the cache | |
481 | thumbnail = self.memcache.get(cache_key) | |
482 | if thumbnail: | |
483 | return thumbnail | |
484 | ||
485 | # Generate the thumbnail | |
486 | thumbnail = util.generate_thumbnail(self.blob, size) | |
487 | ||
488 | # Put it into the cache for forever | |
489 | self.memcache.set(cache_key, thumbnail) | |
490 | ||
491 | return thumbnail | |
492 | ||
493 | ||
494 | class WikiRenderer(misc.Object): | |
495 | # Wiki links | |
496 | wiki_link = re.compile(r"\[\[([\w\d\/\-\.]+)(?:\|(.+?))?\]\]") | |
497 | ||
498 | # External links | |
499 | external_link = re.compile(r"\[\[((?:ftp|git|https?|rsync|sftp|ssh|webcal)\:\/\/.+?)(?:\|(.+?))?\]\]") | |
500 | ||
501 | # Interwiki links e.g. [[wp>IPFire]] | |
502 | interwiki_link = re.compile(r"\[\[(\w+)>(.+?)(?:\|(.+?))?\]\]") | |
503 | ||
504 | # Mail link | |
505 | email_link = re.compile(r"\[\[([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)(?:\|(.+?))?\]\]") | |
506 | ||
507 | # Images | |
508 | images = re.compile(r"{{([\w\d\/\-\.]+)(?:\|(.+?))?}}") | |
509 | ||
510 | def init(self, path): | |
511 | self.path = path | |
512 | ||
513 | def _render_wiki_link(self, m): | |
514 | path, alias = m.groups() | |
515 | ||
516 | path = self.backend.wiki.make_path(self.path, path) | |
517 | ||
518 | return """<a href="%s">%s</a>""" % ( | |
519 | path, | |
520 | alias or self.backend.wiki.get_page_title(path), | |
521 | ) | |
522 | ||
523 | def _render_external_link(self, m): | |
524 | url, alias = m.groups() | |
525 | ||
526 | return """<a class="link-external" href="%s">%s</a>""" % (url, alias or url) | |
527 | ||
528 | def _render_interwiki_link(self, m): | |
529 | wiki = m.group(1) | |
530 | if not wiki: | |
531 | return | |
532 | ||
533 | # Retrieve URL | |
534 | try: | |
535 | url, repl, icon = INTERWIKIS[wiki] | |
536 | except KeyError: | |
537 | logging.warning("Invalid interwiki: %s" % wiki) | |
538 | return | |
539 | ||
540 | # Name of the page | |
541 | name = m.group(2) | |
542 | ||
543 | # Expand URL | |
544 | url = url % { | |
545 | "name" : name, | |
546 | "url" : urllib.parse.quote(name), | |
547 | } | |
548 | ||
549 | # Get alias (if present) | |
550 | alias = m.group(3) | |
551 | ||
552 | if not alias and repl: | |
553 | alias = repl % name | |
554 | ||
555 | # Put everything together | |
556 | s = [] | |
557 | ||
558 | if icon: | |
559 | s.append("<span class=\"%s\"></span>" % icon) | |
560 | ||
561 | s.append("""<a class="link-external" href="%s">%s</a>""" % (url, alias or name)) | |
562 | ||
563 | return " ".join(s) | |
564 | ||
565 | def _render_email_link(self, m): | |
566 | address, alias = m.groups() | |
567 | ||
568 | return """<a class="link-external" href="mailto:%s">%s</a>""" \ | |
569 | % (address, alias or address) | |
570 | ||
571 | def _render_image(self, m): | |
572 | url, text = m.groups() | |
573 | ||
574 | # Skip any absolute and external URLs | |
575 | if url.startswith("/") or url.startswith("https://") or url.startswith("http://"): | |
576 | return """<img src="%s" alt="%s">""" % (url, text or "") | |
577 | ||
578 | # Try to split query string | |
579 | url, delimiter, qs = url.partition("?") | |
580 | ||
581 | # Parse query arguments | |
582 | args = urllib.parse.parse_qs(qs) | |
583 | ||
584 | # Build absolute path | |
585 | url = self.backend.wiki.make_path(self.path, url) | |
586 | ||
587 | # Find image | |
588 | file = self.backend.wiki.get_file_by_path(url) | |
589 | if not file or not file.is_image(): | |
590 | return "<!-- Could not find image %s in %s -->" % (url, self.path) | |
591 | ||
592 | # Scale down the image if not already done | |
593 | if not "s" in args: | |
594 | args["s"] = "768" | |
595 | ||
596 | return """<a href="%s?action=detail"><img src="%s?%s" alt="%s"></a>""" \ | |
597 | % (url, url, urllib.parse.urlencode(args), text or "") | |
598 | ||
599 | def render(self, text): | |
600 | logging.debug("Rendering %s" % self.path) | |
601 | ||
602 | # Handle wiki links | |
603 | text = self.wiki_link.sub(self._render_wiki_link, text) | |
604 | ||
605 | # Handle interwiki links | |
606 | text = self.interwiki_link.sub(self._render_interwiki_link, text) | |
607 | ||
608 | # Handle external links | |
609 | text = self.external_link.sub(self._render_external_link, text) | |
610 | ||
611 | # Handle email links | |
612 | text = self.email_link.sub(self._render_email_link, text) | |
613 | ||
614 | # Handle images | |
615 | text = self.images.sub(self._render_image, text) | |
616 | ||
617 | # Borrow this from the blog | |
618 | return self.backend.blog._render_text(text, lang="markdown") |