wiki: Implement embedding images with {{...}} syntax
[ipfire.org.git] / src / backend / wiki.py
CommitLineData
181d08f3
MT
1#!/usr/bin/python3
2
4ed1dadb 3import difflib
181d08f3 4import logging
6ac7e934 5import os.path
181d08f3 6import re
addc18d5 7import tornado.gen
9e90e800 8import urllib.parse
181d08f3
MT
9
10from . import misc
9523790a 11from . import util
181d08f3
MT
12from .decorators import *
13
c683a1ff
MT
14INTERWIKIS = {
15 "google" : ("https://www.google.com/search?q=%(url)s", None, "fab fa-google"),
16 "rfc" : ("https://tools.ietf.org/html/rfc%(name)s", "RFC %s", None),
17 "wp" : ("https://en.wikipedia.org/wiki/%(name)s", None, "fab fa-wikipedia-w"),
18}
19
181d08f3
MT
20class Wiki(misc.Object):
21 def _get_pages(self, query, *args):
22 res = self.db.query(query, *args)
23
24 for row in res:
25 yield Page(self.backend, row.id, data=row)
26
d398ca08
MT
27 def _get_page(self, query, *args):
28 res = self.db.get(query, *args)
29
30 if res:
31 return Page(self.backend, res.id, data=res)
32
c78ad26e
MT
33 def make_path(self, page, path):
34 # Nothing to do for absolute links
35 if path.startswith("/"):
36 pass
37
38 # Relative links (one-level down)
39 elif path.startswith("./"):
40 path = os.path.join(page, path)
41
42 # All other relative links
43 else:
44 p = os.path.dirname(page)
45 path = os.path.join(p, path)
46
47 # Normalise links
48 return os.path.normpath(path)
49
6ac7e934 50 def get_page_title(self, page, default=None):
50c8dc11
MT
51 # Try to retrieve title from cache
52 title = self.memcache.get("wiki:title:%s" % page)
53 if title:
54 return title
55
56 # If the title has not been in the cache, we will
57 # have to look it up
6ac7e934
MT
58 doc = self.get_page(page)
59 if doc:
50c8dc11
MT
60 title = doc.title
61 else:
62 title = os.path.basename(page)
6ac7e934 63
50c8dc11
MT
64 # Save in cache for forever
65 self.memcache.set("wiki:title:%s" % page, title)
66
67 return title
6ac7e934 68
181d08f3
MT
69 def get_page(self, page, revision=None):
70 page = Page.sanitise_page_name(page)
71 assert page
72
73 if revision:
d398ca08 74 return self._get_page("SELECT * FROM wiki WHERE page = %s \
181d08f3
MT
75 AND timestamp = %s", page, revision)
76 else:
d398ca08 77 return self._get_page("SELECT * FROM wiki WHERE page = %s \
181d08f3
MT
78 ORDER BY timestamp DESC LIMIT 1", page)
79
11afe905
MT
80 def get_recent_changes(self, account, limit=None):
81 pages = self._get_pages("SELECT * FROM wiki \
11afe905
MT
82 ORDER BY timestamp DESC")
83
84 for page in pages:
85 if not page.check_acl(account):
86 continue
87
88 yield page
89
90 limit -= 1
91 if not limit:
92 break
181d08f3 93
495e9dc4 94 def create_page(self, page, author, content, changes=None, address=None):
181d08f3
MT
95 page = Page.sanitise_page_name(page)
96
aba5e58a
MT
97 # Write page to the database
98 page = self._get_page("INSERT INTO wiki(page, author_uid, markdown, changes, address) \
df01767e 99 VALUES(%s, %s, %s, %s, %s) RETURNING *", page, author.uid, content or None, changes, address)
181d08f3 100
50c8dc11 101 # Update cache
980e486d 102 self.memcache.set("wiki:title:%s" % page.page, page.title)
50c8dc11 103
aba5e58a
MT
104 # Send email to all watchers
105 page._send_watcher_emails(excludes=[author])
106
107 return page
108
495e9dc4 109 def delete_page(self, page, author, **kwargs):
181d08f3
MT
110 # Do nothing if the page does not exist
111 if not self.get_page(page):
112 return
113
114 # Just creates a blank last version of the page
495e9dc4 115 self.create_page(page, author=author, content=None, **kwargs)
181d08f3 116
3168788e
MT
117 def make_breadcrumbs(self, url):
118 # Split and strip all empty elements (double slashes)
181d08f3
MT
119 parts = list(e for e in url.split("/") if e)
120
3168788e 121 ret = []
b1bf7d48 122 for part in ("/".join(parts[:i]) for i in range(1, len(parts))):
3168788e 123 ret.append(("/%s" % part, self.get_page_title(part, os.path.basename(part))))
181d08f3 124
3168788e 125 return ret
181d08f3 126
11afe905 127 def search(self, query, account=None, limit=None):
9523790a
MT
128 query = util.parse_search_query(query)
129
130 res = self._get_pages("SELECT wiki.* FROM wiki_search_index search_index \
131 LEFT JOIN wiki ON search_index.wiki_id = wiki.id \
132 WHERE search_index.document @@ to_tsquery('english', %s) \
11afe905
MT
133 ORDER BY ts_rank(search_index.document, to_tsquery('english', %s)) DESC",
134 query, query)
9523790a 135
df80be2c 136 pages = []
11afe905
MT
137 for page in res:
138 # Skip any pages the user doesn't have permission for
139 if not page.check_acl(account):
140 continue
141
142 # Return any other pages
df80be2c 143 pages.append(page)
11afe905 144
df80be2c
MT
145 # Break when we have found enough pages
146 if limit and len(pages) >= limit:
11afe905 147 break
9523790a 148
df80be2c
MT
149 return pages
150
9523790a
MT
151 def refresh(self):
152 """
153 Needs to be called after a page has been changed
154 """
155 self.db.execute("REFRESH MATERIALIZED VIEW wiki_search_index")
156
2f23c558
MT
157 def get_watchlist(self, account):
158 pages = self._get_pages(
159 "WITH pages AS (SELECT * FROM wiki_current \
160 LEFT JOIN wiki ON wiki_current.id = wiki.id) \
161 SELECT * FROM wiki_watchlist watchlist \
162 LEFT JOIN pages ON watchlist.page = pages.page \
163 WHERE watchlist.uid = %s",
164 account.uid,
165 )
166
167 return sorted(pages)
168
11afe905
MT
169 # ACL
170
171 def check_acl(self, page, account):
172 res = self.db.query("SELECT * FROM wiki_acls \
173 WHERE %s ILIKE (path || '%%') ORDER BY LENGTH(path) DESC LIMIT 1", page)
174
175 for row in res:
176 # Access not permitted when user is not logged in
177 if not account:
178 return False
179
180 # If user is in a matching group, we grant permission
181 for group in row.groups:
182 if group in account.groups:
183 return True
184
185 # Otherwise access is not permitted
186 return False
187
188 # If no ACLs are found, we permit access
189 return True
190
f2cfd873
MT
191 # Files
192
193 def _get_files(self, query, *args):
194 res = self.db.query(query, *args)
195
196 for row in res:
197 yield File(self.backend, row.id, data=row)
198
199 def _get_file(self, query, *args):
200 res = self.db.get(query, *args)
201
202 if res:
203 return File(self.backend, res.id, data=res)
204
205 def get_files(self, path):
206 files = self._get_files("SELECT * FROM wiki_files \
207 WHERE path = %s AND deleted_at IS NULL ORDER BY filename", path)
208
209 return list(files)
210
211 def get_file_by_path(self, path):
212 path, filename = os.path.dirname(path), os.path.basename(path)
213
214 return self._get_file("SELECT * FROM wiki_files \
215 WHERE path = %s AND filename = %s AND deleted_at IS NULL", path, filename)
216
217 def upload(self, path, filename, data, mimetype, author, address):
218 # Upload the blob first
219 blob = self.db.get("INSERT INTO wiki_blobs(data) VALUES(%s) RETURNING id", data)
220
221 # Create entry for file
222 return self._get_file("INSERT INTO wiki_files(path, filename, author_uid, address, \
223 mimetype, blob_id, size) VALUES(%s, %s, %s, %s, %s, %s, %s) RETURNING *", path,
224 filename, author.uid, address, mimetype, blob.id, len(data))
225
2901b734
MT
226 def render(self, path, text):
227 r = WikiRenderer(self.backend, path)
181d08f3 228
2901b734 229 return r.render(text)
e2205cff 230
154f6179 231
2901b734 232class Page(misc.Object):
181d08f3
MT
233 def init(self, id, data=None):
234 self.id = id
235 self.data = data
236
dc847af5
MT
237 def __repr__(self):
238 return "<%s %s %s>" % (self.__class__.__name__, self.page, self.timestamp)
239
c21ffadb
MT
240 def __eq__(self, other):
241 if isinstance(other, self.__class__):
242 return self.id == other.id
243
181d08f3
MT
244 def __lt__(self, other):
245 if isinstance(other, self.__class__):
246 if self.page == other.page:
247 return self.timestamp < other.timestamp
248
249 return self.page < other.page
250
251 @staticmethod
252 def sanitise_page_name(page):
253 if not page:
254 return "/"
255
256 # Make sure that the page name does NOT end with a /
257 if page.endswith("/"):
258 page = page[:-1]
259
260 # Make sure the page name starts with a /
261 if not page.startswith("/"):
262 page = "/%s" % page
263
264 # Remove any double slashes
265 page = page.replace("//", "/")
266
267 return page
268
269 @property
270 def url(self):
db8448d9 271 return self.page
181d08f3 272
4ed1dadb
MT
273 @property
274 def full_url(self):
275 return "https://wiki.ipfire.org%s" % self.url
276
181d08f3
MT
277 @property
278 def page(self):
279 return self.data.page
280
281 @property
282 def title(self):
51e7a876 283 return self._title or os.path.basename(self.page[1:])
181d08f3
MT
284
285 @property
286 def _title(self):
287 if not self.markdown:
288 return
289
290 # Find first H1 headline in markdown
291 markdown = self.markdown.splitlines()
292
293 m = re.match(r"^# (.*)( #)?$", markdown[0])
294 if m:
295 return m.group(1)
296
3b05ef6e
MT
297 @lazy_property
298 def author(self):
299 if self.data.author_uid:
300 return self.backend.accounts.get_by_uid(self.data.author_uid)
301
181d08f3
MT
302 @property
303 def markdown(self):
c21ffadb 304 return self.data.markdown or ""
181d08f3
MT
305
306 @property
307 def html(self):
2901b734 308 return self.backend.wiki.render(self.page, self.markdown)
addc18d5 309
181d08f3
MT
310 @property
311 def timestamp(self):
312 return self.data.timestamp
313
314 def was_deleted(self):
315 return self.markdown is None
316
317 @lazy_property
318 def breadcrumbs(self):
319 return self.backend.wiki.make_breadcrumbs(self.page)
320
321 def get_latest_revision(self):
7d699684
MT
322 revisions = self.get_revisions()
323
324 # Return first object
325 for rev in revisions:
326 return rev
327
328 def get_revisions(self):
329 return self.backend.wiki._get_pages("SELECT * FROM wiki \
330 WHERE page = %s ORDER BY timestamp DESC", self.page)
091ac36b 331
c21ffadb
MT
332 @lazy_property
333 def previous_revision(self):
334 return self.backend.wiki._get_page("SELECT * FROM wiki \
335 WHERE page = %s AND timestamp < %s ORDER BY timestamp DESC \
336 LIMIT 1", self.page, self.timestamp)
337
d398ca08
MT
338 @property
339 def changes(self):
340 return self.data.changes
341
11afe905
MT
342 # ACL
343
344 def check_acl(self, account):
345 return self.backend.wiki.check_acl(self.page, account)
346
091ac36b
MT
347 # Sidebar
348
349 @lazy_property
350 def sidebar(self):
351 parts = self.page.split("/")
352
353 while parts:
3cc5f666 354 sidebar = self.backend.wiki.get_page("%s/sidebar" % os.path.join(*parts))
091ac36b
MT
355 if sidebar:
356 return sidebar
357
358 parts.pop()
f2cfd873 359
d64a1e35
MT
360 # Watchers
361
4ed1dadb
MT
362 @lazy_property
363 def diff(self):
364 if self.previous_revision:
365 diff = difflib.unified_diff(
366 self.previous_revision.markdown.splitlines(),
367 self.markdown.splitlines(),
368 )
369
370 return "\n".join(diff)
371
aba5e58a
MT
372 @property
373 def watchers(self):
374 res = self.db.query("SELECT uid FROM wiki_watchlist \
375 WHERE page = %s", self.page)
376
377 for row in res:
378 # Search for account by UID and skip if none was found
379 account = self.backend.accounts.get_by_uid(row.uid)
380 if not account:
381 continue
382
383 # Return the account
384 yield account
385
f2e25ded 386 def is_watched_by(self, account):
d64a1e35
MT
387 res = self.db.get("SELECT 1 FROM wiki_watchlist \
388 WHERE page = %s AND uid = %s", self.page, account.uid)
389
390 if res:
391 return True
392
393 return False
394
395 def add_watcher(self, account):
f2e25ded 396 if self.is_watched_by(account):
d64a1e35
MT
397 return
398
399 self.db.execute("INSERT INTO wiki_watchlist(page, uid) \
400 VALUES(%s, %s)", self.page, account.uid)
401
402 def remove_watcher(self, account):
403 self.db.execute("DELETE FROM wiki_watchlist \
404 WHERE page = %s AND uid = %s", self.page, account.uid)
405
aba5e58a
MT
406 def _send_watcher_emails(self, excludes=[]):
407 # Nothing to do if there was no previous revision
408 if not self.previous_revision:
409 return
410
411 for watcher in self.watchers:
412 # Skip everyone who is excluded
413 if watcher in excludes:
414 logging.debug("Excluding %s" % watcher)
415 continue
416
516da0a9
MT
417 # Check permissions
418 if not self.backend.wiki.check_acl(self.page, watcher):
419 logging.debug("Watcher %s does not have permissions" % watcher)
420 continue
421
aba5e58a
MT
422 logging.debug("Sending watcher email to %s" % watcher)
423
4ed1dadb
MT
424 # Compose message
425 self.backend.messages.send_template("wiki/messages/page-changed",
426 recipients=[watcher], page=self, priority=-10)
aba5e58a 427
f2cfd873
MT
428
429class File(misc.Object):
430 def init(self, id, data):
431 self.id = id
432 self.data = data
433
434 @property
435 def url(self):
436 return os.path.join(self.path, self.filename)
437
438 @property
439 def path(self):
440 return self.data.path
441
442 @property
443 def filename(self):
444 return self.data.filename
445
446 @property
447 def mimetype(self):
448 return self.data.mimetype
449
450 @property
451 def size(self):
452 return self.data.size
453
8cb0bea4
MT
454 @lazy_property
455 def author(self):
456 if self.data.author_uid:
457 return self.backend.accounts.get_by_uid(self.data.author_uid)
458
459 @property
460 def created_at(self):
461 return self.data.created_at
462
463 def is_pdf(self):
464 return self.mimetype in ("application/pdf", "application/x-pdf")
465
f2cfd873
MT
466 def is_image(self):
467 return self.mimetype.startswith("image/")
468
469 @lazy_property
470 def blob(self):
471 res = self.db.get("SELECT data FROM wiki_blobs \
472 WHERE id = %s", self.data.blob_id)
473
474 if res:
475 return bytes(res.data)
79dd9a0f
MT
476
477 def get_thumbnail(self, size):
75d9b3da
MT
478 cache_key = "-".join((self.path, util.normalize(self.filename), self.created_at.isoformat(), "%spx" % size))
479
480 # Try to fetch the data from the cache
481 thumbnail = self.memcache.get(cache_key)
482 if thumbnail:
483 return thumbnail
484
485 # Generate the thumbnail
5ef115cd 486 thumbnail = util.generate_thumbnail(self.blob, size)
75d9b3da
MT
487
488 # Put it into the cache for forever
489 self.memcache.set(cache_key, thumbnail)
490
491 return thumbnail
2901b734
MT
492
493
494class WikiRenderer(misc.Object):
495 # Wiki links
496 wiki_link = re.compile(r"\[\[([\w\d\/\-\.]+)(?:\|(.+?))?\]\]")
497
498 # External links
499 external_link = re.compile(r"\[\[((?:ftp|git|https?|rsync|sftp|ssh|webcal)\:\/\/.+?)(?:\|(.+?))?\]\]")
500
501 # Interwiki links e.g. [[wp>IPFire]]
502 interwiki_link = re.compile(r"\[\[(\w+)>(.+?)(?:\|(.+?))?\]\]")
503
504 # Mail link
505 email_link = re.compile(r"\[\[([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)(?:\|(.+?))?\]\]")
506
c78ad26e
MT
507 # Images
508 images = re.compile(r"{{([\w\d\/\-\.]+)(?:\|(.+?))?}}")
509
2901b734
MT
510 def init(self, path):
511 self.path = path
512
513 def _render_wiki_link(self, m):
514 path, alias = m.groups()
515
c78ad26e 516 path = self.backend.wiki.make_path(self.path, path)
2901b734
MT
517
518 return """<a href="%s">%s</a>""" % (
519 path,
520 alias or self.backend.wiki.get_page_title(path),
521 )
522
523 def _render_external_link(self, m):
524 url, alias = m.groups()
525
526 return """<a class="link-external" href="%s">%s</a>""" % (url, alias or url)
527
528 def _render_interwiki_link(self, m):
529 wiki = m.group(1)
530 if not wiki:
531 return
532
533 # Retrieve URL
534 try:
535 url, repl, icon = INTERWIKIS[wiki]
536 except KeyError:
537 logging.warning("Invalid interwiki: %s" % wiki)
538 return
539
540 # Name of the page
541 name = m.group(2)
542
543 # Expand URL
544 url = url % {
545 "name" : name,
546 "url" : urllib.parse.quote(name),
547 }
548
549 # Get alias (if present)
550 alias = m.group(3)
551
552 if not alias and repl:
553 alias = repl % name
554
555 # Put everything together
556 s = []
557
558 if icon:
559 s.append("<span class=\"%s\"></span>" % icon)
560
561 s.append("""<a class="link-external" href="%s">%s</a>""" % (url, alias or name))
562
563 return " ".join(s)
564
565 def _render_email_link(self, m):
566 address, alias = m.groups()
567
568 return """<a class="link-external" href="mailto:%s">%s</a>""" \
569 % (address, alias or address)
570
c78ad26e
MT
571 def _render_image(self, m):
572 url, text = m.groups()
2901b734 573
c78ad26e
MT
574 # Skip any absolute and external URLs
575 if url.startswith("/") or url.startswith("https://") or url.startswith("http://"):
576 return """<img src="%s" alt="%s">""" % (url, text or "")
2901b734 577
c78ad26e
MT
578 # Try to split query string
579 url, delimiter, qs = url.partition("?")
2901b734 580
c78ad26e
MT
581 # Parse query arguments
582 args = urllib.parse.parse_qs(qs)
2901b734 583
c78ad26e
MT
584 # Build absolute path
585 url = self.backend.wiki.make_path(self.path, url)
2901b734 586
c78ad26e
MT
587 # Find image
588 file = self.backend.wiki.get_file_by_path(url)
589 if not file or not file.is_image():
590 return "<!-- Could not find image %s in %s -->" % (url, self.path)
2901b734 591
c78ad26e
MT
592 # Scale down the image if not already done
593 if not "s" in args:
594 args["s"] = "768"
2901b734 595
c78ad26e
MT
596 return """<a href="%s?action=detail"><img src="%s?%s" alt="%s"></a>""" \
597 % (url, url, urllib.parse.urlencode(args), text or "")
2901b734 598
c78ad26e
MT
599 def render(self, text):
600 logging.debug("Rendering %s" % self.path)
2901b734
MT
601
602 # Handle wiki links
603 text = self.wiki_link.sub(self._render_wiki_link, text)
604
605 # Handle interwiki links
606 text = self.interwiki_link.sub(self._render_interwiki_link, text)
607
608 # Handle external links
609 text = self.external_link.sub(self._render_external_link, text)
610
611 # Handle email links
612 text = self.email_link.sub(self._render_email_link, text)
613
c78ad26e
MT
614 # Handle images
615 text = self.images.sub(self._render_image, text)
616
2901b734
MT
617 # Borrow this from the blog
618 return self.backend.blog._render_text(text, lang="markdown")