wiki: Implement embedding images with {{...}} syntax
[ipfire.org.git] / src / backend / wiki.py
1 #!/usr/bin/python3
2
3 import difflib
4 import logging
5 import os.path
6 import re
7 import tornado.gen
8 import urllib.parse
9
10 from . import misc
11 from . import util
12 from .decorators import *
13
14 INTERWIKIS = {
15         "google"    : ("https://www.google.com/search?q=%(url)s", None, "fab fa-google"),
16         "rfc"       : ("https://tools.ietf.org/html/rfc%(name)s", "RFC %s", None),
17         "wp"        : ("https://en.wikipedia.org/wiki/%(name)s", None, "fab fa-wikipedia-w"),
18 }
19
20 class Wiki(misc.Object):
21         def _get_pages(self, query, *args):
22                 res = self.db.query(query, *args)
23
24                 for row in res:
25                         yield Page(self.backend, row.id, data=row)
26
27         def _get_page(self, query, *args):
28                 res = self.db.get(query, *args)
29
30                 if res:
31                         return Page(self.backend, res.id, data=res)
32
33         def make_path(self, page, path):
34                 # Nothing to do for absolute links
35                 if path.startswith("/"):
36                         pass
37
38                 # Relative links (one-level down)
39                 elif path.startswith("./"):
40                         path = os.path.join(page, path)
41
42                 # All other relative links
43                 else:
44                         p = os.path.dirname(page)
45                         path = os.path.join(p, path)
46
47                 # Normalise links
48                 return os.path.normpath(path)
49
50         def get_page_title(self, page, default=None):
51                 # Try to retrieve title from cache
52                 title = self.memcache.get("wiki:title:%s" % page)
53                 if title:
54                         return title
55
56                 # If the title has not been in the cache, we will
57                 # have to look it up
58                 doc = self.get_page(page)
59                 if doc:
60                         title = doc.title
61                 else:
62                         title = os.path.basename(page)
63
64                 # Save in cache for forever
65                 self.memcache.set("wiki:title:%s" % page, title)
66
67                 return title
68
69         def get_page(self, page, revision=None):
70                 page = Page.sanitise_page_name(page)
71                 assert page
72
73                 if revision:
74                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
75                                 AND timestamp = %s", page, revision)
76                 else:
77                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
78                                 ORDER BY timestamp DESC LIMIT 1", page)
79
80         def get_recent_changes(self, account, limit=None):
81                 pages = self._get_pages("SELECT * FROM wiki \
82                         ORDER BY timestamp DESC")
83
84                 for page in pages:
85                         if not page.check_acl(account):
86                                 continue
87
88                         yield page
89
90                         limit -= 1
91                         if not limit:
92                                 break
93
94         def create_page(self, page, author, content, changes=None, address=None):
95                 page = Page.sanitise_page_name(page)
96
97                 # Write page to the database
98                 page = self._get_page("INSERT INTO wiki(page, author_uid, markdown, changes, address) \
99                         VALUES(%s, %s, %s, %s, %s) RETURNING *", page, author.uid, content or None, changes, address)
100
101                 # Update cache
102                 self.memcache.set("wiki:title:%s" % page.page, page.title)
103
104                 # Send email to all watchers
105                 page._send_watcher_emails(excludes=[author])
106
107                 return page
108
109         def delete_page(self, page, author, **kwargs):
110                 # Do nothing if the page does not exist
111                 if not self.get_page(page):
112                         return
113
114                 # Just creates a blank last version of the page
115                 self.create_page(page, author=author, content=None, **kwargs)
116
117         def make_breadcrumbs(self, url):
118                 # Split and strip all empty elements (double slashes)
119                 parts = list(e for e in url.split("/") if e)
120
121                 ret = []
122                 for part in ("/".join(parts[:i]) for i in range(1, len(parts))):
123                         ret.append(("/%s" % part, self.get_page_title(part, os.path.basename(part))))
124
125                 return ret
126
127         def search(self, query, account=None, limit=None):
128                 query = util.parse_search_query(query)
129
130                 res = self._get_pages("SELECT wiki.* FROM wiki_search_index search_index \
131                         LEFT JOIN wiki ON search_index.wiki_id = wiki.id \
132                         WHERE search_index.document @@ to_tsquery('english', %s) \
133                                 ORDER BY ts_rank(search_index.document, to_tsquery('english', %s)) DESC",
134                         query, query)
135
136                 pages = []
137                 for page in res:
138                         # Skip any pages the user doesn't have permission for
139                         if not page.check_acl(account):
140                                 continue
141
142                         # Return any other pages
143                         pages.append(page)
144
145                         # Break when we have found enough pages
146                         if limit and len(pages) >= limit:
147                                 break
148
149                 return pages
150
151         def refresh(self):
152                 """
153                         Needs to be called after a page has been changed
154                 """
155                 self.db.execute("REFRESH MATERIALIZED VIEW wiki_search_index")
156
157         def get_watchlist(self, account):
158                 pages = self._get_pages(
159                         "WITH pages AS (SELECT * FROM wiki_current \
160                                         LEFT JOIN wiki ON wiki_current.id = wiki.id) \
161                         SELECT * FROM wiki_watchlist watchlist \
162                                 LEFT JOIN pages ON watchlist.page = pages.page \
163                                 WHERE watchlist.uid = %s",
164                         account.uid,
165                 )
166
167                 return sorted(pages)
168
169         # ACL
170
171         def check_acl(self, page, account):
172                 res = self.db.query("SELECT * FROM wiki_acls \
173                         WHERE %s ILIKE (path || '%%') ORDER BY LENGTH(path) DESC LIMIT 1", page)
174
175                 for row in res:
176                         # Access not permitted when user is not logged in
177                         if not account:
178                                 return False
179
180                         # If user is in a matching group, we grant permission
181                         for group in row.groups:
182                                 if group in account.groups:
183                                         return True
184
185                         # Otherwise access is not permitted
186                         return False
187
188                 # If no ACLs are found, we permit access
189                 return True
190
191         # Files
192
193         def _get_files(self, query, *args):
194                 res = self.db.query(query, *args)
195
196                 for row in res:
197                         yield File(self.backend, row.id, data=row)
198
199         def _get_file(self, query, *args):
200                 res = self.db.get(query, *args)
201
202                 if res:
203                         return File(self.backend, res.id, data=res)
204
205         def get_files(self, path):
206                 files = self._get_files("SELECT * FROM wiki_files \
207                         WHERE path = %s AND deleted_at IS NULL ORDER BY filename", path)
208
209                 return list(files)
210
211         def get_file_by_path(self, path):
212                 path, filename = os.path.dirname(path), os.path.basename(path)
213
214                 return self._get_file("SELECT * FROM wiki_files \
215                         WHERE path = %s AND filename = %s AND deleted_at IS NULL", path, filename)
216
217         def upload(self, path, filename, data, mimetype, author, address):
218                 # Upload the blob first
219                 blob = self.db.get("INSERT INTO wiki_blobs(data) VALUES(%s) RETURNING id", data)
220
221                 # Create entry for file
222                 return self._get_file("INSERT INTO wiki_files(path, filename, author_uid, address, \
223                         mimetype, blob_id, size) VALUES(%s,  %s, %s, %s, %s, %s, %s) RETURNING *", path,
224                         filename, author.uid, address, mimetype, blob.id, len(data))
225
226         def render(self, path, text):
227                 r = WikiRenderer(self.backend, path)
228
229                 return r.render(text)
230
231
232 class Page(misc.Object):
233         def init(self, id, data=None):
234                 self.id = id
235                 self.data = data
236
237         def __repr__(self):
238                 return "<%s %s %s>" % (self.__class__.__name__, self.page, self.timestamp)
239
240         def __eq__(self, other):
241                 if isinstance(other, self.__class__):
242                         return self.id == other.id
243
244         def __lt__(self, other):
245                 if isinstance(other, self.__class__):
246                         if self.page == other.page:
247                                 return self.timestamp < other.timestamp
248
249                         return self.page < other.page
250
251         @staticmethod
252         def sanitise_page_name(page):
253                 if not page:
254                         return "/"
255
256                 # Make sure that the page name does NOT end with a /
257                 if page.endswith("/"):
258                         page = page[:-1]
259
260                 # Make sure the page name starts with a /
261                 if not page.startswith("/"):
262                         page = "/%s" % page
263
264                 # Remove any double slashes
265                 page = page.replace("//", "/")
266
267                 return page
268
269         @property
270         def url(self):
271                 return self.page
272
273         @property
274         def full_url(self):
275                 return "https://wiki.ipfire.org%s" % self.url
276
277         @property
278         def page(self):
279                 return self.data.page
280
281         @property
282         def title(self):
283                 return self._title or os.path.basename(self.page[1:])
284
285         @property
286         def _title(self):
287                 if not self.markdown:
288                         return
289
290                 # Find first H1 headline in markdown
291                 markdown = self.markdown.splitlines()
292
293                 m = re.match(r"^# (.*)( #)?$", markdown[0])
294                 if m:
295                         return m.group(1)
296
297         @lazy_property
298         def author(self):
299                 if self.data.author_uid:
300                         return self.backend.accounts.get_by_uid(self.data.author_uid)
301
302         @property
303         def markdown(self):
304                 return self.data.markdown or ""
305
306         @property
307         def html(self):
308                 return self.backend.wiki.render(self.page, self.markdown)
309
310         @property
311         def timestamp(self):
312                 return self.data.timestamp
313
314         def was_deleted(self):
315                 return self.markdown is None
316
317         @lazy_property
318         def breadcrumbs(self):
319                 return self.backend.wiki.make_breadcrumbs(self.page)
320
321         def get_latest_revision(self):
322                 revisions = self.get_revisions()
323
324                 # Return first object
325                 for rev in revisions:
326                         return rev
327
328         def get_revisions(self):
329                 return self.backend.wiki._get_pages("SELECT * FROM wiki \
330                         WHERE page = %s ORDER BY timestamp DESC", self.page)
331
332         @lazy_property
333         def previous_revision(self):
334                 return self.backend.wiki._get_page("SELECT * FROM wiki \
335                         WHERE page = %s AND timestamp < %s ORDER BY timestamp DESC \
336                         LIMIT 1", self.page, self.timestamp)
337
338         @property
339         def changes(self):
340                 return self.data.changes
341
342         # ACL
343
344         def check_acl(self, account):
345                 return self.backend.wiki.check_acl(self.page, account)
346
347         # Sidebar
348
349         @lazy_property
350         def sidebar(self):
351                 parts = self.page.split("/")
352
353                 while parts:
354                         sidebar = self.backend.wiki.get_page("%s/sidebar" % os.path.join(*parts))
355                         if sidebar:
356                                 return sidebar
357
358                         parts.pop()
359
360         # Watchers
361
362         @lazy_property
363         def diff(self):
364                 if self.previous_revision:
365                         diff = difflib.unified_diff(
366                                 self.previous_revision.markdown.splitlines(),
367                                 self.markdown.splitlines(),
368                         )
369
370                         return "\n".join(diff)
371
372         @property
373         def watchers(self):
374                 res = self.db.query("SELECT uid FROM wiki_watchlist \
375                         WHERE page = %s", self.page)
376
377                 for row in res:
378                         # Search for account by UID and skip if none was found
379                         account = self.backend.accounts.get_by_uid(row.uid)
380                         if not account:
381                                 continue
382
383                         # Return the account
384                         yield account
385
386         def is_watched_by(self, account):
387                 res = self.db.get("SELECT 1 FROM wiki_watchlist \
388                         WHERE page = %s AND uid = %s", self.page, account.uid)
389
390                 if res:
391                         return True
392
393                 return False
394
395         def add_watcher(self, account):
396                 if self.is_watched_by(account):
397                         return
398
399                 self.db.execute("INSERT INTO wiki_watchlist(page, uid) \
400                         VALUES(%s, %s)", self.page, account.uid)
401
402         def remove_watcher(self, account):
403                 self.db.execute("DELETE FROM wiki_watchlist \
404                         WHERE page = %s AND uid = %s", self.page, account.uid)
405
406         def _send_watcher_emails(self, excludes=[]):
407                 # Nothing to do if there was no previous revision
408                 if not self.previous_revision:
409                         return
410
411                 for watcher in self.watchers:
412                         # Skip everyone who is excluded
413                         if watcher in excludes:
414                                 logging.debug("Excluding %s" % watcher)
415                                 continue
416
417                         # Check permissions
418                         if not self.backend.wiki.check_acl(self.page, watcher):
419                                 logging.debug("Watcher %s does not have permissions" % watcher)
420                                 continue
421
422                         logging.debug("Sending watcher email to %s" % watcher)
423
424                         # Compose message
425                         self.backend.messages.send_template("wiki/messages/page-changed",
426                                 recipients=[watcher], page=self, priority=-10)
427
428
429 class File(misc.Object):
430         def init(self, id, data):
431                 self.id   = id
432                 self.data = data
433
434         @property
435         def url(self):
436                 return os.path.join(self.path, self.filename)
437
438         @property
439         def path(self):
440                 return self.data.path
441
442         @property
443         def filename(self):
444                 return self.data.filename
445
446         @property
447         def mimetype(self):
448                 return self.data.mimetype
449
450         @property
451         def size(self):
452                 return self.data.size
453
454         @lazy_property
455         def author(self):
456                 if self.data.author_uid:
457                         return self.backend.accounts.get_by_uid(self.data.author_uid)
458
459         @property
460         def created_at(self):
461                 return self.data.created_at
462
463         def is_pdf(self):
464                 return self.mimetype in ("application/pdf", "application/x-pdf")
465
466         def is_image(self):
467                 return self.mimetype.startswith("image/")
468
469         @lazy_property
470         def blob(self):
471                 res = self.db.get("SELECT data FROM wiki_blobs \
472                         WHERE id = %s", self.data.blob_id)
473
474                 if res:
475                         return bytes(res.data)
476
477         def get_thumbnail(self, size):
478                 cache_key = "-".join((self.path, util.normalize(self.filename), self.created_at.isoformat(), "%spx" % size))
479
480                 # Try to fetch the data from the cache
481                 thumbnail = self.memcache.get(cache_key)
482                 if thumbnail:
483                         return thumbnail
484
485                 # Generate the thumbnail
486                 thumbnail = util.generate_thumbnail(self.blob, size)
487
488                 # Put it into the cache for forever
489                 self.memcache.set(cache_key, thumbnail)
490
491                 return thumbnail
492
493
494 class WikiRenderer(misc.Object):
495         # Wiki links
496         wiki_link = re.compile(r"\[\[([\w\d\/\-\.]+)(?:\|(.+?))?\]\]")
497
498         # External links
499         external_link = re.compile(r"\[\[((?:ftp|git|https?|rsync|sftp|ssh|webcal)\:\/\/.+?)(?:\|(.+?))?\]\]")
500
501         # Interwiki links e.g. [[wp>IPFire]]
502         interwiki_link = re.compile(r"\[\[(\w+)>(.+?)(?:\|(.+?))?\]\]")
503
504         # Mail link
505         email_link = re.compile(r"\[\[([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)(?:\|(.+?))?\]\]")
506
507         # Images
508         images = re.compile(r"{{([\w\d\/\-\.]+)(?:\|(.+?))?}}")
509
510         def init(self, path):
511                 self.path = path
512
513         def _render_wiki_link(self, m):
514                 path, alias = m.groups()
515
516                 path = self.backend.wiki.make_path(self.path, path)
517
518                 return """<a href="%s">%s</a>""" % (
519                         path,
520                         alias or self.backend.wiki.get_page_title(path),
521                 )
522
523         def _render_external_link(self, m):
524                 url, alias = m.groups()
525
526                 return """<a class="link-external" href="%s">%s</a>""" % (url, alias or url)
527
528         def _render_interwiki_link(self, m):
529                 wiki = m.group(1)
530                 if not wiki:
531                         return
532
533                 # Retrieve URL
534                 try:
535                         url, repl, icon = INTERWIKIS[wiki]
536                 except KeyError:
537                         logging.warning("Invalid interwiki: %s" % wiki)
538                         return
539
540                 # Name of the page
541                 name = m.group(2)
542
543                 # Expand URL
544                 url = url % {
545                         "name" : name,
546                         "url"  : urllib.parse.quote(name),
547                 }
548
549                 # Get alias (if present)
550                 alias = m.group(3)
551
552                 if not alias and repl:
553                         alias = repl % name
554
555                 # Put everything together
556                 s = []
557
558                 if icon:
559                         s.append("<span class=\"%s\"></span>" % icon)
560
561                 s.append("""<a class="link-external" href="%s">%s</a>""" % (url, alias or name))
562
563                 return " ".join(s)
564
565         def _render_email_link(self, m):
566                 address, alias = m.groups()
567
568                 return """<a class="link-external" href="mailto:%s">%s</a>""" \
569                         % (address, alias or address)
570
571         def _render_image(self, m):
572                 url, text = m.groups()
573
574                 # Skip any absolute and external URLs
575                 if url.startswith("/") or url.startswith("https://") or url.startswith("http://"):
576                         return """<img src="%s" alt="%s">""" % (url, text or "")
577
578                 # Try to split query string
579                 url, delimiter, qs = url.partition("?")
580
581                 # Parse query arguments
582                 args = urllib.parse.parse_qs(qs)
583
584                 # Build absolute path
585                 url = self.backend.wiki.make_path(self.path, url)
586
587                 # Find image
588                 file = self.backend.wiki.get_file_by_path(url)
589                 if not file or not file.is_image():
590                         return "<!-- Could not find image %s in %s -->" % (url, self.path)
591
592                 # Scale down the image if not already done
593                 if not "s" in args:
594                         args["s"] = "768"
595
596                 return """<a href="%s?action=detail"><img src="%s?%s" alt="%s"></a>""" \
597                         % (url, url, urllib.parse.urlencode(args), text or "")
598
599         def render(self, text):
600                 logging.debug("Rendering %s" % self.path)
601
602                 # Handle wiki links
603                 text = self.wiki_link.sub(self._render_wiki_link, text)
604
605                 # Handle interwiki links
606                 text = self.interwiki_link.sub(self._render_interwiki_link, text)
607
608                 # Handle external links
609                 text = self.external_link.sub(self._render_external_link, text)
610
611                 # Handle email links
612                 text = self.email_link.sub(self._render_email_link, text)
613
614                 # Handle images
615                 text = self.images.sub(self._render_image, text)
616
617                 # Borrow this from the blog
618                 return self.backend.blog._render_text(text, lang="markdown")