src/backend/wiki.py

   1 #!/usr/bin/python3
   2
   3 import PIL
   4 import io
   5 import logging
   6 import os.path
   7 import re
   8 import urllib.parse
   9
  10 from . import misc
  11 from . import util
  12 from .decorators import *
  13
  14 class Wiki(misc.Object):
  15         def _get_pages(self, query, *args):
  16                 res = self.db.query(query, *args)
  17
  18                 for row in res:
  19                         yield Page(self.backend, row.id, data=row)
  20
  21         def _get_page(self, query, *args):
  22                 res = self.db.get(query, *args)
  23
  24                 if res:
  25                         return Page(self.backend, res.id, data=res)
  26
  27         def get_page_title(self, page, default=None):
  28                 doc = self.get_page(page)
  29                 if doc:
  30                         return doc.title
  31
  32                 return default or os.path.basename(page)
  33
  34         def get_page(self, page, revision=None):
  35                 page = Page.sanitise_page_name(page)
  36                 assert page
  37
  38                 if revision:
  39                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
  40                                 AND timestamp = %s", page, revision)
  41                 else:
  42                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
  43                                 ORDER BY timestamp DESC LIMIT 1", page)
  44
  45         def get_recent_changes(self, account, limit=None):
  46                 pages = self._get_pages("SELECT * FROM wiki \
  47                         WHERE timestamp >= NOW() - INTERVAL '4 weeks' \
  48                         ORDER BY timestamp DESC")
  49
  50                 for page in pages:
  51                         if not page.check_acl(account):
  52                                 continue
  53
  54                         yield page
  55
  56                         limit -= 1
  57                         if not limit:
  58                                 break
  59
  60         def create_page(self, page, author, content, changes=None, address=None):
  61                 page = Page.sanitise_page_name(page)
  62
  63                 # Write page to the database
  64                 page = self._get_page("INSERT INTO wiki(page, author_uid, markdown, changes, address) \
  65                         VALUES(%s, %s, %s, %s, %s) RETURNING *", page, author.uid, content or None, changes, address)
  66
  67                 # Send email to all watchers
  68                 page._send_watcher_emails(excludes=[author])
  69
  70                 return page
  71
  72         def delete_page(self, page, author, **kwargs):
  73                 # Do nothing if the page does not exist
  74                 if not self.get_page(page):
  75                         return
  76
  77                 # Just creates a blank last version of the page
  78                 self.create_page(page, author=author, content=None, **kwargs)
  79
  80         def make_breadcrumbs(self, url):
  81                 # Split and strip all empty elements (double slashes)
  82                 parts = list(e for e in url.split("/") if e)
  83
  84                 ret = []
  85                 for part in ("/".join(parts[:i]) for i in range(1, len(parts))):
  86                         ret.append(("/%s" % part, self.get_page_title(part, os.path.basename(part))))
  87
  88                 return ret
  89
  90         def search(self, query, account=None, limit=None):
  91                 query = util.parse_search_query(query)
  92
  93                 res = self._get_pages("SELECT wiki.* FROM wiki_search_index search_index \
  94                         LEFT JOIN wiki ON search_index.wiki_id = wiki.id \
  95                         WHERE search_index.document @@ to_tsquery('english', %s) \
  96                                 ORDER BY ts_rank(search_index.document, to_tsquery('english', %s)) DESC",
  97                         query, query)
  98
  99                 for page in res:
 100                         # Skip any pages the user doesn't have permission for
 101                         if not page.check_acl(account):
 102                                 continue
 103
 104                         # Return any other pages
 105                         yield page
 106
 107                         limit -= 1
 108                         if not limit:
 109                                 break
 110
 111         def refresh(self):
 112                 """
 113                         Needs to be called after a page has been changed
 114                 """
 115                 self.db.execute("REFRESH MATERIALIZED VIEW wiki_search_index")
 116
 117         # ACL
 118
 119         def check_acl(self, page, account):
 120                 res = self.db.query("SELECT * FROM wiki_acls \
 121                         WHERE %s ILIKE (path || '%%') ORDER BY LENGTH(path) DESC LIMIT 1", page)
 122
 123                 for row in res:
 124                         # Access not permitted when user is not logged in
 125                         if not account:
 126                                 return False
 127
 128                         # If user is in a matching group, we grant permission
 129                         for group in row.groups:
 130                                 if group in account.groups:
 131                                         return True
 132
 133                         # Otherwise access is not permitted
 134                         return False
 135
 136                 # If no ACLs are found, we permit access
 137                 return True
 138
 139         # Files
 140
 141         def _get_files(self, query, *args):
 142                 res = self.db.query(query, *args)
 143
 144                 for row in res:
 145                         yield File(self.backend, row.id, data=row)
 146
 147         def _get_file(self, query, *args):
 148                 res = self.db.get(query, *args)
 149
 150                 if res:
 151                         return File(self.backend, res.id, data=res)
 152
 153         def get_files(self, path):
 154                 files = self._get_files("SELECT * FROM wiki_files \
 155                         WHERE path = %s AND deleted_at IS NULL ORDER BY filename", path)
 156
 157                 return list(files)
 158
 159         def get_file_by_path(self, path):
 160                 path, filename = os.path.dirname(path), os.path.basename(path)
 161
 162                 return self._get_file("SELECT * FROM wiki_files \
 163                         WHERE path = %s AND filename = %s AND deleted_at IS NULL", path, filename)
 164
 165         def upload(self, path, filename, data, mimetype, author, address):
 166                 # Upload the blob first
 167                 blob = self.db.get("INSERT INTO wiki_blobs(data) VALUES(%s) RETURNING id", data)
 168
 169                 # Create entry for file
 170                 return self._get_file("INSERT INTO wiki_files(path, filename, author_uid, address, \
 171                         mimetype, blob_id, size) VALUES(%s,  %s, %s, %s, %s, %s, %s) RETURNING *", path,
 172                         filename, author.uid, address, mimetype, blob.id, len(data))
 173
 174         def find_image(self, path, filename):
 175                 for p in (path, os.path.dirname(path)):
 176                         file = self.get_file_by_path(os.path.join(p, filename))
 177
 178                         if file and file.is_image():
 179                                 return file
 180
 181
 182 class Page(misc.Object):
 183         def init(self, id, data=None):
 184                 self.id = id
 185                 self.data = data
 186
 187         def __repr__(self):
 188                 return "<%s %s %s>" % (self.__class__.__name__, self.page, self.timestamp)
 189
 190         def __eq__(self, other):
 191                 if isinstance(other, self.__class__):
 192                         return self.id == other.id
 193
 194         def __lt__(self, other):
 195                 if isinstance(other, self.__class__):
 196                         if self.page == other.page:
 197                                 return self.timestamp < other.timestamp
 198
 199                         return self.page < other.page
 200
 201         @staticmethod
 202         def sanitise_page_name(page):
 203                 if not page:
 204                         return "/"
 205
 206                 # Make sure that the page name does NOT end with a /
 207                 if page.endswith("/"):
 208                         page = page[:-1]
 209
 210                 # Make sure the page name starts with a /
 211                 if not page.startswith("/"):
 212                         page = "/%s" % page
 213
 214                 # Remove any double slashes
 215                 page = page.replace("//", "/")
 216
 217                 return page
 218
 219         @property
 220         def url(self):
 221                 return self.page
 222
 223         @property
 224         def page(self):
 225                 return self.data.page
 226
 227         @property
 228         def title(self):
 229                 return self._title or os.path.basename(self.page[1:])
 230
 231         @property
 232         def _title(self):
 233                 if not self.markdown:
 234                         return
 235
 236                 # Find first H1 headline in markdown
 237                 markdown = self.markdown.splitlines()
 238
 239                 m = re.match(r"^# (.*)( #)?$", markdown[0])
 240                 if m:
 241                         return m.group(1)
 242
 243         @lazy_property
 244         def author(self):
 245                 if self.data.author_uid:
 246                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 247
 248         def _render(self, text):
 249                 logging.debug("Rendering %s" % self)
 250
 251                 # Link images
 252                 replacements = []
 253                 for match in re.finditer(r"!\[(.*)\]\((.*)\)", text):
 254                         alt_text, url = match.groups()
 255
 256                         # Skip any absolute and external URLs
 257                         if url.startswith("/") or url.startswith("https://") or url.startswith("http://"):
 258                                 continue
 259
 260                         # Try to split query string
 261                         url, delimiter, qs = url.partition("?")
 262
 263                         # Parse query arguments
 264                         args = urllib.parse.parse_qs(qs)
 265
 266                         # Find image
 267                         file = self.backend.wiki.find_image(self.page, url)
 268                         if not file:
 269                                 continue
 270
 271                         # Scale down the image if not already done
 272                         if not "s" in args:
 273                                 args["s"] = "768"
 274
 275                         # Format URL
 276                         url = "%s?%s" % (file.url, urllib.parse.urlencode(args))
 277
 278                         replacements.append((match.span(), file, alt_text, url))
 279
 280                 # Apply all replacements
 281                 for (start, end), file, alt_text, url in reversed(replacements):
 282                         text = text[:start] + "[![%s](%s)](%s?action=detail)" % (alt_text, url, file.url) + text[end:]
 283
 284                 # Add wiki links
 285                 patterns = (
 286                         (r"\[\[([\w\d\/]+)(?:\|([\w\d\s]+))\]\]", r"/\1", r"\2", None, None),
 287                         (r"\[\[([\w\d\/\-]+)\]\]", r"/\1", r"\1", self.backend.wiki.get_page_title, r"\1"),
 288                 )
 289
 290                 for pattern, link, title, repl, args in patterns:
 291                         replacements = []
 292
 293                         for match in re.finditer(pattern, text):
 294                                 l = match.expand(link)
 295                                 t = match.expand(title)
 296
 297                                 if callable(repl):
 298                                         t = repl(match.expand(args)) or t
 299
 300                                 replacements.append((match.span(), t or l, l))
 301
 302                         # Apply all replacements
 303                         for (start, end), t, l in reversed(replacements):
 304                                 text = text[:start] + "[%s](%s)" % (t, l) + text[end:]
 305
 306                 # Borrow this from the blog
 307                 return self.backend.blog._render_text(text, lang="markdown")
 308
 309         @property
 310         def markdown(self):
 311                 return self.data.markdown or ""
 312
 313         @property
 314         def html(self):
 315                 return self.data.html or self._render(self.markdown)
 316
 317         @property
 318         def timestamp(self):
 319                 return self.data.timestamp
 320
 321         def was_deleted(self):
 322                 return self.markdown is None
 323
 324         @lazy_property
 325         def breadcrumbs(self):
 326                 return self.backend.wiki.make_breadcrumbs(self.page)
 327
 328         def get_latest_revision(self):
 329                 revisions = self.get_revisions()
 330
 331                 # Return first object
 332                 for rev in revisions:
 333                         return rev
 334
 335         def get_revisions(self):
 336                 return self.backend.wiki._get_pages("SELECT * FROM wiki \
 337                         WHERE page = %s ORDER BY timestamp DESC", self.page)
 338
 339         @lazy_property
 340         def previous_revision(self):
 341                 return self.backend.wiki._get_page("SELECT * FROM wiki \
 342                         WHERE page = %s AND timestamp < %s ORDER BY timestamp DESC \
 343                         LIMIT 1", self.page, self.timestamp)
 344
 345         @property
 346         def changes(self):
 347                 return self.data.changes
 348
 349         # ACL
 350
 351         def check_acl(self, account):
 352                 return self.backend.wiki.check_acl(self.page, account)
 353
 354         # Sidebar
 355
 356         @lazy_property
 357         def sidebar(self):
 358                 parts = self.page.split("/")
 359
 360                 while parts:
 361                         sidebar = self.backend.wiki.get_page("%s/sidebar" % os.path.join(*parts))
 362                         if sidebar:
 363                                 return sidebar
 364
 365                         parts.pop()
 366
 367         # Watchers
 368
 369         @property
 370         def watchers(self):
 371                 res = self.db.query("SELECT uid FROM wiki_watchlist \
 372                         WHERE page = %s", self.page)
 373
 374                 for row in res:
 375                         # Search for account by UID and skip if none was found
 376                         account = self.backend.accounts.get_by_uid(row.uid)
 377                         if not account:
 378                                 continue
 379
 380                         # Return the account
 381                         yield account
 382
 383         def is_watched_by(self, account):
 384                 res = self.db.get("SELECT 1 FROM wiki_watchlist \
 385                         WHERE page = %s AND uid = %s", self.page, account.uid)
 386
 387                 if res:
 388                         return True
 389
 390                 return False
 391
 392         def add_watcher(self, account):
 393                 if self.is_watched_by(account):
 394                         return
 395
 396                 self.db.execute("INSERT INTO wiki_watchlist(page, uid) \
 397                         VALUES(%s, %s)", self.page, account.uid)
 398
 399         def remove_watcher(self, account):
 400                 self.db.execute("DELETE FROM wiki_watchlist \
 401                         WHERE page = %s AND uid = %s", self.page, account.uid)
 402
 403         def _send_watcher_emails(self, excludes=[]):
 404                 # Nothing to do if there was no previous revision
 405                 if not self.previous_revision:
 406                         return
 407
 408                 for watcher in self.watchers:
 409                         # Skip everyone who is excluded
 410                         if watcher in excludes:
 411                                 logging.debug("Excluding %s" % watcher)
 412                                 continue
 413
 414                         logging.debug("Sending watcher email to %s" % watcher)
 415
 416                         pass # TODO
 417
 418
 419 class File(misc.Object):
 420         def init(self, id, data):
 421                 self.id   = id
 422                 self.data = data
 423
 424         @property
 425         def url(self):
 426                 return os.path.join(self.path, self.filename)
 427
 428         @property
 429         def path(self):
 430                 return self.data.path
 431
 432         @property
 433         def filename(self):
 434                 return self.data.filename
 435
 436         @property
 437         def mimetype(self):
 438                 return self.data.mimetype
 439
 440         @property
 441         def size(self):
 442                 return self.data.size
 443
 444         @lazy_property
 445         def author(self):
 446                 if self.data.author_uid:
 447                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 448
 449         @property
 450         def created_at(self):
 451                 return self.data.created_at
 452
 453         def is_pdf(self):
 454                 return self.mimetype in ("application/pdf", "application/x-pdf")
 455
 456         def is_image(self):
 457                 return self.mimetype.startswith("image/")
 458
 459         @lazy_property
 460         def blob(self):
 461                 res = self.db.get("SELECT data FROM wiki_blobs \
 462                         WHERE id = %s", self.data.blob_id)
 463
 464                 if res:
 465                         return bytes(res.data)
 466
 467         def get_thumbnail(self, size):
 468                 cache_key = "-".join((self.path, util.normalize(self.filename), self.created_at.isoformat(), "%spx" % size))
 469
 470                 # Try to fetch the data from the cache
 471                 thumbnail = self.memcache.get(cache_key)
 472                 if thumbnail:
 473                         return thumbnail
 474
 475                 # Generate the thumbnail
 476                 thumbnail = self._generate_thumbnail(size)
 477
 478                 # Put it into the cache for forever
 479                 self.memcache.set(cache_key, thumbnail)
 480
 481                 return thumbnail
 482
 483         def _generate_thumbnail(self, size):
 484                 image = PIL.Image.open(io.BytesIO(self.blob))
 485
 486                 # Resize the image to the desired resolution
 487                 image.thumbnail((size, size), PIL.Image.ANTIALIAS)
 488
 489                 with io.BytesIO() as f:
 490                         # If writing out the image does not work with optimization,
 491                         # we try to write it out without any optimization.
 492                         try:
 493                                 image.save(f, image.format, optimize=True, quality=98)
 494                         except:
 495                                 image.save(f, image.format, quality=98)
 496
 497                         return f.getvalue()