src/backend/wiki.py

   1 #!/usr/bin/python3
   2
   3 import difflib
   4 import logging
   5 import os.path
   6 import re
   7 import tornado.gen
   8 import urllib.parse
   9
  10 from . import misc
  11 from . import util
  12 from .decorators import *
  13
  14 class Wiki(misc.Object):
  15         def _get_pages(self, query, *args):
  16                 res = self.db.query(query, *args)
  17
  18                 for row in res:
  19                         yield Page(self.backend, row.id, data=row)
  20
  21         def _get_page(self, query, *args):
  22                 res = self.db.get(query, *args)
  23
  24                 if res:
  25                         return Page(self.backend, res.id, data=res)
  26
  27         def make_path(self, page, path):
  28                 # Nothing to do for absolute links
  29                 if path.startswith("/"):
  30                         pass
  31
  32                 # Relative links (one-level down)
  33                 elif path.startswith("./"):
  34                         path = os.path.join(page, path)
  35
  36                 # All other relative links
  37                 else:
  38                         p = os.path.dirname(page)
  39                         path = os.path.join(p, path)
  40
  41                 # Normalise links
  42                 return os.path.normpath(path)
  43
  44         def page_exists(self, path):
  45                 page = self.get_page(path)
  46
  47                 # Page must have been found and not deleted
  48                 return page and not page.was_deleted()
  49
  50         def get_page_title(self, page, default=None):
  51                 # Try to retrieve title from cache
  52                 title = self.memcache.get("wiki:title:%s" % page)
  53                 if title:
  54                         return title
  55
  56                 # If the title has not been in the cache, we will
  57                 # have to look it up
  58                 doc = self.get_page(page)
  59                 if doc:
  60                         title = doc.title
  61                 else:
  62                         title = os.path.basename(page)
  63
  64                 # Save in cache for forever
  65                 self.memcache.set("wiki:title:%s" % page, title)
  66
  67                 return title
  68
  69         def get_page(self, page, revision=None):
  70                 page = Page.sanitise_page_name(page)
  71                 assert page
  72
  73                 if revision:
  74                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
  75                                 AND timestamp = %s", page, revision)
  76                 else:
  77                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
  78                                 ORDER BY timestamp DESC LIMIT 1", page)
  79
  80         def get_recent_changes(self, account, limit=None):
  81                 pages = self._get_pages("SELECT * FROM wiki \
  82                         ORDER BY timestamp DESC")
  83
  84                 for page in pages:
  85                         if not page.check_acl(account):
  86                                 continue
  87
  88                         yield page
  89
  90                         limit -= 1
  91                         if not limit:
  92                                 break
  93
  94         def create_page(self, page, author, content, changes=None, address=None):
  95                 page = Page.sanitise_page_name(page)
  96
  97                 # Write page to the database
  98                 page = self._get_page("INSERT INTO wiki(page, author_uid, markdown, changes, address) \
  99                         VALUES(%s, %s, %s, %s, %s) RETURNING *", page, author.uid, content or None, changes, address)
 100
 101                 # Update cache
 102                 self.memcache.set("wiki:title:%s" % page.page, page.title)
 103
 104                 # Send email to all watchers
 105                 page._send_watcher_emails(excludes=[author])
 106
 107                 return page
 108
 109         def delete_page(self, page, author, **kwargs):
 110                 # Do nothing if the page does not exist
 111                 if not self.get_page(page):
 112                         return
 113
 114                 # Just creates a blank last version of the page
 115                 self.create_page(page, author=author, content=None, **kwargs)
 116
 117         def make_breadcrumbs(self, url):
 118                 # Split and strip all empty elements (double slashes)
 119                 parts = list(e for e in url.split("/") if e)
 120
 121                 ret = []
 122                 for part in ("/".join(parts[:i]) for i in range(1, len(parts))):
 123                         ret.append(("/%s" % part, self.get_page_title(part, os.path.basename(part))))
 124
 125                 return ret
 126
 127         def search(self, query, account=None, limit=None):
 128                 query = util.parse_search_query(query)
 129
 130                 res = self._get_pages("SELECT wiki.* FROM wiki_search_index search_index \
 131                         LEFT JOIN wiki ON search_index.wiki_id = wiki.id \
 132                         WHERE search_index.document @@ to_tsquery('english', %s) \
 133                                 ORDER BY ts_rank(search_index.document, to_tsquery('english', %s)) DESC",
 134                         query, query)
 135
 136                 pages = []
 137                 for page in res:
 138                         # Skip any pages the user doesn't have permission for
 139                         if not page.check_acl(account):
 140                                 continue
 141
 142                         # Return any other pages
 143                         pages.append(page)
 144
 145                         # Break when we have found enough pages
 146                         if limit and len(pages) >= limit:
 147                                 break
 148
 149                 return pages
 150
 151         def refresh(self):
 152                 """
 153                         Needs to be called after a page has been changed
 154                 """
 155                 self.db.execute("REFRESH MATERIALIZED VIEW wiki_search_index")
 156
 157         def get_watchlist(self, account):
 158                 pages = self._get_pages(
 159                         "WITH pages AS (SELECT * FROM wiki_current \
 160                                         LEFT JOIN wiki ON wiki_current.id = wiki.id) \
 161                         SELECT * FROM wiki_watchlist watchlist \
 162                                 LEFT JOIN pages ON watchlist.page = pages.page \
 163                                 WHERE watchlist.uid = %s",
 164                         account.uid,
 165                 )
 166
 167                 return sorted(pages)
 168
 169         # ACL
 170
 171         def check_acl(self, page, account):
 172                 res = self.db.query("SELECT * FROM wiki_acls \
 173                         WHERE %s ILIKE (path || '%%') ORDER BY LENGTH(path) DESC LIMIT 1", page)
 174
 175                 for row in res:
 176                         # Access not permitted when user is not logged in
 177                         if not account:
 178                                 return False
 179
 180                         # If user is in a matching group, we grant permission
 181                         for group in row.groups:
 182                                 if group in account.groups:
 183                                         return True
 184
 185                         # Otherwise access is not permitted
 186                         return False
 187
 188                 # If no ACLs are found, we permit access
 189                 return True
 190
 191         # Files
 192
 193         def _get_files(self, query, *args):
 194                 res = self.db.query(query, *args)
 195
 196                 for row in res:
 197                         yield File(self.backend, row.id, data=row)
 198
 199         def _get_file(self, query, *args):
 200                 res = self.db.get(query, *args)
 201
 202                 if res:
 203                         return File(self.backend, res.id, data=res)
 204
 205         def get_files(self, path):
 206                 files = self._get_files("SELECT * FROM wiki_files \
 207                         WHERE path = %s AND deleted_at IS NULL ORDER BY filename", path)
 208
 209                 return list(files)
 210
 211         def get_file_by_path(self, path, revision=None):
 212                 path, filename = os.path.dirname(path), os.path.basename(path)
 213
 214                 if revision:
 215                         # Fetch a specific revision
 216                         return self._get_file("SELECT * FROM wiki_files \
 217                                 WHERE path = %s AND filename = %s AND created_at <= %s \
 218                                 ORDER BY created_at DESC LIMIT 1", path, filename, revision)
 219
 220                 # Fetch latest version
 221                 return self._get_file("SELECT * FROM wiki_files \
 222                         WHERE path = %s AND filename = %s AND deleted_at IS NULL",
 223                         path, filename)
 224
 225         def get_file_by_path_and_filename(self, path, filename):
 226                 return self._get_file("SELECT * FROM wiki_files \
 227                         WHERE path = %s AND filename = %s AND deleted_at IS NULL",
 228                         path, filename)
 229
 230         def upload(self, path, filename, data, mimetype, author, address):
 231                 # Replace any existing files
 232                 file = self.get_file_by_path_and_filename(path, filename)
 233                 if file:
 234                         file.delete(author)
 235
 236                 # Upload the blob first
 237                 blob = self.db.get("INSERT INTO wiki_blobs(data) VALUES(%s) \
 238                         ON CONFLICT (digest(data, %s)) DO UPDATE SET data = EXCLUDED.data \
 239                         RETURNING id", data, "MD5")
 240
 241                 # Create entry for file
 242                 return self._get_file("INSERT INTO wiki_files(path, filename, author_uid, address, \
 243                         mimetype, blob_id, size) VALUES(%s,  %s, %s, %s, %s, %s, %s) RETURNING *", path,
 244                         filename, author.uid, address, mimetype, blob.id, len(data))
 245
 246         def render(self, path, text):
 247                 r = WikiRenderer(self.backend, path)
 248
 249                 return r.render(text)
 250
 251
 252 class Page(misc.Object):
 253         def init(self, id, data=None):
 254                 self.id = id
 255                 self.data = data
 256
 257         def __repr__(self):
 258                 return "<%s %s %s>" % (self.__class__.__name__, self.page, self.timestamp)
 259
 260         def __eq__(self, other):
 261                 if isinstance(other, self.__class__):
 262                         return self.id == other.id
 263
 264         def __lt__(self, other):
 265                 if isinstance(other, self.__class__):
 266                         if self.page == other.page:
 267                                 return self.timestamp < other.timestamp
 268
 269                         return self.page < other.page
 270
 271         @staticmethod
 272         def sanitise_page_name(page):
 273                 if not page:
 274                         return "/"
 275
 276                 # Make sure that the page name does NOT end with a /
 277                 if page.endswith("/"):
 278                         page = page[:-1]
 279
 280                 # Make sure the page name starts with a /
 281                 if not page.startswith("/"):
 282                         page = "/%s" % page
 283
 284                 # Remove any double slashes
 285                 page = page.replace("//", "/")
 286
 287                 return page
 288
 289         @property
 290         def url(self):
 291                 return self.page
 292
 293         @property
 294         def full_url(self):
 295                 return "https://wiki.ipfire.org%s" % self.url
 296
 297         @property
 298         def page(self):
 299                 return self.data.page
 300
 301         @property
 302         def title(self):
 303                 return self._title or os.path.basename(self.page[1:])
 304
 305         @property
 306         def _title(self):
 307                 if not self.markdown:
 308                         return
 309
 310                 # Find first H1 headline in markdown
 311                 markdown = self.markdown.splitlines()
 312
 313                 m = re.match(r"^#\s*(.*)( #)?$", markdown[0])
 314                 if m:
 315                         return m.group(1)
 316
 317         @lazy_property
 318         def author(self):
 319                 if self.data.author_uid:
 320                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 321
 322         @property
 323         def markdown(self):
 324                 return self.data.markdown or ""
 325
 326         @property
 327         def html(self):
 328                 return self.backend.wiki.render(self.page, self.markdown)
 329
 330         @property
 331         def timestamp(self):
 332                 return self.data.timestamp
 333
 334         def was_deleted(self):
 335                 return self.markdown is None
 336
 337         @lazy_property
 338         def breadcrumbs(self):
 339                 return self.backend.wiki.make_breadcrumbs(self.page)
 340
 341         def is_latest_revision(self):
 342                 return self.get_latest_revision() == self
 343
 344         def get_latest_revision(self):
 345                 revisions = self.get_revisions()
 346
 347                 # Return first object
 348                 for rev in revisions:
 349                         return rev
 350
 351         def get_revisions(self):
 352                 return self.backend.wiki._get_pages("SELECT * FROM wiki \
 353                         WHERE page = %s ORDER BY timestamp DESC", self.page)
 354
 355         @lazy_property
 356         def previous_revision(self):
 357                 return self.backend.wiki._get_page("SELECT * FROM wiki \
 358                         WHERE page = %s AND timestamp < %s ORDER BY timestamp DESC \
 359                         LIMIT 1", self.page, self.timestamp)
 360
 361         @property
 362         def changes(self):
 363                 return self.data.changes
 364
 365         # ACL
 366
 367         def check_acl(self, account):
 368                 return self.backend.wiki.check_acl(self.page, account)
 369
 370         # Sidebar
 371
 372         @lazy_property
 373         def sidebar(self):
 374                 parts = self.page.split("/")
 375
 376                 while parts:
 377                         sidebar = self.backend.wiki.get_page("%s/sidebar" % os.path.join(*parts))
 378                         if sidebar:
 379                                 return sidebar
 380
 381                         parts.pop()
 382
 383         # Watchers
 384
 385         @lazy_property
 386         def diff(self):
 387                 if self.previous_revision:
 388                         diff = difflib.unified_diff(
 389                                 self.previous_revision.markdown.splitlines(),
 390                                 self.markdown.splitlines(),
 391                         )
 392
 393                         return "\n".join(diff)
 394
 395         @property
 396         def watchers(self):
 397                 res = self.db.query("SELECT uid FROM wiki_watchlist \
 398                         WHERE page = %s", self.page)
 399
 400                 for row in res:
 401                         # Search for account by UID and skip if none was found
 402                         account = self.backend.accounts.get_by_uid(row.uid)
 403                         if not account:
 404                                 continue
 405
 406                         # Return the account
 407                         yield account
 408
 409         def is_watched_by(self, account):
 410                 res = self.db.get("SELECT 1 FROM wiki_watchlist \
 411                         WHERE page = %s AND uid = %s", self.page, account.uid)
 412
 413                 if res:
 414                         return True
 415
 416                 return False
 417
 418         def add_watcher(self, account):
 419                 if self.is_watched_by(account):
 420                         return
 421
 422                 self.db.execute("INSERT INTO wiki_watchlist(page, uid) \
 423                         VALUES(%s, %s)", self.page, account.uid)
 424
 425         def remove_watcher(self, account):
 426                 self.db.execute("DELETE FROM wiki_watchlist \
 427                         WHERE page = %s AND uid = %s", self.page, account.uid)
 428
 429         def _send_watcher_emails(self, excludes=[]):
 430                 # Nothing to do if there was no previous revision
 431                 if not self.previous_revision:
 432                         return
 433
 434                 for watcher in self.watchers:
 435                         # Skip everyone who is excluded
 436                         if watcher in excludes:
 437                                 logging.debug("Excluding %s" % watcher)
 438                                 continue
 439
 440                         # Check permissions
 441                         if not self.backend.wiki.check_acl(self.page, watcher):
 442                                 logging.debug("Watcher %s does not have permissions" % watcher)
 443                                 continue
 444
 445                         logging.debug("Sending watcher email to %s" % watcher)
 446
 447                         # Compose message
 448                         self.backend.messages.send_template("wiki/messages/page-changed",
 449                                 sender="IPFire Wiki <wiki@ipfire.org>", recipients=[watcher],
 450                                 page=self, priority=-10)
 451
 452         def restore(self, author, address):
 453                 changes = "Restore to revision from %s" % self.timestamp.isoformat()
 454
 455                 return self.backend.wiki.create_page(self.page,
 456                         author, self.markdown, changes=changes, address=address)
 457
 458
 459 class File(misc.Object):
 460         def init(self, id, data):
 461                 self.id   = id
 462                 self.data = data
 463
 464         def __eq__(self, other):
 465                 if isinstance(other, self.__class__):
 466                         return self.id == other.id
 467
 468         @property
 469         def url(self):
 470                 return os.path.join(self.path, self.filename)
 471
 472         @property
 473         def path(self):
 474                 return self.data.path
 475
 476         @property
 477         def filename(self):
 478                 return self.data.filename
 479
 480         @property
 481         def mimetype(self):
 482                 return self.data.mimetype
 483
 484         @property
 485         def size(self):
 486                 return self.data.size
 487
 488         @lazy_property
 489         def author(self):
 490                 if self.data.author_uid:
 491                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 492
 493         @property
 494         def created_at(self):
 495                 return self.data.created_at
 496
 497         def delete(self, author=None):
 498                 self.db.execute("UPDATE wiki_files SET deleted_at = NOW(), deleted_by = %s \
 499                         WHERE id = %s", author.uid if author else None, self.id)
 500
 501         @property
 502         def deleted_at(self):
 503                 return self.data.deleted_at
 504
 505         def get_latest_revision(self):
 506                 revisions = self.get_revisions()
 507
 508                 # Return first object
 509                 for rev in revisions:
 510                         return rev
 511
 512         def get_revisions(self):
 513                 revisions = self.backend.wiki._get_files("SELECT * FROM wiki_files \
 514                         WHERE path = %s ORDER BY created_at DESC", self.path)
 515
 516                 return list(revisions)
 517
 518         def is_pdf(self):
 519                 return self.mimetype in ("application/pdf", "application/x-pdf")
 520
 521         def is_image(self):
 522                 return self.mimetype.startswith("image/")
 523
 524         @lazy_property
 525         def blob(self):
 526                 res = self.db.get("SELECT data FROM wiki_blobs \
 527                         WHERE id = %s", self.data.blob_id)
 528
 529                 if res:
 530                         return bytes(res.data)
 531
 532         def get_thumbnail(self, size):
 533                 cache_key = "-".join((self.path, util.normalize(self.filename), self.created_at.isoformat(), "%spx" % size))
 534
 535                 # Try to fetch the data from the cache
 536                 thumbnail = self.memcache.get(cache_key)
 537                 if thumbnail:
 538                         return thumbnail
 539
 540                 # Generate the thumbnail
 541                 thumbnail = util.generate_thumbnail(self.blob, size)
 542
 543                 # Put it into the cache for forever
 544                 self.memcache.set(cache_key, thumbnail)
 545
 546                 return thumbnail
 547
 548
 549 class WikiRenderer(misc.Object):
 550         schemas = (
 551                 "ftp://",
 552                 "git://",
 553                 "http://",
 554                 "https://",
 555                 "rsync://",
 556                 "sftp://",
 557                 "ssh://",
 558                 "webcal://",
 559         )
 560
 561         # Links
 562         links = re.compile(r"<a href=\"(.*?)\">(.*?)</a>")
 563
 564         # Images
 565         images = re.compile(r"<img alt(?:=\"(.*?)\")? src=\"(.*?)\" (?:title=\"(.*?)\" )?/>")
 566
 567         def init(self, path):
 568                 self.path = path
 569
 570         def _render_link(self, m):
 571                 url, text = m.groups()
 572
 573                 # Emails
 574                 if "@" in url:
 575                         # Strip mailto:
 576                         if url.startswith("mailto:"):
 577                                 url = url[7:]
 578
 579                         return """<a class="link-external" href="mailto:%s">%s</a>""" % \
 580                                 (url, text or url)
 581
 582                 # External Links
 583                 for schema in self.schemas:
 584                         if url.startswith(schema):
 585                                 return """<a class="link-external" href="%s">%s</a>""" % \
 586                                         (url, text or url)
 587
 588                 # Everything else must be an internal link
 589                 path = self.backend.wiki.make_path(self.path, url)
 590
 591                 return """<a href="%s">%s</a>""" % \
 592                         (path, text or self.backend.wiki.get_page_title(path))
 593
 594         def _render_image(self, m):
 595                 alt_text, url, caption = m.groups()
 596
 597                 # Skip any absolute and external URLs
 598                 if url.startswith("/") or url.startswith("https://") or url.startswith("http://"):
 599                         return """<figure class="figure"><img src="%s" class="figure-img img-fluid rounded" alt="%s">
 600                                 <figcaption class="figure-caption">%s</figcaption></figure>
 601                         """ % (url, alt_text, caption or "")
 602
 603                 # Try to split query string
 604                 url, delimiter, qs = url.partition("?")
 605
 606                 # Parse query arguments
 607                 args = urllib.parse.parse_qs(qs)
 608
 609                 # Build absolute path
 610                 url = self.backend.wiki.make_path(self.path, url)
 611
 612                 # Find image
 613                 file = self.backend.wiki.get_file_by_path(url)
 614                 if not file or not file.is_image():
 615                         return "<!-- Could not find image %s in %s -->" % (url, self.path)
 616
 617                 # Scale down the image if not already done
 618                 if not "s" in args:
 619                         args["s"] = "920"
 620
 621                 return """<figure class="figure"><img src="%s?%s" class="figure-img img-fluid rounded" alt="%s">
 622                 <figcaption class="figure-caption">%s</figcaption></figure>
 623                 """ % (url, urllib.parse.urlencode(args), caption, caption or "")
 624
 625         def render(self, text):
 626                 logging.debug("Rendering %s" % self.path)
 627
 628                 # Borrow this from the blog
 629                 text = self.backend.blog._render_text(text, lang="markdown")
 630
 631                 # Postprocess links
 632                 text = self.links.sub(self._render_link, text)
 633
 634                 # Postprocess images to <figure>
 635                 text = self.images.sub(self._render_image, text)
 636
 637                 return text