src/backend/wiki.py

   1 #!/usr/bin/python3
   2
   3 import difflib
   4 import logging
   5 import os.path
   6 import re
   7 import urllib.parse
   8
   9 from . import misc
  10 from . import util
  11 from .decorators import *
  12
  13 class Wiki(misc.Object):
  14         def _get_pages(self, query, *args):
  15                 res = self.db.query(query, *args)
  16
  17                 for row in res:
  18                         yield Page(self.backend, row.id, data=row)
  19
  20         def _get_page(self, query, *args):
  21                 res = self.db.get(query, *args)
  22
  23                 if res:
  24                         return Page(self.backend, res.id, data=res)
  25
  26         def make_path(self, page, path):
  27                 # Nothing to do for absolute links
  28                 if path.startswith("/"):
  29                         pass
  30
  31                 # Relative links (one-level down)
  32                 elif path.startswith("./"):
  33                         path = os.path.join(page, path)
  34
  35                 # All other relative links
  36                 else:
  37                         p = os.path.dirname(page)
  38                         path = os.path.join(p, path)
  39
  40                 # Normalise links
  41                 return os.path.normpath(path)
  42
  43         def page_exists(self, path):
  44                 page = self.get_page(path)
  45
  46                 # Page must have been found and not deleted
  47                 return page and not page.was_deleted()
  48
  49         def get_page_title(self, page, default=None):
  50                 # Try to retrieve title from cache
  51                 title = self.memcache.get("wiki:title:%s" % page)
  52                 if title:
  53                         return title
  54
  55                 # If the title has not been in the cache, we will
  56                 # have to look it up
  57                 doc = self.get_page(page)
  58                 if doc:
  59                         title = doc.title
  60                 else:
  61                         title = os.path.basename(page)
  62
  63                 # Save in cache for forever
  64                 self.memcache.set("wiki:title:%s" % page, title)
  65
  66                 return title
  67
  68         def get_page(self, page, revision=None):
  69                 page = Page.sanitise_page_name(page)
  70                 assert page
  71
  72                 if revision:
  73                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
  74                                 AND timestamp = %s", page, revision)
  75                 else:
  76                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
  77                                 ORDER BY timestamp DESC LIMIT 1", page)
  78
  79         def get_recent_changes(self, account, limit=None):
  80                 pages = self._get_pages("SELECT * FROM wiki \
  81                         ORDER BY timestamp DESC")
  82
  83                 for page in pages:
  84                         if not page.check_acl(account):
  85                                 continue
  86
  87                         yield page
  88
  89                         limit -= 1
  90                         if not limit:
  91                                 break
  92
  93         def create_page(self, page, author, content, changes=None, address=None):
  94                 page = Page.sanitise_page_name(page)
  95
  96                 # Write page to the database
  97                 page = self._get_page("INSERT INTO wiki(page, author_uid, markdown, changes, address) \
  98                         VALUES(%s, %s, %s, %s, %s) RETURNING *", page, author.uid, content or None, changes, address)
  99
 100                 # Update cache
 101                 self.memcache.set("wiki:title:%s" % page.page, page.title)
 102
 103                 # Send email to all watchers
 104                 page._send_watcher_emails(excludes=[author])
 105
 106                 return page
 107
 108         def delete_page(self, page, author, **kwargs):
 109                 # Do nothing if the page does not exist
 110                 if not self.get_page(page):
 111                         return
 112
 113                 # Just creates a blank last version of the page
 114                 self.create_page(page, author=author, content=None, **kwargs)
 115
 116         def make_breadcrumbs(self, url):
 117                 # Split and strip all empty elements (double slashes)
 118                 parts = list(e for e in url.split("/") if e)
 119
 120                 ret = []
 121                 for part in ("/".join(parts[:i]) for i in range(1, len(parts))):
 122                         ret.append(("/%s" % part, self.get_page_title(part, os.path.basename(part))))
 123
 124                 return ret
 125
 126         def search(self, query, account=None, limit=None):
 127                 res = self._get_pages("SELECT wiki.* FROM wiki_search_index search_index \
 128                         LEFT JOIN wiki ON search_index.wiki_id = wiki.id \
 129                         WHERE search_index.document @@ websearch_to_tsquery('english', %s) \
 130                                 ORDER BY ts_rank(search_index.document, websearch_to_tsquery('english', %s)) DESC",
 131                         query, query)
 132
 133                 pages = []
 134                 for page in res:
 135                         # Skip any pages the user doesn't have permission for
 136                         if not page.check_acl(account):
 137                                 continue
 138
 139                         # Return any other pages
 140                         pages.append(page)
 141
 142                         # Break when we have found enough pages
 143                         if limit and len(pages) >= limit:
 144                                 break
 145
 146                 return pages
 147
 148         def refresh(self):
 149                 """
 150                         Needs to be called after a page has been changed
 151                 """
 152                 self.db.execute("REFRESH MATERIALIZED VIEW wiki_search_index")
 153
 154         def get_watchlist(self, account):
 155                 pages = self._get_pages(
 156                         "WITH pages AS (SELECT * FROM wiki_current \
 157                                         LEFT JOIN wiki ON wiki_current.id = wiki.id) \
 158                         SELECT * FROM wiki_watchlist watchlist \
 159                                 LEFT JOIN pages ON watchlist.page = pages.page \
 160                                 WHERE watchlist.uid = %s",
 161                         account.uid,
 162                 )
 163
 164                 return sorted(pages)
 165
 166         # ACL
 167
 168         def check_acl(self, page, account):
 169                 res = self.db.query("SELECT * FROM wiki_acls \
 170                         WHERE %s ILIKE (path || '%%') ORDER BY LENGTH(path) DESC LIMIT 1", page)
 171
 172                 for row in res:
 173                         # Access not permitted when user is not logged in
 174                         if not account:
 175                                 return False
 176
 177                         # If user is in a matching group, we grant permission
 178                         for group in row.groups:
 179                                 if account.is_member_of_group(group):
 180                                         return True
 181
 182                         # Otherwise access is not permitted
 183                         return False
 184
 185                 # If no ACLs are found, we permit access
 186                 return True
 187
 188         # Files
 189
 190         def _get_files(self, query, *args):
 191                 res = self.db.query(query, *args)
 192
 193                 for row in res:
 194                         yield File(self.backend, row.id, data=row)
 195
 196         def _get_file(self, query, *args):
 197                 res = self.db.get(query, *args)
 198
 199                 if res:
 200                         return File(self.backend, res.id, data=res)
 201
 202         def get_files(self, path):
 203                 files = self._get_files("SELECT * FROM wiki_files \
 204                         WHERE path = %s AND deleted_at IS NULL ORDER BY filename", path)
 205
 206                 return list(files)
 207
 208         def get_file_by_path(self, path, revision=None):
 209                 path, filename = os.path.dirname(path), os.path.basename(path)
 210
 211                 if revision:
 212                         # Fetch a specific revision
 213                         return self._get_file("SELECT * FROM wiki_files \
 214                                 WHERE path = %s AND filename = %s AND created_at <= %s \
 215                                 ORDER BY created_at DESC LIMIT 1", path, filename, revision)
 216
 217                 # Fetch latest version
 218                 return self._get_file("SELECT * FROM wiki_files \
 219                         WHERE path = %s AND filename = %s AND deleted_at IS NULL",
 220                         path, filename)
 221
 222         def get_file_by_path_and_filename(self, path, filename):
 223                 return self._get_file("SELECT * FROM wiki_files \
 224                         WHERE path = %s AND filename = %s AND deleted_at IS NULL",
 225                         path, filename)
 226
 227         def upload(self, path, filename, data, mimetype, author, address):
 228                 # Replace any existing files
 229                 file = self.get_file_by_path_and_filename(path, filename)
 230                 if file:
 231                         file.delete(author)
 232
 233                 # Upload the blob first
 234                 blob = self.db.get("INSERT INTO wiki_blobs(data) VALUES(%s) \
 235                         ON CONFLICT (digest(data, %s)) DO UPDATE SET data = EXCLUDED.data \
 236                         RETURNING id", data, "MD5")
 237
 238                 # Create entry for file
 239                 return self._get_file("INSERT INTO wiki_files(path, filename, author_uid, address, \
 240                         mimetype, blob_id, size) VALUES(%s,  %s, %s, %s, %s, %s, %s) RETURNING *", path,
 241                         filename, author.uid, address, mimetype, blob.id, len(data))
 242
 243         def render(self, path, text):
 244                 r = WikiRenderer(self.backend, path)
 245
 246                 return r.render(text)
 247
 248
 249 class Page(misc.Object):
 250         def init(self, id, data=None):
 251                 self.id = id
 252                 self.data = data
 253
 254         def __repr__(self):
 255                 return "<%s %s %s>" % (self.__class__.__name__, self.page, self.timestamp)
 256
 257         def __eq__(self, other):
 258                 if isinstance(other, self.__class__):
 259                         return self.id == other.id
 260
 261         def __lt__(self, other):
 262                 if isinstance(other, self.__class__):
 263                         if self.page == other.page:
 264                                 return self.timestamp < other.timestamp
 265
 266                         return self.page < other.page
 267
 268         @staticmethod
 269         def sanitise_page_name(page):
 270                 if not page:
 271                         return "/"
 272
 273                 # Make sure that the page name does NOT end with a /
 274                 if page.endswith("/"):
 275                         page = page[:-1]
 276
 277                 # Make sure the page name starts with a /
 278                 if not page.startswith("/"):
 279                         page = "/%s" % page
 280
 281                 # Remove any double slashes
 282                 page = page.replace("//", "/")
 283
 284                 return page
 285
 286         @property
 287         def url(self):
 288                 return self.page
 289
 290         @property
 291         def full_url(self):
 292                 return "https://wiki.ipfire.org%s" % self.url
 293
 294         @property
 295         def page(self):
 296                 return self.data.page
 297
 298         @property
 299         def title(self):
 300                 return self._title or os.path.basename(self.page[1:])
 301
 302         @property
 303         def _title(self):
 304                 if not self.markdown:
 305                         return
 306
 307                 # Find first H1 headline in markdown
 308                 markdown = self.markdown.splitlines()
 309
 310                 m = re.match(r"^#\s*(.*)( #)?$", markdown[0])
 311                 if m:
 312                         return m.group(1)
 313
 314         @lazy_property
 315         def author(self):
 316                 if self.data.author_uid:
 317                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 318
 319         @property
 320         def markdown(self):
 321                 return self.data.markdown or ""
 322
 323         @property
 324         def html(self):
 325                 return self.backend.wiki.render(self.page, self.markdown)
 326
 327         @property
 328         def timestamp(self):
 329                 return self.data.timestamp
 330
 331         def was_deleted(self):
 332                 return not self.markdown
 333
 334         @lazy_property
 335         def breadcrumbs(self):
 336                 return self.backend.wiki.make_breadcrumbs(self.page)
 337
 338         def is_latest_revision(self):
 339                 return self.get_latest_revision() == self
 340
 341         def get_latest_revision(self):
 342                 revisions = self.get_revisions()
 343
 344                 # Return first object
 345                 for rev in revisions:
 346                         return rev
 347
 348         def get_revisions(self):
 349                 return self.backend.wiki._get_pages("SELECT * FROM wiki \
 350                         WHERE page = %s ORDER BY timestamp DESC", self.page)
 351
 352         @lazy_property
 353         def previous_revision(self):
 354                 return self.backend.wiki._get_page("SELECT * FROM wiki \
 355                         WHERE page = %s AND timestamp < %s ORDER BY timestamp DESC \
 356                         LIMIT 1", self.page, self.timestamp)
 357
 358         @property
 359         def changes(self):
 360                 return self.data.changes
 361
 362         # ACL
 363
 364         def check_acl(self, account):
 365                 return self.backend.wiki.check_acl(self.page, account)
 366
 367         # Sidebar
 368
 369         @lazy_property
 370         def sidebar(self):
 371                 parts = self.page.split("/")
 372
 373                 while parts:
 374                         sidebar = self.backend.wiki.get_page("%s/sidebar" % os.path.join(*parts))
 375                         if sidebar:
 376                                 return sidebar
 377
 378                         parts.pop()
 379
 380         # Watchers
 381
 382         @lazy_property
 383         def diff(self):
 384                 if self.previous_revision:
 385                         diff = difflib.unified_diff(
 386                                 self.previous_revision.markdown.splitlines(),
 387                                 self.markdown.splitlines(),
 388                         )
 389
 390                         return "\n".join(diff)
 391
 392         @property
 393         def watchers(self):
 394                 res = self.db.query("SELECT uid FROM wiki_watchlist \
 395                         WHERE page = %s", self.page)
 396
 397                 for row in res:
 398                         # Search for account by UID and skip if none was found
 399                         account = self.backend.accounts.get_by_uid(row.uid)
 400                         if not account:
 401                                 continue
 402
 403                         # Return the account
 404                         yield account
 405
 406         def is_watched_by(self, account):
 407                 res = self.db.get("SELECT 1 FROM wiki_watchlist \
 408                         WHERE page = %s AND uid = %s", self.page, account.uid)
 409
 410                 if res:
 411                         return True
 412
 413                 return False
 414
 415         def add_watcher(self, account):
 416                 if self.is_watched_by(account):
 417                         return
 418
 419                 self.db.execute("INSERT INTO wiki_watchlist(page, uid) \
 420                         VALUES(%s, %s)", self.page, account.uid)
 421
 422         def remove_watcher(self, account):
 423                 self.db.execute("DELETE FROM wiki_watchlist \
 424                         WHERE page = %s AND uid = %s", self.page, account.uid)
 425
 426         def _send_watcher_emails(self, excludes=[]):
 427                 # Nothing to do if there was no previous revision
 428                 if not self.previous_revision:
 429                         return
 430
 431                 for watcher in self.watchers:
 432                         # Skip everyone who is excluded
 433                         if watcher in excludes:
 434                                 logging.debug("Excluding %s" % watcher)
 435                                 continue
 436
 437                         # Check permissions
 438                         if not self.backend.wiki.check_acl(self.page, watcher):
 439                                 logging.debug("Watcher %s does not have permissions" % watcher)
 440                                 continue
 441
 442                         logging.debug("Sending watcher email to %s" % watcher)
 443
 444                         # Compose message
 445                         self.backend.messages.send_template("wiki/messages/page-changed",
 446                                 account=watcher, page=self, priority=-10)
 447
 448         def restore(self, author, address):
 449                 changes = "Restore to revision from %s" % self.timestamp.isoformat()
 450
 451                 return self.backend.wiki.create_page(self.page,
 452                         author, self.markdown, changes=changes, address=address)
 453
 454
 455 class File(misc.Object):
 456         def init(self, id, data):
 457                 self.id   = id
 458                 self.data = data
 459
 460         def __eq__(self, other):
 461                 if isinstance(other, self.__class__):
 462                         return self.id == other.id
 463
 464         @property
 465         def url(self):
 466                 return os.path.join(self.path, self.filename)
 467
 468         @property
 469         def path(self):
 470                 return self.data.path
 471
 472         @property
 473         def filename(self):
 474                 return self.data.filename
 475
 476         @property
 477         def mimetype(self):
 478                 return self.data.mimetype
 479
 480         @property
 481         def size(self):
 482                 return self.data.size
 483
 484         @lazy_property
 485         def author(self):
 486                 if self.data.author_uid:
 487                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 488
 489         @property
 490         def created_at(self):
 491                 return self.data.created_at
 492
 493         def delete(self, author=None):
 494                 self.db.execute("UPDATE wiki_files SET deleted_at = NOW(), deleted_by = %s \
 495                         WHERE id = %s", author.uid if author else None, self.id)
 496
 497         @property
 498         def deleted_at(self):
 499                 return self.data.deleted_at
 500
 501         def get_latest_revision(self):
 502                 revisions = self.get_revisions()
 503
 504                 # Return first object
 505                 for rev in revisions:
 506                         return rev
 507
 508         def get_revisions(self):
 509                 revisions = self.backend.wiki._get_files("SELECT * FROM wiki_files \
 510                         WHERE path = %s AND filename = %s ORDER BY created_at DESC", self.path, self.filename)
 511
 512                 return list(revisions)
 513
 514         def is_pdf(self):
 515                 return self.mimetype in ("application/pdf", "application/x-pdf")
 516
 517         def is_image(self):
 518                 return self.mimetype.startswith("image/")
 519
 520         @lazy_property
 521         def blob(self):
 522                 res = self.db.get("SELECT data FROM wiki_blobs \
 523                         WHERE id = %s", self.data.blob_id)
 524
 525                 if res:
 526                         return bytes(res.data)
 527
 528         def get_thumbnail(self, size):
 529                 cache_key = "-".join((self.path, util.normalize(self.filename), self.created_at.isoformat(), "%spx" % size))
 530
 531                 # Try to fetch the data from the cache
 532                 thumbnail = self.memcache.get(cache_key)
 533                 if thumbnail:
 534                         return thumbnail
 535
 536                 # Generate the thumbnail
 537                 thumbnail = util.generate_thumbnail(self.blob, size)
 538
 539                 # Put it into the cache for forever
 540                 self.memcache.set(cache_key, thumbnail)
 541
 542                 return thumbnail
 543
 544
 545 class WikiRenderer(misc.Object):
 546         schemas = (
 547                 "ftp://",
 548                 "git://",
 549                 "http://",
 550                 "https://",
 551                 "rsync://",
 552                 "sftp://",
 553                 "ssh://",
 554                 "webcal://",
 555         )
 556
 557         # Links
 558         links = re.compile(r"<a href=\"(.*?)\">(.*?)</a>")
 559
 560         # Images
 561         images = re.compile(r"<img alt(?:=\"(.*?)\")? src=\"(.*?)\" (?:title=\"(.*?)\" )?/>")
 562
 563         def init(self, path):
 564                 self.path = path
 565
 566         def _render_link(self, m):
 567                 url, text = m.groups()
 568
 569                 # Emails
 570                 if "@" in url:
 571                         # Strip mailto:
 572                         if url.startswith("mailto:"):
 573                                 url = url[7:]
 574
 575                         return """<a class="link-external" href="mailto:%s">%s</a>""" % \
 576                                 (url, text or url)
 577
 578                 # External Links
 579                 for schema in self.schemas:
 580                         if url.startswith(schema):
 581                                 return """<a class="link-external" href="%s">%s</a>""" % \
 582                                         (url, text or url)
 583
 584                 # Everything else must be an internal link
 585                 path = self.backend.wiki.make_path(self.path, url)
 586
 587                 return """<a href="%s">%s</a>""" % \
 588                         (path, text or self.backend.wiki.get_page_title(path))
 589
 590         def _render_image(self, m):
 591                 alt_text, url, caption = m.groups()
 592
 593                 # Skip any absolute and external URLs
 594                 if url.startswith("/") or url.startswith("https://") or url.startswith("http://"):
 595                         return """<figure class="figure"><img src="%s" class="figure-img img-fluid rounded" alt="%s">
 596                                 <figcaption class="figure-caption">%s</figcaption></figure>
 597                         """ % (url, alt_text, caption or "")
 598
 599                 # Try to split query string
 600                 url, delimiter, qs = url.partition("?")
 601
 602                 # Parse query arguments
 603                 args = urllib.parse.parse_qs(qs)
 604
 605                 # Build absolute path
 606                 url = self.backend.wiki.make_path(self.path, url)
 607
 608                 # Find image
 609                 file = self.backend.wiki.get_file_by_path(url)
 610                 if not file or not file.is_image():
 611                         return "<!-- Could not find image %s in %s -->" % (url, self.path)
 612
 613                 # Scale down the image if not already done
 614                 if not "s" in args:
 615                         args["s"] = "920"
 616
 617                 return """<figure class="figure"><img src="%s?%s" class="figure-img img-fluid rounded" alt="%s">
 618                 <figcaption class="figure-caption">%s</figcaption></figure>
 619                 """ % (url, urllib.parse.urlencode(args), caption, caption or "")
 620
 621         def render(self, text):
 622                 logging.debug("Rendering %s" % self.path)
 623
 624                 # Borrow this from the blog
 625                 text = self.backend.blog._render_text(text, lang="markdown")
 626
 627                 # Postprocess links
 628                 text = self.links.sub(self._render_link, text)
 629
 630                 # Postprocess images to <figure>
 631                 text = self.images.sub(self._render_image, text)
 632
 633                 return text