src/backend/wiki.py

   1 #!/usr/bin/python3
   2
   3 import difflib
   4 import logging
   5 import os.path
   6 import re
   7 import urllib.parse
   8
   9 from . import misc
  10 from . import util
  11 from .decorators import *
  12
  13 class Wiki(misc.Object):
  14         def _get_pages(self, query, *args):
  15                 res = self.db.query(query, *args)
  16
  17                 for row in res:
  18                         yield Page(self.backend, row.id, data=row)
  19
  20         def _get_page(self, query, *args):
  21                 res = self.db.get(query, *args)
  22
  23                 if res:
  24                         return Page(self.backend, res.id, data=res)
  25
  26         def __iter__(self):
  27                 return self._get_pages(
  28                         "SELECT wiki.* FROM wiki_current current \
  29                                 LEFT JOIN wiki ON current.id = wiki.id \
  30                                 WHERE current.deleted IS FALSE \
  31                                 ORDER BY page",
  32                 )
  33
  34         def make_path(self, page, path):
  35                 # Nothing to do for absolute links
  36                 if path.startswith("/"):
  37                         pass
  38
  39                 # Relative links (one-level down)
  40                 elif path.startswith("./"):
  41                         path = os.path.join(page, path)
  42
  43                 # All other relative links
  44                 else:
  45                         p = os.path.dirname(page)
  46                         path = os.path.join(p, path)
  47
  48                 # Normalise links
  49                 return os.path.normpath(path)
  50
  51         def page_exists(self, path):
  52                 page = self.get_page(path)
  53
  54                 # Page must have been found and not deleted
  55                 return page and not page.was_deleted()
  56
  57         def get_page_title(self, page, default=None):
  58                 # Try to retrieve title from cache
  59                 title = self.memcache.get("wiki:title:%s" % page)
  60                 if title:
  61                         return title
  62
  63                 # If the title has not been in the cache, we will
  64                 # have to look it up
  65                 doc = self.get_page(page)
  66                 if doc:
  67                         title = doc.title
  68                 else:
  69                         title = os.path.basename(page)
  70
  71                 # Save in cache for forever
  72                 self.memcache.set("wiki:title:%s" % page, title)
  73
  74                 return title
  75
  76         def get_page(self, page, revision=None):
  77                 page = Page.sanitise_page_name(page)
  78                 assert page
  79
  80                 if revision:
  81                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
  82                                 AND timestamp = %s", page, revision)
  83                 else:
  84                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
  85                                 ORDER BY timestamp DESC LIMIT 1", page)
  86
  87         def get_recent_changes(self, account, limit=None):
  88                 pages = self._get_pages("SELECT * FROM wiki \
  89                         ORDER BY timestamp DESC")
  90
  91                 for page in pages:
  92                         if not page.check_acl(account):
  93                                 continue
  94
  95                         yield page
  96
  97                         limit -= 1
  98                         if not limit:
  99                                 break
 100
 101         def create_page(self, page, author, content, changes=None, address=None):
 102                 page = Page.sanitise_page_name(page)
 103
 104                 # Write page to the database
 105                 page = self._get_page("INSERT INTO wiki(page, author_uid, markdown, changes, address) \
 106                         VALUES(%s, %s, %s, %s, %s) RETURNING *", page, author.uid, content or None, changes, address)
 107
 108                 # Update cache
 109                 self.memcache.set("wiki:title:%s" % page.page, page.title)
 110
 111                 # Send email to all watchers
 112                 page._send_watcher_emails(excludes=[author])
 113
 114                 return page
 115
 116         def delete_page(self, page, author, **kwargs):
 117                 # Do nothing if the page does not exist
 118                 if not self.get_page(page):
 119                         return
 120
 121                 # Just creates a blank last version of the page
 122                 self.create_page(page, author=author, content=None, **kwargs)
 123
 124         def make_breadcrumbs(self, url):
 125                 # Split and strip all empty elements (double slashes)
 126                 parts = list(e for e in url.split("/") if e)
 127
 128                 ret = []
 129                 for part in ("/".join(parts[:i]) for i in range(1, len(parts))):
 130                         ret.append(("/%s" % part, self.get_page_title(part, os.path.basename(part))))
 131
 132                 return ret
 133
 134         def search(self, query, account=None, limit=None):
 135                 res = self._get_pages("SELECT wiki.* FROM wiki_search_index search_index \
 136                         LEFT JOIN wiki ON search_index.wiki_id = wiki.id \
 137                         WHERE search_index.document @@ websearch_to_tsquery('english', %s) \
 138                                 ORDER BY ts_rank(search_index.document, websearch_to_tsquery('english', %s)) DESC",
 139                         query, query)
 140
 141                 pages = []
 142                 for page in res:
 143                         # Skip any pages the user doesn't have permission for
 144                         if not page.check_acl(account):
 145                                 continue
 146
 147                         # Return any other pages
 148                         pages.append(page)
 149
 150                         # Break when we have found enough pages
 151                         if limit and len(pages) >= limit:
 152                                 break
 153
 154                 return pages
 155
 156         def refresh(self):
 157                 """
 158                         Needs to be called after a page has been changed
 159                 """
 160                 self.db.execute("REFRESH MATERIALIZED VIEW wiki_search_index")
 161
 162         def get_watchlist(self, account):
 163                 pages = self._get_pages(
 164                         "WITH pages AS (SELECT * FROM wiki_current \
 165                                         LEFT JOIN wiki ON wiki_current.id = wiki.id) \
 166                         SELECT * FROM wiki_watchlist watchlist \
 167                                 LEFT JOIN pages ON watchlist.page = pages.page \
 168                                 WHERE watchlist.uid = %s",
 169                         account.uid,
 170                 )
 171
 172                 return sorted(pages)
 173
 174         # ACL
 175
 176         def check_acl(self, page, account):
 177                 res = self.db.query("SELECT * FROM wiki_acls \
 178                         WHERE %s ILIKE (path || '%%') ORDER BY LENGTH(path) DESC LIMIT 1", page)
 179
 180                 for row in res:
 181                         # Access not permitted when user is not logged in
 182                         if not account:
 183                                 return False
 184
 185                         # If user is in a matching group, we grant permission
 186                         for group in row.groups:
 187                                 if account.is_member_of_group(group):
 188                                         return True
 189
 190                         # Otherwise access is not permitted
 191                         return False
 192
 193                 # If no ACLs are found, we permit access
 194                 return True
 195
 196         # Files
 197
 198         def _get_files(self, query, *args):
 199                 res = self.db.query(query, *args)
 200
 201                 for row in res:
 202                         yield File(self.backend, row.id, data=row)
 203
 204         def _get_file(self, query, *args):
 205                 res = self.db.get(query, *args)
 206
 207                 if res:
 208                         return File(self.backend, res.id, data=res)
 209
 210         def get_files(self, path):
 211                 files = self._get_files("SELECT * FROM wiki_files \
 212                         WHERE path = %s AND deleted_at IS NULL ORDER BY filename", path)
 213
 214                 return list(files)
 215
 216         def get_file_by_path(self, path, revision=None):
 217                 path, filename = os.path.dirname(path), os.path.basename(path)
 218
 219                 if revision:
 220                         # Fetch a specific revision
 221                         return self._get_file("SELECT * FROM wiki_files \
 222                                 WHERE path = %s AND filename = %s AND created_at <= %s \
 223                                 ORDER BY created_at DESC LIMIT 1", path, filename, revision)
 224
 225                 # Fetch latest version
 226                 return self._get_file("SELECT * FROM wiki_files \
 227                         WHERE path = %s AND filename = %s AND deleted_at IS NULL",
 228                         path, filename)
 229
 230         def get_file_by_path_and_filename(self, path, filename):
 231                 return self._get_file("SELECT * FROM wiki_files \
 232                         WHERE path = %s AND filename = %s AND deleted_at IS NULL",
 233                         path, filename)
 234
 235         def upload(self, path, filename, data, mimetype, author, address):
 236                 # Replace any existing files
 237                 file = self.get_file_by_path_and_filename(path, filename)
 238                 if file:
 239                         file.delete(author)
 240
 241                 # Upload the blob first
 242                 blob = self.db.get("INSERT INTO wiki_blobs(data) VALUES(%s) \
 243                         ON CONFLICT (digest(data, %s)) DO UPDATE SET data = EXCLUDED.data \
 244                         RETURNING id", data, "MD5")
 245
 246                 # Create entry for file
 247                 return self._get_file("INSERT INTO wiki_files(path, filename, author_uid, address, \
 248                         mimetype, blob_id, size) VALUES(%s,  %s, %s, %s, %s, %s, %s) RETURNING *", path,
 249                         filename, author.uid, address, mimetype, blob.id, len(data))
 250
 251         def render(self, path, text):
 252                 r = WikiRenderer(self.backend, path)
 253
 254                 return r.render(text)
 255
 256
 257 class Page(misc.Object):
 258         def init(self, id, data=None):
 259                 self.id = id
 260                 self.data = data
 261
 262         def __repr__(self):
 263                 return "<%s %s %s>" % (self.__class__.__name__, self.page, self.timestamp)
 264
 265         def __eq__(self, other):
 266                 if isinstance(other, self.__class__):
 267                         return self.id == other.id
 268
 269                 return NotImplemented
 270
 271         def __lt__(self, other):
 272                 if isinstance(other, self.__class__):
 273                         if self.page == other.page:
 274                                 return self.timestamp < other.timestamp
 275
 276                         return self.page < other.page
 277
 278                 return NotImplemented
 279
 280         @staticmethod
 281         def sanitise_page_name(page):
 282                 if not page:
 283                         return "/"
 284
 285                 # Make sure that the page name does NOT end with a /
 286                 if page.endswith("/"):
 287                         page = page[:-1]
 288
 289                 # Make sure the page name starts with a /
 290                 if not page.startswith("/"):
 291                         page = "/%s" % page
 292
 293                 # Remove any double slashes
 294                 page = page.replace("//", "/")
 295
 296                 return page
 297
 298         @property
 299         def url(self):
 300                 return urllib.parse.urljoin("/docs", self.page)
 301
 302         @property
 303         def full_url(self):
 304                 return "https://www.ipfire.org/docs%s" % self.url
 305
 306         @property
 307         def page(self):
 308                 return self.data.page
 309
 310         @property
 311         def title(self):
 312                 return self._title or os.path.basename(self.page[1:])
 313
 314         @property
 315         def _title(self):
 316                 if not self.markdown:
 317                         return
 318
 319                 # Find first H1 headline in markdown
 320                 markdown = self.markdown.splitlines()
 321
 322                 m = re.match(r"^#\s*(.*)( #)?$", markdown[0])
 323                 if m:
 324                         return m.group(1)
 325
 326         @lazy_property
 327         def author(self):
 328                 if self.data.author_uid:
 329                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 330
 331         @property
 332         def markdown(self):
 333                 return self.data.markdown or ""
 334
 335         @property
 336         def html(self):
 337                 return self.backend.wiki.render(self.page, self.markdown)
 338
 339         @property
 340         def timestamp(self):
 341                 return self.data.timestamp
 342
 343         def was_deleted(self):
 344                 return not self.markdown
 345
 346         @lazy_property
 347         def breadcrumbs(self):
 348                 return self.backend.wiki.make_breadcrumbs(self.page)
 349
 350         def is_latest_revision(self):
 351                 return self.get_latest_revision() == self
 352
 353         def get_latest_revision(self):
 354                 revisions = self.get_revisions()
 355
 356                 # Return first object
 357                 for rev in revisions:
 358                         return rev
 359
 360         def get_revisions(self):
 361                 return self.backend.wiki._get_pages("SELECT * FROM wiki \
 362                         WHERE page = %s ORDER BY timestamp DESC", self.page)
 363
 364         @lazy_property
 365         def previous_revision(self):
 366                 return self.backend.wiki._get_page("SELECT * FROM wiki \
 367                         WHERE page = %s AND timestamp < %s ORDER BY timestamp DESC \
 368                         LIMIT 1", self.page, self.timestamp)
 369
 370         @property
 371         def changes(self):
 372                 return self.data.changes
 373
 374         # ACL
 375
 376         def check_acl(self, account):
 377                 return self.backend.wiki.check_acl(self.page, account)
 378
 379         # Sidebar
 380
 381         @lazy_property
 382         def sidebar(self):
 383                 parts = self.page.split("/")
 384
 385                 while parts:
 386                         sidebar = self.backend.wiki.get_page("%s/sidebar" % os.path.join(*parts))
 387                         if sidebar:
 388                                 return sidebar
 389
 390                         parts.pop()
 391
 392         # Watchers
 393
 394         @lazy_property
 395         def diff(self):
 396                 if self.previous_revision:
 397                         diff = difflib.unified_diff(
 398                                 self.previous_revision.markdown.splitlines(),
 399                                 self.markdown.splitlines(),
 400                         )
 401
 402                         return "\n".join(diff)
 403
 404         @property
 405         def watchers(self):
 406                 res = self.db.query("SELECT uid FROM wiki_watchlist \
 407                         WHERE page = %s", self.page)
 408
 409                 for row in res:
 410                         # Search for account by UID and skip if none was found
 411                         account = self.backend.accounts.get_by_uid(row.uid)
 412                         if not account:
 413                                 continue
 414
 415                         # Return the account
 416                         yield account
 417
 418         def is_watched_by(self, account):
 419                 res = self.db.get("SELECT 1 FROM wiki_watchlist \
 420                         WHERE page = %s AND uid = %s", self.page, account.uid)
 421
 422                 if res:
 423                         return True
 424
 425                 return False
 426
 427         def add_watcher(self, account):
 428                 if self.is_watched_by(account):
 429                         return
 430
 431                 self.db.execute("INSERT INTO wiki_watchlist(page, uid) \
 432                         VALUES(%s, %s)", self.page, account.uid)
 433
 434         def remove_watcher(self, account):
 435                 self.db.execute("DELETE FROM wiki_watchlist \
 436                         WHERE page = %s AND uid = %s", self.page, account.uid)
 437
 438         def _send_watcher_emails(self, excludes=[]):
 439                 # Nothing to do if there was no previous revision
 440                 if not self.previous_revision:
 441                         return
 442
 443                 for watcher in self.watchers:
 444                         # Skip everyone who is excluded
 445                         if watcher in excludes:
 446                                 logging.debug("Excluding %s" % watcher)
 447                                 continue
 448
 449                         # Check permissions
 450                         if not self.backend.wiki.check_acl(self.page, watcher):
 451                                 logging.debug("Watcher %s does not have permissions" % watcher)
 452                                 continue
 453
 454                         logging.debug("Sending watcher email to %s" % watcher)
 455
 456                         # Compose message
 457                         self.backend.messages.send_template("wiki/messages/page-changed",
 458                                 account=watcher, page=self, priority=-10)
 459
 460         def restore(self, author, address, comment=None):
 461                 changes = "Restore to revision from %s" % self.timestamp.isoformat()
 462
 463                 # Append comment
 464                 if comment:
 465                         changes = "%s: %s" % (changes, comment)
 466
 467                 return self.backend.wiki.create_page(self.page,
 468                         author, self.markdown, changes=changes, address=address)
 469
 470
 471 class File(misc.Object):
 472         def init(self, id, data):
 473                 self.id   = id
 474                 self.data = data
 475
 476         def __eq__(self, other):
 477                 if isinstance(other, self.__class__):
 478                         return self.id == other.id
 479
 480         @property
 481         def url(self):
 482                 return os.path.join(self.path, self.filename)
 483
 484         @property
 485         def path(self):
 486                 return self.data.path
 487
 488         @property
 489         def filename(self):
 490                 return self.data.filename
 491
 492         @property
 493         def mimetype(self):
 494                 return self.data.mimetype
 495
 496         @property
 497         def size(self):
 498                 return self.data.size
 499
 500         @lazy_property
 501         def author(self):
 502                 if self.data.author_uid:
 503                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 504
 505         @property
 506         def created_at(self):
 507                 return self.data.created_at
 508
 509         def delete(self, author=None):
 510                 self.db.execute("UPDATE wiki_files SET deleted_at = NOW(), deleted_by = %s \
 511                         WHERE id = %s", author.uid if author else None, self.id)
 512
 513         @property
 514         def deleted_at(self):
 515                 return self.data.deleted_at
 516
 517         def get_latest_revision(self):
 518                 revisions = self.get_revisions()
 519
 520                 # Return first object
 521                 for rev in revisions:
 522                         return rev
 523
 524         def get_revisions(self):
 525                 revisions = self.backend.wiki._get_files("SELECT * FROM wiki_files \
 526                         WHERE path = %s AND filename = %s ORDER BY created_at DESC", self.path, self.filename)
 527
 528                 return list(revisions)
 529
 530         def is_pdf(self):
 531                 return self.mimetype in ("application/pdf", "application/x-pdf")
 532
 533         def is_image(self):
 534                 return self.mimetype.startswith("image/")
 535
 536         def is_vector_image(self):
 537                 return self.mimetype in ("image/svg+xml",)
 538
 539         def is_bitmap_image(self):
 540                 return self.is_image() and not self.is_vector_image()
 541
 542         @lazy_property
 543         def blob(self):
 544                 res = self.db.get("SELECT data FROM wiki_blobs \
 545                         WHERE id = %s", self.data.blob_id)
 546
 547                 if res:
 548                         return bytes(res.data)
 549
 550         def get_thumbnail(self, size):
 551                 assert self.is_bitmap_image()
 552
 553                 cache_key = "-".join((self.path, util.normalize(self.filename), self.created_at.isoformat(), "%spx" % size))
 554
 555                 # Try to fetch the data from the cache
 556                 thumbnail = self.memcache.get(cache_key)
 557                 if thumbnail:
 558                         return thumbnail
 559
 560                 # Generate the thumbnail
 561                 thumbnail = util.generate_thumbnail(self.blob, size)
 562
 563                 # Put it into the cache for forever
 564                 self.memcache.set(cache_key, thumbnail)
 565
 566                 return thumbnail
 567
 568
 569 class WikiRenderer(misc.Object):
 570         schemas = (
 571                 "ftp://",
 572                 "git://",
 573                 "http://",
 574                 "https://",
 575                 "rsync://",
 576                 "sftp://",
 577                 "ssh://",
 578                 "webcal://",
 579         )
 580
 581         # Links
 582         links = re.compile(r"<a href=\"(.*?)\">(.*?)</a>")
 583
 584         # Images
 585         images = re.compile(r"<img alt(?:=\"(.*?)\")? src=\"(.*?)\" (?:title=\"(.*?)\" )?/>")
 586
 587         def init(self, path):
 588                 self.path = path
 589
 590         def _render_link(self, m):
 591                 url, text = m.groups()
 592
 593                 # Emails
 594                 if "@" in url:
 595                         # Strip mailto:
 596                         if url.startswith("mailto:"):
 597                                 url = url[7:]
 598
 599                         return """<a class="link-external" href="mailto:%s">%s</a>""" % \
 600                                 (url, text or url)
 601
 602                 # External Links
 603                 for schema in self.schemas:
 604                         if url.startswith(schema):
 605                                 return """<a class="link-external" href="%s">%s</a>""" % \
 606                                         (url, text or url)
 607
 608                 # Everything else must be an internal link
 609                 path = self.backend.wiki.make_path(self.path, url)
 610
 611                 return """<a href="/docs%s">%s</a>""" % \
 612                         (path, text or self.backend.wiki.get_page_title(path))
 613
 614         def _render_image(self, m):
 615                 alt_text, url, caption = m.groups()
 616
 617                 # Skip any absolute and external URLs
 618                 if url.startswith("/") or url.startswith("https://") or url.startswith("http://"):
 619                         return """<figure class="figure"><img src="%s" class="figure-img img-fluid rounded" alt="%s">
 620                                 <figcaption class="figure-caption">%s</figcaption></figure>
 621                         """ % (url, alt_text, caption or "")
 622
 623                 # Try to split query string
 624                 url, delimiter, qs = url.partition("?")
 625
 626                 # Parse query arguments
 627                 args = urllib.parse.parse_qs(qs)
 628
 629                 # Build absolute path
 630                 url = self.backend.wiki.make_path(self.path, url)
 631
 632                 # Find image
 633                 file = self.backend.wiki.get_file_by_path(url)
 634                 if not file or not file.is_image():
 635                         return "<!-- Could not find image %s in %s -->" % (url, self.path)
 636
 637                 # Scale down the image if not already done
 638                 if not "s" in args:
 639                         args["s"] = "920"
 640
 641                 return """<figure class="figure"><img src="%s?%s" class="figure-img img-fluid rounded" alt="%s">
 642                 <figcaption class="figure-caption">%s</figcaption></figure>
 643                 """ % (url, urllib.parse.urlencode(args), caption, caption or "")
 644
 645         def render(self, text):
 646                 logging.debug("Rendering %s" % self.path)
 647
 648                 # Borrow this from the blog
 649                 text = self.backend.blog._render_text(text, lang="markdown")
 650
 651                 # Postprocess links
 652                 text = self.links.sub(self._render_link, text)
 653
 654                 # Postprocess images to <figure>
 655                 text = self.images.sub(self._render_image, text)
 656
 657                 return text