src/backend/wiki.py

   1 #!/usr/bin/python3
   2
   3 import difflib
   4 import logging
   5 import os.path
   6 import re
   7 import tornado.gen
   8 import urllib.parse
   9
  10 from . import misc
  11 from . import util
  12 from .decorators import *
  13
  14 INTERWIKIS = {
  15         "google"    : ("https://www.google.com/search?q=%(url)s", None, "fab fa-google"),
  16         "rfc"       : ("https://tools.ietf.org/html/rfc%(name)s", "RFC %s", None),
  17         "wp"        : ("https://en.wikipedia.org/wiki/%(name)s", None, "fab fa-wikipedia-w"),
  18 }
  19
  20 class Wiki(misc.Object):
  21         def _get_pages(self, query, *args):
  22                 res = self.db.query(query, *args)
  23
  24                 for row in res:
  25                         yield Page(self.backend, row.id, data=row)
  26
  27         def _get_page(self, query, *args):
  28                 res = self.db.get(query, *args)
  29
  30                 if res:
  31                         return Page(self.backend, res.id, data=res)
  32
  33         def get_page_title(self, page, default=None):
  34                 # Try to retrieve title from cache
  35                 title = self.memcache.get("wiki:title:%s" % page)
  36                 if title:
  37                         return title
  38
  39                 # If the title has not been in the cache, we will
  40                 # have to look it up
  41                 doc = self.get_page(page)
  42                 if doc:
  43                         title = doc.title
  44                 else:
  45                         title = os.path.basename(page)
  46
  47                 # Save in cache for forever
  48                 self.memcache.set("wiki:title:%s" % page, title)
  49
  50                 return title
  51
  52         def get_page(self, page, revision=None):
  53                 page = Page.sanitise_page_name(page)
  54                 assert page
  55
  56                 if revision:
  57                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
  58                                 AND timestamp = %s", page, revision)
  59                 else:
  60                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
  61                                 ORDER BY timestamp DESC LIMIT 1", page)
  62
  63         def get_recent_changes(self, account, limit=None):
  64                 pages = self._get_pages("SELECT * FROM wiki \
  65                         WHERE timestamp >= NOW() - INTERVAL '4 weeks' \
  66                         ORDER BY timestamp DESC")
  67
  68                 for page in pages:
  69                         if not page.check_acl(account):
  70                                 continue
  71
  72                         yield page
  73
  74                         limit -= 1
  75                         if not limit:
  76                                 break
  77
  78         def create_page(self, page, author, content, changes=None, address=None):
  79                 page = Page.sanitise_page_name(page)
  80
  81                 # Write page to the database
  82                 page = self._get_page("INSERT INTO wiki(page, author_uid, markdown, changes, address) \
  83                         VALUES(%s, %s, %s, %s, %s) RETURNING *", page, author.uid, content or None, changes, address)
  84
  85                 # Update cache
  86                 self.memcache.set("wiki:title:%s" % page.page, page.title)
  87
  88                 # Send email to all watchers
  89                 page._send_watcher_emails(excludes=[author])
  90
  91                 return page
  92
  93         def delete_page(self, page, author, **kwargs):
  94                 # Do nothing if the page does not exist
  95                 if not self.get_page(page):
  96                         return
  97
  98                 # Just creates a blank last version of the page
  99                 self.create_page(page, author=author, content=None, **kwargs)
 100
 101         def make_breadcrumbs(self, url):
 102                 # Split and strip all empty elements (double slashes)
 103                 parts = list(e for e in url.split("/") if e)
 104
 105                 ret = []
 106                 for part in ("/".join(parts[:i]) for i in range(1, len(parts))):
 107                         ret.append(("/%s" % part, self.get_page_title(part, os.path.basename(part))))
 108
 109                 return ret
 110
 111         def search(self, query, account=None, limit=None):
 112                 query = util.parse_search_query(query)
 113
 114                 res = self._get_pages("SELECT wiki.* FROM wiki_search_index search_index \
 115                         LEFT JOIN wiki ON search_index.wiki_id = wiki.id \
 116                         WHERE search_index.document @@ to_tsquery('english', %s) \
 117                                 ORDER BY ts_rank(search_index.document, to_tsquery('english', %s)) DESC",
 118                         query, query)
 119
 120                 pages = []
 121                 for page in res:
 122                         # Skip any pages the user doesn't have permission for
 123                         if not page.check_acl(account):
 124                                 continue
 125
 126                         # Return any other pages
 127                         pages.append(page)
 128
 129                         # Break when we have found enough pages
 130                         if limit and len(pages) >= limit:
 131                                 break
 132
 133                 return pages
 134
 135         def refresh(self):
 136                 """
 137                         Needs to be called after a page has been changed
 138                 """
 139                 self.db.execute("REFRESH MATERIALIZED VIEW wiki_search_index")
 140
 141         # ACL
 142
 143         def check_acl(self, page, account):
 144                 res = self.db.query("SELECT * FROM wiki_acls \
 145                         WHERE %s ILIKE (path || '%%') ORDER BY LENGTH(path) DESC LIMIT 1", page)
 146
 147                 for row in res:
 148                         # Access not permitted when user is not logged in
 149                         if not account:
 150                                 return False
 151
 152                         # If user is in a matching group, we grant permission
 153                         for group in row.groups:
 154                                 if group in account.groups:
 155                                         return True
 156
 157                         # Otherwise access is not permitted
 158                         return False
 159
 160                 # If no ACLs are found, we permit access
 161                 return True
 162
 163         # Files
 164
 165         def _get_files(self, query, *args):
 166                 res = self.db.query(query, *args)
 167
 168                 for row in res:
 169                         yield File(self.backend, row.id, data=row)
 170
 171         def _get_file(self, query, *args):
 172                 res = self.db.get(query, *args)
 173
 174                 if res:
 175                         return File(self.backend, res.id, data=res)
 176
 177         def get_files(self, path):
 178                 files = self._get_files("SELECT * FROM wiki_files \
 179                         WHERE path = %s AND deleted_at IS NULL ORDER BY filename", path)
 180
 181                 return list(files)
 182
 183         def get_file_by_path(self, path):
 184                 path, filename = os.path.dirname(path), os.path.basename(path)
 185
 186                 return self._get_file("SELECT * FROM wiki_files \
 187                         WHERE path = %s AND filename = %s AND deleted_at IS NULL", path, filename)
 188
 189         def upload(self, path, filename, data, mimetype, author, address):
 190                 # Upload the blob first
 191                 blob = self.db.get("INSERT INTO wiki_blobs(data) VALUES(%s) RETURNING id", data)
 192
 193                 # Create entry for file
 194                 return self._get_file("INSERT INTO wiki_files(path, filename, author_uid, address, \
 195                         mimetype, blob_id, size) VALUES(%s,  %s, %s, %s, %s, %s, %s) RETURNING *", path,
 196                         filename, author.uid, address, mimetype, blob.id, len(data))
 197
 198         def find_image(self, path, filename):
 199                 for p in (path, os.path.dirname(path)):
 200                         file = self.get_file_by_path(os.path.join(p, filename))
 201
 202                         if file and file.is_image():
 203                                 return file
 204
 205
 206 class Page(misc.Object):
 207         # External links
 208         external_link = re.compile(r"\[\[((?:ftp|git|https?|rsync|sftp|ssh|webcal)\:\/\/.+?)(?:\|(.+?))\]\]")
 209
 210         # Interwiki links e.g. [[wp>IPFire]]
 211         interwiki_link = re.compile(r"\[\[(\w+)>(.+?)(?:\|(.+?))?\]\]")
 212
 213         # Mail link
 214         email_link = re.compile(r"\[\[([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)(?:\|(.+?))?\]\]")
 215
 216         def init(self, id, data=None):
 217                 self.id = id
 218                 self.data = data
 219
 220         def __repr__(self):
 221                 return "<%s %s %s>" % (self.__class__.__name__, self.page, self.timestamp)
 222
 223         def __eq__(self, other):
 224                 if isinstance(other, self.__class__):
 225                         return self.id == other.id
 226
 227         def __lt__(self, other):
 228                 if isinstance(other, self.__class__):
 229                         if self.page == other.page:
 230                                 return self.timestamp < other.timestamp
 231
 232                         return self.page < other.page
 233
 234         @staticmethod
 235         def sanitise_page_name(page):
 236                 if not page:
 237                         return "/"
 238
 239                 # Make sure that the page name does NOT end with a /
 240                 if page.endswith("/"):
 241                         page = page[:-1]
 242
 243                 # Make sure the page name starts with a /
 244                 if not page.startswith("/"):
 245                         page = "/%s" % page
 246
 247                 # Remove any double slashes
 248                 page = page.replace("//", "/")
 249
 250                 return page
 251
 252         @property
 253         def url(self):
 254                 return self.page
 255
 256         @property
 257         def full_url(self):
 258                 return "https://wiki.ipfire.org%s" % self.url
 259
 260         @property
 261         def page(self):
 262                 return self.data.page
 263
 264         @property
 265         def title(self):
 266                 return self._title or os.path.basename(self.page[1:])
 267
 268         @property
 269         def _title(self):
 270                 if not self.markdown:
 271                         return
 272
 273                 # Find first H1 headline in markdown
 274                 markdown = self.markdown.splitlines()
 275
 276                 m = re.match(r"^# (.*)( #)?$", markdown[0])
 277                 if m:
 278                         return m.group(1)
 279
 280         @lazy_property
 281         def author(self):
 282                 if self.data.author_uid:
 283                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 284
 285         def _render_external_link(self, m):
 286                 url, alias = m.groups()
 287
 288                 return """<a class="link-external" href="%s">%s</a>""" % (url, alias or url)
 289
 290         def _render_interwiki_link(self, m):
 291                 wiki = m.group(1)
 292                 if not wiki:
 293                         return
 294
 295                 # Retrieve URL
 296                 try:
 297                         url, repl, icon = INTERWIKIS[wiki]
 298                 except KeyError:
 299                         logging.warning("Invalid interwiki: %s" % wiki)
 300                         return
 301
 302                 # Name of the page
 303                 name = m.group(2)
 304
 305                 # Expand URL
 306                 url = url % {
 307                         "name" : name,
 308                         "url"  : urllib.parse.quote(name),
 309                 }
 310
 311                 # Get alias (if present)
 312                 alias = m.group(3)
 313
 314                 if not alias and repl:
 315                         alias = repl % name
 316
 317                 # Put everything together
 318                 s = []
 319
 320                 if icon:
 321                         s.append("<span class=\"%s\"></span>" % icon)
 322
 323                 s.append("""<a class="link-external" href="%s">%s</a>""" % (url, alias or name))
 324
 325                 return " ".join(s)
 326
 327         def _render_email_link(self, m):
 328                 address, alias = m.groups()
 329
 330                 return """<a class="link-external" href="mailto:%s">%s</a>""" \
 331                         % (address, alias or address)
 332
 333         def _render(self, text):
 334                 logging.debug("Rendering %s" % self)
 335
 336                 # Link images
 337                 replacements = []
 338                 for match in re.finditer(r"!\[(.*?)\]\((.*?)\)", text):
 339                         alt_text, url = match.groups()
 340
 341                         # Skip any absolute and external URLs
 342                         if url.startswith("/") or url.startswith("https://") or url.startswith("http://"):
 343                                 continue
 344
 345                         # Try to split query string
 346                         url, delimiter, qs = url.partition("?")
 347
 348                         # Parse query arguments
 349                         args = urllib.parse.parse_qs(qs)
 350
 351                         # Find image
 352                         file = self.backend.wiki.find_image(self.page, url)
 353                         if not file:
 354                                 continue
 355
 356                         # Scale down the image if not already done
 357                         if not "s" in args:
 358                                 args["s"] = "768"
 359
 360                         # Format URL
 361                         url = "%s?%s" % (file.url, urllib.parse.urlencode(args))
 362
 363                         replacements.append((match.span(), file, alt_text, url))
 364
 365                 # Apply all replacements
 366                 for (start, end), file, alt_text, url in reversed(replacements):
 367                         text = text[:start] + "[![%s](%s)](%s?action=detail)" % (alt_text, url, file.url) + text[end:]
 368
 369                 # Handle interwiki links
 370                 text = self.interwiki_link.sub(self._render_interwiki_link, text)
 371
 372                 # Handle external links
 373                 text = self.external_link.sub(self._render_external_link, text)
 374
 375                 # Handle email links
 376                 text = self.email_link.sub(self._render_email_link, text)
 377
 378                 # Add wiki links
 379                 patterns = (
 380                         (r"\[\[([\w\d\/\-\.]+)(?:\|(.+?))\]\]", r"\1", r"\2", None, True),
 381                         (r"\[\[([\w\d\/\-\.]+)\]\]", r"\1", r"\1", self.backend.wiki.get_page_title, True),
 382                 )
 383
 384                 for pattern, link, title, repl, internal in patterns:
 385                         replacements = []
 386
 387                         for match in re.finditer(pattern, text):
 388                                 l = match.expand(link)
 389                                 t = match.expand(title)
 390
 391                                 if internal:
 392                                         # Allow relative links
 393                                         if not l.startswith("/"):
 394                                                 l = os.path.join(self.page, l)
 395
 396                                         # Normalise links
 397                                         l = os.path.normpath(l)
 398
 399                                 if callable(repl):
 400                                         t = repl(l) or t
 401
 402                                 replacements.append((match.span(), t or l, l))
 403
 404                         # Apply all replacements
 405                         for (start, end), t, l in reversed(replacements):
 406                                 text = text[:start] + "[%s](%s)" % (t, l) + text[end:]
 407
 408                 # Borrow this from the blog
 409                 return self.backend.blog._render_text(text, lang="markdown")
 410
 411         @property
 412         def markdown(self):
 413                 return self.data.markdown or ""
 414
 415         @property
 416         def html(self):
 417                 return self._render(self.markdown)
 418
 419         @property
 420         def timestamp(self):
 421                 return self.data.timestamp
 422
 423         def was_deleted(self):
 424                 return self.markdown is None
 425
 426         @lazy_property
 427         def breadcrumbs(self):
 428                 return self.backend.wiki.make_breadcrumbs(self.page)
 429
 430         def get_latest_revision(self):
 431                 revisions = self.get_revisions()
 432
 433                 # Return first object
 434                 for rev in revisions:
 435                         return rev
 436
 437         def get_revisions(self):
 438                 return self.backend.wiki._get_pages("SELECT * FROM wiki \
 439                         WHERE page = %s ORDER BY timestamp DESC", self.page)
 440
 441         @lazy_property
 442         def previous_revision(self):
 443                 return self.backend.wiki._get_page("SELECT * FROM wiki \
 444                         WHERE page = %s AND timestamp < %s ORDER BY timestamp DESC \
 445                         LIMIT 1", self.page, self.timestamp)
 446
 447         @property
 448         def changes(self):
 449                 return self.data.changes
 450
 451         # ACL
 452
 453         def check_acl(self, account):
 454                 return self.backend.wiki.check_acl(self.page, account)
 455
 456         # Sidebar
 457
 458         @lazy_property
 459         def sidebar(self):
 460                 parts = self.page.split("/")
 461
 462                 while parts:
 463                         sidebar = self.backend.wiki.get_page("%s/sidebar" % os.path.join(*parts))
 464                         if sidebar:
 465                                 return sidebar
 466
 467                         parts.pop()
 468
 469         # Watchers
 470
 471         @lazy_property
 472         def diff(self):
 473                 if self.previous_revision:
 474                         diff = difflib.unified_diff(
 475                                 self.previous_revision.markdown.splitlines(),
 476                                 self.markdown.splitlines(),
 477                         )
 478
 479                         return "\n".join(diff)
 480
 481         @property
 482         def watchers(self):
 483                 res = self.db.query("SELECT uid FROM wiki_watchlist \
 484                         WHERE page = %s", self.page)
 485
 486                 for row in res:
 487                         # Search for account by UID and skip if none was found
 488                         account = self.backend.accounts.get_by_uid(row.uid)
 489                         if not account:
 490                                 continue
 491
 492                         # Return the account
 493                         yield account
 494
 495         def is_watched_by(self, account):
 496                 res = self.db.get("SELECT 1 FROM wiki_watchlist \
 497                         WHERE page = %s AND uid = %s", self.page, account.uid)
 498
 499                 if res:
 500                         return True
 501
 502                 return False
 503
 504         def add_watcher(self, account):
 505                 if self.is_watched_by(account):
 506                         return
 507
 508                 self.db.execute("INSERT INTO wiki_watchlist(page, uid) \
 509                         VALUES(%s, %s)", self.page, account.uid)
 510
 511         def remove_watcher(self, account):
 512                 self.db.execute("DELETE FROM wiki_watchlist \
 513                         WHERE page = %s AND uid = %s", self.page, account.uid)
 514
 515         def _send_watcher_emails(self, excludes=[]):
 516                 # Nothing to do if there was no previous revision
 517                 if not self.previous_revision:
 518                         return
 519
 520                 for watcher in self.watchers:
 521                         # Skip everyone who is excluded
 522                         if watcher in excludes:
 523                                 logging.debug("Excluding %s" % watcher)
 524                                 continue
 525
 526                         logging.debug("Sending watcher email to %s" % watcher)
 527
 528                         # Compose message
 529                         self.backend.messages.send_template("wiki/messages/page-changed",
 530                                 recipients=[watcher], page=self, priority=-10)
 531
 532
 533 class File(misc.Object):
 534         def init(self, id, data):
 535                 self.id   = id
 536                 self.data = data
 537
 538         @property
 539         def url(self):
 540                 return os.path.join(self.path, self.filename)
 541
 542         @property
 543         def path(self):
 544                 return self.data.path
 545
 546         @property
 547         def filename(self):
 548                 return self.data.filename
 549
 550         @property
 551         def mimetype(self):
 552                 return self.data.mimetype
 553
 554         @property
 555         def size(self):
 556                 return self.data.size
 557
 558         @lazy_property
 559         def author(self):
 560                 if self.data.author_uid:
 561                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 562
 563         @property
 564         def created_at(self):
 565                 return self.data.created_at
 566
 567         def is_pdf(self):
 568                 return self.mimetype in ("application/pdf", "application/x-pdf")
 569
 570         def is_image(self):
 571                 return self.mimetype.startswith("image/")
 572
 573         @lazy_property
 574         def blob(self):
 575                 res = self.db.get("SELECT data FROM wiki_blobs \
 576                         WHERE id = %s", self.data.blob_id)
 577
 578                 if res:
 579                         return bytes(res.data)
 580
 581         def get_thumbnail(self, size):
 582                 cache_key = "-".join((self.path, util.normalize(self.filename), self.created_at.isoformat(), "%spx" % size))
 583
 584                 # Try to fetch the data from the cache
 585                 thumbnail = self.memcache.get(cache_key)
 586                 if thumbnail:
 587                         return thumbnail
 588
 589                 # Generate the thumbnail
 590                 thumbnail = util.generate_thumbnail(self.blob, size)
 591
 592                 # Put it into the cache for forever
 593                 self.memcache.set(cache_key, thumbnail)
 594
 595                 return thumbnail