src/backend/wiki.py

   1 #!/usr/bin/python3
   2
   3 import difflib
   4 import logging
   5 import os.path
   6 import re
   7 import tornado.gen
   8 import urllib.parse
   9
  10 from . import misc
  11 from . import util
  12 from .decorators import *
  13
  14 class Wiki(misc.Object):
  15         def _get_pages(self, query, *args):
  16                 res = self.db.query(query, *args)
  17
  18                 for row in res:
  19                         yield Page(self.backend, row.id, data=row)
  20
  21         def _get_page(self, query, *args):
  22                 res = self.db.get(query, *args)
  23
  24                 if res:
  25                         return Page(self.backend, res.id, data=res)
  26
  27         def get_page_title(self, page, default=None):
  28                 # Try to retrieve title from cache
  29                 title = self.memcache.get("wiki:title:%s" % page)
  30                 if title:
  31                         return title
  32
  33                 # If the title has not been in the cache, we will
  34                 # have to look it up
  35                 doc = self.get_page(page)
  36                 if doc:
  37                         title = doc.title
  38                 else:
  39                         title = os.path.basename(page)
  40
  41                 # Save in cache for forever
  42                 self.memcache.set("wiki:title:%s" % page, title)
  43
  44                 return title
  45
  46         def get_page(self, page, revision=None):
  47                 page = Page.sanitise_page_name(page)
  48                 assert page
  49
  50                 if revision:
  51                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
  52                                 AND timestamp = %s", page, revision)
  53                 else:
  54                         return self._get_page("SELECT * FROM wiki WHERE page = %s \
  55                                 ORDER BY timestamp DESC LIMIT 1", page)
  56
  57         def get_recent_changes(self, account, limit=None):
  58                 pages = self._get_pages("SELECT * FROM wiki \
  59                         WHERE timestamp >= NOW() - INTERVAL '4 weeks' \
  60                         ORDER BY timestamp DESC")
  61
  62                 for page in pages:
  63                         if not page.check_acl(account):
  64                                 continue
  65
  66                         yield page
  67
  68                         limit -= 1
  69                         if not limit:
  70                                 break
  71
  72         def create_page(self, page, author, content, changes=None, address=None):
  73                 page = Page.sanitise_page_name(page)
  74
  75                 # Write page to the database
  76                 page = self._get_page("INSERT INTO wiki(page, author_uid, markdown, changes, address) \
  77                         VALUES(%s, %s, %s, %s, %s) RETURNING *", page, author.uid, content or None, changes, address)
  78
  79                 # Update cache
  80                 self.memcache.set("wiki:title:%s" % page.page, page.title)
  81
  82                 # Send email to all watchers
  83                 page._send_watcher_emails(excludes=[author])
  84
  85                 return page
  86
  87         def delete_page(self, page, author, **kwargs):
  88                 # Do nothing if the page does not exist
  89                 if not self.get_page(page):
  90                         return
  91
  92                 # Just creates a blank last version of the page
  93                 self.create_page(page, author=author, content=None, **kwargs)
  94
  95         def make_breadcrumbs(self, url):
  96                 # Split and strip all empty elements (double slashes)
  97                 parts = list(e for e in url.split("/") if e)
  98
  99                 ret = []
 100                 for part in ("/".join(parts[:i]) for i in range(1, len(parts))):
 101                         ret.append(("/%s" % part, self.get_page_title(part, os.path.basename(part))))
 102
 103                 return ret
 104
 105         def search(self, query, account=None, limit=None):
 106                 query = util.parse_search_query(query)
 107
 108                 res = self._get_pages("SELECT wiki.* FROM wiki_search_index search_index \
 109                         LEFT JOIN wiki ON search_index.wiki_id = wiki.id \
 110                         WHERE search_index.document @@ to_tsquery('english', %s) \
 111                                 ORDER BY ts_rank(search_index.document, to_tsquery('english', %s)) DESC",
 112                         query, query)
 113
 114                 pages = []
 115                 for page in res:
 116                         # Skip any pages the user doesn't have permission for
 117                         if not page.check_acl(account):
 118                                 continue
 119
 120                         # Return any other pages
 121                         pages.append(page)
 122
 123                         # Break when we have found enough pages
 124                         if limit and len(pages) >= limit:
 125                                 break
 126
 127                 return pages
 128
 129         def refresh(self):
 130                 """
 131                         Needs to be called after a page has been changed
 132                 """
 133                 self.db.execute("REFRESH MATERIALIZED VIEW wiki_search_index")
 134
 135         # ACL
 136
 137         def check_acl(self, page, account):
 138                 res = self.db.query("SELECT * FROM wiki_acls \
 139                         WHERE %s ILIKE (path || '%%') ORDER BY LENGTH(path) DESC LIMIT 1", page)
 140
 141                 for row in res:
 142                         # Access not permitted when user is not logged in
 143                         if not account:
 144                                 return False
 145
 146                         # If user is in a matching group, we grant permission
 147                         for group in row.groups:
 148                                 if group in account.groups:
 149                                         return True
 150
 151                         # Otherwise access is not permitted
 152                         return False
 153
 154                 # If no ACLs are found, we permit access
 155                 return True
 156
 157         # Files
 158
 159         def _get_files(self, query, *args):
 160                 res = self.db.query(query, *args)
 161
 162                 for row in res:
 163                         yield File(self.backend, row.id, data=row)
 164
 165         def _get_file(self, query, *args):
 166                 res = self.db.get(query, *args)
 167
 168                 if res:
 169                         return File(self.backend, res.id, data=res)
 170
 171         def get_files(self, path):
 172                 files = self._get_files("SELECT * FROM wiki_files \
 173                         WHERE path = %s AND deleted_at IS NULL ORDER BY filename", path)
 174
 175                 return list(files)
 176
 177         def get_file_by_path(self, path):
 178                 path, filename = os.path.dirname(path), os.path.basename(path)
 179
 180                 return self._get_file("SELECT * FROM wiki_files \
 181                         WHERE path = %s AND filename = %s AND deleted_at IS NULL", path, filename)
 182
 183         def upload(self, path, filename, data, mimetype, author, address):
 184                 # Upload the blob first
 185                 blob = self.db.get("INSERT INTO wiki_blobs(data) VALUES(%s) RETURNING id", data)
 186
 187                 # Create entry for file
 188                 return self._get_file("INSERT INTO wiki_files(path, filename, author_uid, address, \
 189                         mimetype, blob_id, size) VALUES(%s,  %s, %s, %s, %s, %s, %s) RETURNING *", path,
 190                         filename, author.uid, address, mimetype, blob.id, len(data))
 191
 192         def find_image(self, path, filename):
 193                 for p in (path, os.path.dirname(path)):
 194                         file = self.get_file_by_path(os.path.join(p, filename))
 195
 196                         if file and file.is_image():
 197                                 return file
 198
 199
 200 class Page(misc.Object):
 201         def init(self, id, data=None):
 202                 self.id = id
 203                 self.data = data
 204
 205         def __repr__(self):
 206                 return "<%s %s %s>" % (self.__class__.__name__, self.page, self.timestamp)
 207
 208         def __eq__(self, other):
 209                 if isinstance(other, self.__class__):
 210                         return self.id == other.id
 211
 212         def __lt__(self, other):
 213                 if isinstance(other, self.__class__):
 214                         if self.page == other.page:
 215                                 return self.timestamp < other.timestamp
 216
 217                         return self.page < other.page
 218
 219         @staticmethod
 220         def sanitise_page_name(page):
 221                 if not page:
 222                         return "/"
 223
 224                 # Make sure that the page name does NOT end with a /
 225                 if page.endswith("/"):
 226                         page = page[:-1]
 227
 228                 # Make sure the page name starts with a /
 229                 if not page.startswith("/"):
 230                         page = "/%s" % page
 231
 232                 # Remove any double slashes
 233                 page = page.replace("//", "/")
 234
 235                 return page
 236
 237         @property
 238         def url(self):
 239                 return self.page
 240
 241         @property
 242         def full_url(self):
 243                 return "https://wiki.ipfire.org%s" % self.url
 244
 245         @property
 246         def page(self):
 247                 return self.data.page
 248
 249         @property
 250         def title(self):
 251                 return self._title or os.path.basename(self.page[1:])
 252
 253         @property
 254         def _title(self):
 255                 if not self.markdown:
 256                         return
 257
 258                 # Find first H1 headline in markdown
 259                 markdown = self.markdown.splitlines()
 260
 261                 m = re.match(r"^# (.*)( #)?$", markdown[0])
 262                 if m:
 263                         return m.group(1)
 264
 265         @lazy_property
 266         def author(self):
 267                 if self.data.author_uid:
 268                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 269
 270         def _render(self, text):
 271                 logging.debug("Rendering %s" % self)
 272
 273                 # Link images
 274                 replacements = []
 275                 for match in re.finditer(r"!\[(.*?)\]\((.*?)\)", text):
 276                         alt_text, url = match.groups()
 277
 278                         # Skip any absolute and external URLs
 279                         if url.startswith("/") or url.startswith("https://") or url.startswith("http://"):
 280                                 continue
 281
 282                         # Try to split query string
 283                         url, delimiter, qs = url.partition("?")
 284
 285                         # Parse query arguments
 286                         args = urllib.parse.parse_qs(qs)
 287
 288                         # Find image
 289                         file = self.backend.wiki.find_image(self.page, url)
 290                         if not file:
 291                                 continue
 292
 293                         # Scale down the image if not already done
 294                         if not "s" in args:
 295                                 args["s"] = "768"
 296
 297                         # Format URL
 298                         url = "%s?%s" % (file.url, urllib.parse.urlencode(args))
 299
 300                         replacements.append((match.span(), file, alt_text, url))
 301
 302                 # Apply all replacements
 303                 for (start, end), file, alt_text, url in reversed(replacements):
 304                         text = text[:start] + "[![%s](%s)](%s?action=detail)" % (alt_text, url, file.url) + text[end:]
 305
 306                 # Add wiki links
 307                 patterns = (
 308                         (r"\[\[([\w\d\/\-\.]+)(?:\|(.+?))\]\]", r"\1", r"\2", None, True),
 309                         (r"\[\[([\w\d\/\-\.]+)\]\]", r"\1", r"\1", self.backend.wiki.get_page_title, True),
 310
 311                         # External links
 312                         (r"\[\[((?:ftp|git|https?|rsync|sftp|ssh|webcal)\:\/\/.+?)(?:\|(.+?))\]\]",
 313                                 r"\1", r"\2", None, False),
 314
 315                         # Mail
 316                         (r"\[\[([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)\]\]",
 317                                 r"\1", r"\1", None, False),
 318                 )
 319
 320                 for pattern, link, title, repl, internal in patterns:
 321                         replacements = []
 322
 323                         for match in re.finditer(pattern, text):
 324                                 l = match.expand(link)
 325                                 t = match.expand(title)
 326
 327                                 if internal:
 328                                         # Allow relative links
 329                                         if not l.startswith("/"):
 330                                                 l = os.path.join(self.page, l)
 331
 332                                         # Normalise links
 333                                         l = os.path.normpath(l)
 334
 335                                 if callable(repl):
 336                                         t = repl(l) or t
 337
 338                                 replacements.append((match.span(), t or l, l))
 339
 340                         # Apply all replacements
 341                         for (start, end), t, l in reversed(replacements):
 342                                 text = text[:start] + "[%s](%s)" % (t, l) + text[end:]
 343
 344                 # Borrow this from the blog
 345                 return self.backend.blog._render_text(text, lang="markdown")
 346
 347         @property
 348         def markdown(self):
 349                 return self.data.markdown or ""
 350
 351         @property
 352         def html(self):
 353                 return self._render(self.markdown)
 354
 355         @property
 356         def timestamp(self):
 357                 return self.data.timestamp
 358
 359         def was_deleted(self):
 360                 return self.markdown is None
 361
 362         @lazy_property
 363         def breadcrumbs(self):
 364                 return self.backend.wiki.make_breadcrumbs(self.page)
 365
 366         def get_latest_revision(self):
 367                 revisions = self.get_revisions()
 368
 369                 # Return first object
 370                 for rev in revisions:
 371                         return rev
 372
 373         def get_revisions(self):
 374                 return self.backend.wiki._get_pages("SELECT * FROM wiki \
 375                         WHERE page = %s ORDER BY timestamp DESC", self.page)
 376
 377         @lazy_property
 378         def previous_revision(self):
 379                 return self.backend.wiki._get_page("SELECT * FROM wiki \
 380                         WHERE page = %s AND timestamp < %s ORDER BY timestamp DESC \
 381                         LIMIT 1", self.page, self.timestamp)
 382
 383         @property
 384         def changes(self):
 385                 return self.data.changes
 386
 387         # ACL
 388
 389         def check_acl(self, account):
 390                 return self.backend.wiki.check_acl(self.page, account)
 391
 392         # Sidebar
 393
 394         @lazy_property
 395         def sidebar(self):
 396                 parts = self.page.split("/")
 397
 398                 while parts:
 399                         sidebar = self.backend.wiki.get_page("%s/sidebar" % os.path.join(*parts))
 400                         if sidebar:
 401                                 return sidebar
 402
 403                         parts.pop()
 404
 405         # Watchers
 406
 407         @lazy_property
 408         def diff(self):
 409                 if self.previous_revision:
 410                         diff = difflib.unified_diff(
 411                                 self.previous_revision.markdown.splitlines(),
 412                                 self.markdown.splitlines(),
 413                         )
 414
 415                         return "\n".join(diff)
 416
 417         @property
 418         def watchers(self):
 419                 res = self.db.query("SELECT uid FROM wiki_watchlist \
 420                         WHERE page = %s", self.page)
 421
 422                 for row in res:
 423                         # Search for account by UID and skip if none was found
 424                         account = self.backend.accounts.get_by_uid(row.uid)
 425                         if not account:
 426                                 continue
 427
 428                         # Return the account
 429                         yield account
 430
 431         def is_watched_by(self, account):
 432                 res = self.db.get("SELECT 1 FROM wiki_watchlist \
 433                         WHERE page = %s AND uid = %s", self.page, account.uid)
 434
 435                 if res:
 436                         return True
 437
 438                 return False
 439
 440         def add_watcher(self, account):
 441                 if self.is_watched_by(account):
 442                         return
 443
 444                 self.db.execute("INSERT INTO wiki_watchlist(page, uid) \
 445                         VALUES(%s, %s)", self.page, account.uid)
 446
 447         def remove_watcher(self, account):
 448                 self.db.execute("DELETE FROM wiki_watchlist \
 449                         WHERE page = %s AND uid = %s", self.page, account.uid)
 450
 451         def _send_watcher_emails(self, excludes=[]):
 452                 # Nothing to do if there was no previous revision
 453                 if not self.previous_revision:
 454                         return
 455
 456                 for watcher in self.watchers:
 457                         # Skip everyone who is excluded
 458                         if watcher in excludes:
 459                                 logging.debug("Excluding %s" % watcher)
 460                                 continue
 461
 462                         logging.debug("Sending watcher email to %s" % watcher)
 463
 464                         # Compose message
 465                         self.backend.messages.send_template("wiki/messages/page-changed",
 466                                 recipients=[watcher], page=self, priority=-10)
 467
 468
 469 class File(misc.Object):
 470         def init(self, id, data):
 471                 self.id   = id
 472                 self.data = data
 473
 474         @property
 475         def url(self):
 476                 return os.path.join(self.path, self.filename)
 477
 478         @property
 479         def path(self):
 480                 return self.data.path
 481
 482         @property
 483         def filename(self):
 484                 return self.data.filename
 485
 486         @property
 487         def mimetype(self):
 488                 return self.data.mimetype
 489
 490         @property
 491         def size(self):
 492                 return self.data.size
 493
 494         @lazy_property
 495         def author(self):
 496                 if self.data.author_uid:
 497                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 498
 499         @property
 500         def created_at(self):
 501                 return self.data.created_at
 502
 503         def is_pdf(self):
 504                 return self.mimetype in ("application/pdf", "application/x-pdf")
 505
 506         def is_image(self):
 507                 return self.mimetype.startswith("image/")
 508
 509         @lazy_property
 510         def blob(self):
 511                 res = self.db.get("SELECT data FROM wiki_blobs \
 512                         WHERE id = %s", self.data.blob_id)
 513
 514                 if res:
 515                         return bytes(res.data)
 516
 517         def get_thumbnail(self, size):
 518                 cache_key = "-".join((self.path, util.normalize(self.filename), self.created_at.isoformat(), "%spx" % size))
 519
 520                 # Try to fetch the data from the cache
 521                 thumbnail = self.memcache.get(cache_key)
 522                 if thumbnail:
 523                         return thumbnail
 524
 525                 # Generate the thumbnail
 526                 thumbnail = util.generate_thumbnail(self.blob, size)
 527
 528                 # Put it into the cache for forever
 529                 self.memcache.set(cache_key, thumbnail)
 530
 531                 return thumbnail