]>
git.ipfire.org Git - ipfire.org.git/blob - src/backend/wiki.py
7 import markdown
.extensions
8 import markdown
.preprocessors
15 from .decorators
import *
17 class Wiki(misc
.Object
):
18 def _get_pages(self
, query
, *args
):
19 res
= self
.db
.query(query
, *args
)
22 yield Page(self
.backend
, row
.id, data
=row
)
24 def _get_page(self
, query
, *args
):
25 res
= self
.db
.get(query
, *args
)
28 return Page(self
.backend
, res
.id, data
=res
)
31 return self
._get
_pages
("""
37 wiki ON current.id = wiki.id
39 current.deleted IS FALSE
44 def make_path(self
, page
, path
):
45 # Nothing to do for absolute links
46 if path
.startswith("/"):
49 # Relative links (one-level down)
50 elif path
.startswith("./"):
51 path
= os
.path
.join(page
, path
)
53 # All other relative links
55 p
= os
.path
.dirname(page
)
56 path
= os
.path
.join(p
, path
)
59 return os
.path
.normpath(path
)
61 def _make_url(self
, path
):
63 Composes the URL out of the path
65 # Remove any leading slashes (if present)
66 path
= path
.removeprefix("/")
68 return os
.path
.join("/docs", path
)
70 def get_page_title(self
, page
, default
=None):
71 doc
= self
.get_page(page
)
75 title
= os
.path
.basename(page
)
79 def get_page(self
, page
, revision
=None):
80 page
= Page
.sanitise_page_name(page
)
82 # Split the path into parts
83 parts
= page
.split("/")
85 # Check if this is an action
86 if any((part
.startswith("_") for part
in parts
)):
90 return self
._get
_page
("SELECT * FROM wiki WHERE page = %s \
91 AND timestamp = %s", page
, revision
)
93 return self
._get
_page
("SELECT * FROM wiki WHERE page = %s \
94 ORDER BY timestamp DESC LIMIT 1", page
)
96 def get_recent_changes(self
, account
, limit
=None):
97 pages
= self
._get
_pages
("SELECT * FROM wiki \
98 ORDER BY timestamp DESC")
101 if not page
.check_acl(account
):
110 def create_page(self
, page
, author
, content
, changes
=None, address
=None):
111 page
= Page
.sanitise_page_name(page
)
113 # Write page to the database
114 page
= self
._get
_page
("""
127 """, page
, author
.uid
, content
or None, changes
, address
,
130 # Store any linked files
131 page
._store
_linked
_files
()
133 # Send email to all watchers
134 page
._send
_watcher
_emails
(excludes
=[author
])
138 def delete_page(self
, page
, author
, **kwargs
):
139 # Do nothing if the page does not exist
140 if not self
.get_page(page
):
143 # Just creates a blank last version of the page
144 self
.create_page(page
, author
=author
, content
=None, **kwargs
)
146 def make_breadcrumbs(self
, path
):
150 # Cut off everything after the last slash
151 path
, _
, _
= path
.rpartition("/")
153 # Do not include the root
158 page
= self
.get_page(path
)
160 # Append the URL and title to the output
162 page
.url
if page
else self
._make
_url
(path
),
163 page
.title
if page
else os
.path
.basename(path
),
166 # Return the breadcrumbs in order
169 def search(self
, query
, account
=None, limit
=None):
170 res
= self
._get
_pages
("""
174 wiki_search_index search_index
176 wiki ON search_index.wiki_id = wiki.id
178 search_index.document @@ websearch_to_tsquery('english', %s)
180 ts_rank(search_index.document, websearch_to_tsquery('english', %s)) DESC
186 # Skip any pages the user doesn't have permission for
187 if not page
.check_acl(account
):
190 # Return any other pages
193 # Break when we have found enough pages
194 if limit
and len(pages
) >= limit
:
201 Needs to be called after a page has been changed
203 self
.db
.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY wiki_search_index")
205 def get_watchlist(self
, account
):
206 pages
= self
._get
_pages
("""
213 wiki ON wiki_current.id = wiki.id
219 wiki_watchlist watchlist
221 pages ON watchlist.page = pages.page
231 def check_acl(self
, page
, account
):
232 res
= self
.db
.query("""
238 %s ILIKE (path || '%%')
246 # Access not permitted when user is not logged in
250 # If user is in a matching group, we grant permission
251 for group
in row
.groups
:
252 if account
.is_member_of_group(group
):
255 # Otherwise access is not permitted
258 # If no ACLs are found, we permit access
263 def _get_files(self
, query
, *args
):
264 res
= self
.db
.query(query
, *args
)
267 yield File(self
.backend
, row
.id, data
=row
)
269 def _get_file(self
, query
, *args
):
270 res
= self
.db
.get(query
, *args
)
273 return File(self
.backend
, res
.id, data
=res
)
275 def get_files(self
, path
):
276 files
= self
._get
_files
("""
291 def get_file_by_path(self
, path
, revision
=None):
292 path
, filename
= os
.path
.dirname(path
), os
.path
.basename(path
)
295 # Fetch a specific revision
296 return self
._get
_file
("""
310 """, path
, filename
, revision
,
313 # Fetch latest version
314 return self
._get
_file
("""
328 def get_file_by_path_and_filename(self
, path
, filename
):
329 return self
._get
_file
("""
343 def upload(self
, path
, filename
, data
, mimetype
, author
, address
):
344 # Replace any existing files
345 file = self
.get_file_by_path_and_filename(path
, filename
)
349 # Upload the blob first
350 blob
= self
.db
.get("""
358 SET data = EXCLUDED.data
363 # Create entry for file
364 return self
._get
_file
("""
376 %s, %s, %s, %s, %s, %s, %s
379 """, path
, filename
, author
.uid
, address
, mimetype
, blob
.id, len(data
),
382 def render(self
, path
, text
, **kwargs
):
383 return WikiRenderer(self
.backend
, path
, text
, **kwargs
)
386 class Page(misc
.Object
):
387 def init(self
, id, data
=None):
392 return "<%s %s %s>" % (self
.__class
__.__name
__, self
.page
, self
.timestamp
)
394 def __eq__(self
, other
):
395 if isinstance(other
, self
.__class
__):
396 return self
.id == other
.id
398 return NotImplemented
400 def __lt__(self
, other
):
401 if isinstance(other
, self
.__class
__):
402 if self
.page
== other
.page
:
403 return self
.timestamp
< other
.timestamp
405 return self
.page
< other
.page
407 return NotImplemented
410 return hash(self
.page
)
413 def sanitise_page_name(page
):
417 # Make sure that the page name does NOT end with a /
418 if page
.endswith("/"):
421 # Make sure the page name starts with a /
422 if not page
.startswith("/"):
425 # Remove any double slashes
426 page
= page
.replace("//", "/")
432 return self
.backend
.wiki
._make
_url
(self
.page
)
436 return "https://www.ipfire.org%s" % self
.url
440 return self
.data
.page
444 return self
._title
or os
.path
.basename(self
.page
[1:])
448 if not self
.markdown
:
451 # Find first H1 headline in markdown
452 markdown
= self
.markdown
.splitlines()
454 m
= re
.match(r
"^#\s*(.*)( #)?$", markdown
[0])
460 if self
.data
.author_uid
:
461 return self
.backend
.accounts
.get_by_uid(self
.data
.author_uid
)
465 return self
.data
.markdown
or ""
471 # Strip off the first line if it contains a heading (as it will be shown separately)
472 for i
, line
in enumerate(self
.markdown
.splitlines()):
473 if i
== 0 and line
.startswith("#"):
478 renderer
= self
.backend
.wiki
.render(self
.page
, "\n".join(lines
), revision
=self
.timestamp
)
486 renderer
= self
.backend
.wiki
.render(self
.page
, self
.markdown
, revision
=self
.timestamp
)
488 return renderer
.files
490 def _store_linked_files(self
):
491 self
.db
.executemany("INSERT INTO wiki_linked_files(page_id, path) \
492 VALUES(%s, %s)", ((self
.id, file) for file in self
.files
))
496 return self
.data
.timestamp
498 def was_deleted(self
):
499 return not self
.markdown
502 def breadcrumbs(self
):
503 return self
.backend
.wiki
.make_breadcrumbs(self
.page
)
505 def is_latest_revision(self
):
506 return self
.get_latest_revision() == self
508 def get_latest_revision(self
):
509 revisions
= self
.get_revisions()
511 # Return first object
512 for rev
in revisions
:
515 def get_revisions(self
):
516 return self
.backend
.wiki
._get
_pages
("SELECT * FROM wiki \
517 WHERE page = %s ORDER BY timestamp DESC", self
.page
)
520 def previous_revision(self
):
521 return self
.backend
.wiki
._get
_page
("SELECT * FROM wiki \
522 WHERE page = %s AND timestamp < %s ORDER BY timestamp DESC \
523 LIMIT 1", self
.page
, self
.timestamp
)
527 return self
.data
.changes
531 def check_acl(self
, account
):
532 return self
.backend
.wiki
.check_acl(self
.page
, account
)
538 if self
.previous_revision
:
539 diff
= difflib
.unified_diff(
540 self
.previous_revision
.markdown
.splitlines(),
541 self
.markdown
.splitlines(),
544 return "\n".join(diff
)
548 res
= self
.db
.query("SELECT uid FROM wiki_watchlist \
549 WHERE page = %s", self
.page
)
552 # Search for account by UID and skip if none was found
553 account
= self
.backend
.accounts
.get_by_uid(row
.uid
)
560 def is_watched_by(self
, account
):
561 res
= self
.db
.get("SELECT 1 FROM wiki_watchlist \
562 WHERE page = %s AND uid = %s", self
.page
, account
.uid
)
569 def add_watcher(self
, account
):
570 if self
.is_watched_by(account
):
573 self
.db
.execute("INSERT INTO wiki_watchlist(page, uid) \
574 VALUES(%s, %s)", self
.page
, account
.uid
)
576 def remove_watcher(self
, account
):
577 self
.db
.execute("DELETE FROM wiki_watchlist \
578 WHERE page = %s AND uid = %s", self
.page
, account
.uid
)
580 def _send_watcher_emails(self
, excludes
=[]):
581 # Nothing to do if there was no previous revision
582 if not self
.previous_revision
:
585 for watcher
in self
.watchers
:
586 # Skip everyone who is excluded
587 if watcher
in excludes
:
588 logging
.debug("Excluding %s" % watcher
)
592 if not self
.backend
.wiki
.check_acl(self
.page
, watcher
):
593 logging
.debug("Watcher %s does not have permissions" % watcher
)
596 logging
.debug("Sending watcher email to %s" % watcher
)
599 self
.backend
.messages
.send_template("wiki/messages/page-changed",
600 account
=watcher
, page
=self
, priority
=-10)
602 def restore(self
, author
, address
, comment
=None):
603 changes
= "Restore to revision from %s" % self
.timestamp
.isoformat()
607 changes
= "%s: %s" % (changes
, comment
)
609 return self
.backend
.wiki
.create_page(self
.page
,
610 author
, self
.markdown
, changes
=changes
, address
=address
)
613 class File(misc
.Object
):
614 def init(self
, id, data
):
618 def __eq__(self
, other
):
619 if isinstance(other
, self
.__class
__):
620 return self
.id == other
.id
622 return NotImplemented
626 return "/docs%s" % os
.path
.join(self
.path
, self
.filename
)
630 return self
.data
.path
634 return self
.data
.filename
638 return self
.data
.mimetype
642 return self
.data
.size
646 if self
.data
.author_uid
:
647 return self
.backend
.accounts
.get_by_uid(self
.data
.author_uid
)
650 def created_at(self
):
651 return self
.data
.created_at
653 timestamp
= created_at
655 def delete(self
, author
=None):
656 if not self
.can_be_deleted():
657 raise RuntimeError("Cannot delete %s" % self
)
659 self
.db
.execute("UPDATE wiki_files SET deleted_at = NOW(), deleted_by = %s \
660 WHERE id = %s", author
.uid
if author
else None, self
.id)
662 def can_be_deleted(self
):
663 # Cannot be deleted if still in use
671 def deleted_at(self
):
672 return self
.data
.deleted_at
674 def get_latest_revision(self
):
675 revisions
= self
.get_revisions()
677 # Return first object
678 for rev
in revisions
:
681 def get_revisions(self
):
682 revisions
= self
.backend
.wiki
._get
_files
("SELECT * FROM wiki_files \
683 WHERE path = %s AND filename = %s ORDER BY created_at DESC", self
.path
, self
.filename
)
685 return list(revisions
)
688 return self
.mimetype
in ("application/pdf", "application/x-pdf")
691 return self
.mimetype
.startswith("image/")
693 def is_vector_image(self
):
694 return self
.mimetype
in ("image/svg+xml",)
696 def is_bitmap_image(self
):
697 return self
.is_image() and not self
.is_vector_image()
701 res
= self
.db
.get("SELECT data FROM wiki_blobs \
702 WHERE id = %s", self
.data
.blob_id
)
705 return bytes(res
.data
)
707 async def get_thumbnail(self
, size
, format
=None):
708 assert self
.is_bitmap_image()
710 # Let thumbnails live in the cache for up to 24h
713 cache_key
= ":".join((
717 util
.normalize(self
.filename
),
718 self
.created_at
.isoformat(),
723 # Try to fetch the data from the cache
724 async with
await self
.backend
.cache
.pipeline() as p
:
726 await p
.get(cache_key
)
729 await p
.expire(cache_key
, ttl
)
731 # Execute the pipeline
732 thumbnail
, _
= await p
.execute()
734 # Return the cached value
738 # Generate the thumbnail
739 thumbnail
= util
.generate_thumbnail(self
.blob
, size
, format
=format
, quality
=95)
741 # Put it into the cache for 24h
742 await self
.backend
.cache
.set(cache_key
, thumbnail
, ttl
)
749 Returns a list of all pages this file is linked by
751 pages
= self
.backend
.wiki
._get
_pages
("""
757 wiki_current ON wiki_linked_files.page_id = wiki_current.id
759 wiki ON wiki_linked_files.page_id = wiki.id
761 wiki_linked_files.path = %s
764 """, os
.path
.join(self
.path
, self
.filename
),
770 class WikiRenderer(misc
.Object
):
783 _links
= re
.compile(r
"<a href=\"(.*?
)\">(.*?
)</a
>")
786 _images = re.compile(r"<img
alt(?
:=\"(.*?
)\")? src
=\"(.*?
)\" (?
:title
=\"(.*?
)\" )?
/>")
788 def init(self, path, text, revision=None):
792 # Optionally, the revision of the rendered page
793 self.revision = revision
796 self.renderer = Markdown(
799 LinkedFilesExtractorExtension(),
800 PrettyLinksExtension(),
812 self.html = self._render()
814 def _render_link(self, m):
815 url, text = m.groups()
817 # Treat linkes starting with a double slash as absolute
818 if url.startswith("//"):
819 # Remove the double-lash
820 url = url.removeprefix("/")
823 return """<a href="%s">%s</a>""" % (url, text or url)
826 for schema in self.schemas:
827 if url.startswith(schema):
828 return """<a class="link
-external
" href="%s">%s</a>""" % \
834 if url.startswith("mailto
:"):
837 return """<a class="link
-external
" href="mailto
:%s">%s</a>""" % \
840 # Everything else must be an internal link
841 path = self.backend.wiki.make_path(self.path, url)
843 return """<a href="/docs
%s">%s</a>""" % \
844 (path, text or self.backend.wiki.get_page_title(path))
846 def _render_image(self, m):
847 alt_text, url, caption = m.groups()
849 # Compute a hash over the URL
850 h = hashlib.new("md5
")
851 h.update(url.encode())
855 <div class="columns
is-centered
">
856 <div class="column
is-8">
857 <figure class="image modal
-trigger
" data-target="%(id)s">
858 <img src="/docs
%(url)s?s
=960&
;%(args)s" alt="%(caption)s">
860 <figcaption class="figure
-caption
">%(caption)s</figcaption>
863 <div class="modal
is-large
" id="%(id)s">
864 <div class="modal
-background
"></div>
866 <div class="modal
-content
">
868 <img src="/docs
%(url)s?s
=2048&
;%(args)s" alt="%(caption)s"
872 <a class="button
is-small
" href="/docs
%(url)s?action
=detail
">
874 <i class="fa
-solid fa
-circle
-info
"></i>
879 <button class="modal
-close
is-large
" aria-label="close
"></button>
885 # Try to split query string
886 url, delimiter, qs = url.partition("?
")
888 # Parse query arguments
889 args = urllib.parse.parse_qs(qs)
891 # Skip any absolute and external URLs
892 if url.startswith("https
://") or url.startswith("http
://"):
894 "caption
" : caption or "",
900 # Build absolute path
901 url = self.backend.wiki.make_path(self.path, url)
904 file = self.backend.wiki.get_file_by_path(url, revision=self.revision)
905 if not file or not file.is_image():
906 return "<!-- Could
not find image
%s in %s -->" % (url, self.path)
908 # Remove any requested size
912 # Link the image that has been the current version at the time of the page edit
914 args["revision
"] = file.timestamp
917 "caption
" : caption or "",
920 "args
" : urllib.parse.urlencode(args),
924 logging.debug("Rendering
%s" % self.path)
927 text = self.renderer.convert(self.text)
930 text = self._links.sub(self._render_link, text)
932 # Postprocess images to <figure>
933 text = self._images.sub(self._render_image, text)
940 A list of all linked files that have been part of the rendered markup
944 for url in self.renderer.files:
945 # Skip external images
946 if url.startswith("https
://") or url.startswith("http
://"):
949 # Make the URL absolute
950 url = self.backend.wiki.make_path(self.path, url)
952 # Check if this is a file (it could also just be a page)
953 file = self.backend.wiki.get_file_by_path(url)
960 class Markdown(markdown.Markdown):
961 def __init__(self, backend, *args, **kwargs):
963 self.backend = backend
965 # Call inherited setup routine
966 super().__init__(*args, **kwargs)
969 class PrettyLinksExtension(markdown.extensions.Extension):
970 def extendMarkdown(self, md):
971 # Create links to Bugzilla
972 md.preprocessors.register(BugzillaLinksPreprocessor(md), "bugzilla
", 10)
974 # Create links to CVE
975 md.preprocessors.register(CVELinksPreprocessor(md), "cve
", 10)
977 # Link mentioned users
978 md.preprocessors.register(UserMentionPreprocessor(md), "user
-mention
", 10)
981 class BugzillaLinksPreprocessor(markdown.preprocessors.Preprocessor):
982 regex = re.compile(r"(?
:#(\d{5,}))", re.I)
984 def run(self
, lines
):
986 yield self
.regex
.sub(r
"[#\1](https://bugzilla.ipfire.org/show_bug.cgi?id=\1)", line
)
989 class CVELinksPreprocessor(markdown
.preprocessors
.Preprocessor
):
990 regex
= re
.compile(r
"(?:CVE)[\s\-](\d{4}\-\d+)")
992 def run(self
, lines
):
994 yield self
.regex
.sub(r
"[CVE-\1](https://cve.mitre.org/cgi-bin/cvename.cgi?name=\1)", line
)
997 class UserMentionPreprocessor(markdown
.preprocessors
.Preprocessor
):
998 regex
= re
.compile(r
"\b@(\w+)")
1000 def run(self
, lines
):
1002 yield self
.regex
.sub(self
._replace
, line
)
1004 def _replace(self
, m
):
1005 # Fetch the user's handle
1009 user
= self
.md
.backend
.accounts
.get_by_uid(uid
)
1011 # If the user was not found, we put back the matched text
1016 return "[%s](//users/%s)" % (user
, user
.uid
)
1019 class LinkedFilesExtractor(markdown
.treeprocessors
.Treeprocessor
):
1021 Finds all Linked Files
1023 def __init__(self
, *args
, **kwargs
):
1024 super().__init
__(*args
, **kwargs
)
1028 def run(self
, root
):
1029 # Find all images and store the URLs
1030 for image
in root
.findall(".//img"):
1031 src
= image
.get("src")
1033 self
.md
.files
.append(src
)
1036 for link
in root
.findall(".//a"):
1037 href
= link
.get("href")
1039 self
.md
.files
.append(href
)
1042 class LinkedFilesExtractorExtension(markdown
.extensions
.Extension
):
1043 def extendMarkdown(self
, md
):
1044 md
.treeprocessors
.register(LinkedFilesExtractor(md
), "linked-files-extractor", 10)