src/backend/blog.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import feedparser
   5 import html2text
   6 import markdown
   7 import markdown.extensions
   8 import markdown.preprocessors
   9 import re
  10 import textile
  11 import unicodedata
  12
  13 from . import misc
  14 from . import util
  15 from .decorators import *
  16
  17 class Blog(misc.Object):
  18         def _get_post(self, query, *args):
  19                 res = self.db.get(query, *args)
  20
  21                 if res:
  22                         return Post(self.backend, res.id, data=res)
  23
  24         def _get_posts(self, query, *args):
  25                 res = self.db.query(query, *args)
  26
  27                 for row in res:
  28                         yield Post(self.backend, row.id, data=row)
  29
  30         def get_by_id(self, id):
  31                 return self._get_post("SELECT * FROM blog \
  32                         WHERE id = %s", id)
  33
  34         def get_by_slug(self, slug, published=True):
  35                 if published:
  36                         return self._get_post("SELECT * FROM blog \
  37                                 WHERE slug = %s AND published_at <= NOW()", slug)
  38
  39                 return self._get_post("SELECT * FROM blog \
  40                         WHERE slug = %s", slug)
  41
  42         def get_newest(self, limit=None):
  43                 return self._get_posts("SELECT * FROM blog \
  44                         WHERE published_at IS NOT NULL \
  45                                 AND published_at <= NOW() \
  46                         ORDER BY published_at DESC LIMIT %s", limit)
  47
  48         def get_by_tag(self, tag, limit=None):
  49                 return self._get_posts("SELECT * FROM blog \
  50                         WHERE published_at IS NOT NULL \
  51                                 AND published_at <= NOW() \
  52                                 AND %s = ANY(tags) \
  53                         ORDER BY published_at DESC LIMIT %s", tag, limit)
  54
  55         def get_by_author(self, author, limit=None):
  56                 return self._get_posts("SELECT * FROM blog \
  57                         WHERE (author = %s OR author_uid = %s) \
  58                                 AND published_at IS NOT NULL \
  59                                 AND published_at <= NOW() \
  60                         ORDER BY published_at DESC LIMIT %s",
  61                         author.name, author.uid, limit)
  62
  63         def get_by_year(self, year):
  64                 return self._get_posts("SELECT * FROM blog \
  65                         WHERE EXTRACT(year FROM published_at) = %s \
  66                                 AND published_at IS NOT NULL \
  67                                 AND published_at <= NOW() \
  68                         ORDER BY published_at DESC", year)
  69
  70         def get_drafts(self, author=None, limit=None):
  71                 if author:
  72                         return self._get_posts("SELECT * FROM blog \
  73                                 WHERE author_uid = %s \
  74                                         AND (published_at IS NULL OR published_at > NOW()) \
  75                                 ORDER BY COALESCE(updated_at, created_at) DESC LIMIT %s",
  76                                 author.uid, limit)
  77
  78                 return self._get_posts("SELECT * FROM blog \
  79                         WHERE (published_at IS NULL OR published_at > NOW()) \
  80                         ORDER BY COALESCE(updated_at, created_at) DESC LIMIT %s", limit)
  81
  82         def search(self, query, limit=None):
  83                 query = util.parse_search_query(query)
  84
  85                 return self._get_posts("SELECT blog.* FROM blog \
  86                         LEFT JOIN blog_search_index search_index ON blog.id = search_index.post_id \
  87                         WHERE search_index.document @@ to_tsquery('english', %s) \
  88                                 ORDER BY ts_rank(search_index.document, to_tsquery('english', %s)) DESC \
  89                         LIMIT %s", query, query, limit)
  90
  91         def has_had_recent_activity(self, **kwargs):
  92                 t = datetime.timedelta(**kwargs)
  93
  94                 res = self.db.get("SELECT COUNT(*) AS count FROM blog \
  95                         WHERE published_at IS NOT NULL AND published_at >= NOW() - %s", t)
  96
  97                 if res and res.count > 0:
  98                         return True
  99
 100                 return False
 101
 102         def create_post(self, title, text, author, tags=[], lang="markdown"):
 103                 """
 104                         Creates a new post and returns the resulting Post object
 105                 """
 106                 # Pre-render HTML
 107                 html = self._render_text(text, lang=lang)
 108
 109                 return self._get_post("INSERT INTO blog(title, slug, text, html, lang, author_uid, tags) \
 110                         VALUES(%s, %s, %s, %s, %s, %s, %s) RETURNING *", title, self._make_slug(title), text,
 111                         html, lang, author.uid, list(tags))
 112
 113         def _make_slug(self, s):
 114                 # Remove any non-ASCII characters
 115                 try:
 116                         s = unicodedata.normalize("NFKD", s)
 117                 except TypeError:
 118                         pass
 119
 120                 # Remove excessive whitespace
 121                 s = re.sub(r"[^\w]+", " ", s)
 122
 123                 slug = "-".join(s.split()).lower()
 124
 125                 while True:
 126                         e = self.db.get("SELECT 1 FROM blog WHERE slug = %s", slug)
 127                         if not e:
 128                                 break
 129
 130                         slug += "-"
 131
 132                 return slug
 133
 134         def _render_text(self, text, lang="markdown"):
 135                 if lang == "markdown":
 136                         return markdown.markdown(text,
 137                                 extensions=[
 138                                         PrettyLinksExtension(),
 139                                         "codehilite",
 140                                         "fenced_code",
 141                                         "footnotes",
 142                                         "nl2br",
 143                                         "sane_lists",
 144                                         "tables",
 145                                         "toc",
 146                                 ])
 147
 148                 elif lang == "textile":
 149                         return textile.textile(text)
 150
 151                 return text
 152
 153         def refresh(self):
 154                 """
 155                         Needs to be called after a post has been changed
 156                         and updates the search index.
 157                 """
 158                 self.db.execute("REFRESH MATERIALIZED VIEW blog_search_index")
 159
 160         @property
 161         def years(self):
 162                 res = self.db.query("SELECT DISTINCT EXTRACT(year FROM published_at)::integer AS year \
 163                         FROM blog WHERE published_at IS NOT NULL AND published_at <= NOW() \
 164                         ORDER BY year DESC")
 165
 166                 for row in res:
 167                         yield row.year
 168
 169         async def announce(self):
 170                 posts = self._get_posts("SELECT * FROM blog \
 171                         WHERE (published_at IS NOT NULL AND published_at <= NOW()) \
 172                                 AND announced_at IS NULL")
 173
 174                 for post in posts:
 175                         await post.announce()
 176
 177         async def update_feeds(self):
 178                 """
 179                         Updates all enabled feeds
 180                 """
 181                 for feed in self.db.query("SELECT * FROM blog_feeds WHERE enabled IS TRUE"):
 182                         try:
 183                                 f = feedparser.parse(feed.url)
 184                         except Exception as e:
 185                                 raise e
 186
 187                         with self.db.transaction():
 188                                 # Update name
 189                                 self.db.execute("UPDATE blog_feeds SET name = %s \
 190                                         WHERE id = %s", f.feed.title, feed.id)
 191
 192                                 # Walk through all entries
 193                                 for entry in f.entries:
 194                                         # Skip everything without the "blog.ipfire.org" tag
 195                                         try:
 196                                                 tags = list((t.term for t in entry.tags))
 197
 198                                                 if not "blog.ipfire.org" in tags:
 199                                                         continue
 200                                         except AttributeError:
 201                                                 continue
 202
 203                                         # Get link to the posting site
 204                                         link = entry.links[0].href
 205
 206                                         # Check if the entry has already been imported
 207                                         res = self.db.get("SELECT id, (updated_at < %s) AS needs_update \
 208                                                         FROM blog WHERE feed_id = %s AND foreign_id = %s",
 209                                                         entry.updated, feed.id, entry.id)
 210                                         if res:
 211                                                 # If the post needs to be updated, we do so
 212                                                 if res.needs_update:
 213                                                         self.db.execute("UPDATE blog SET title = %s, author = %s, \
 214                                                                 published_at = %s, updated_at = %s, html = %s, link = %s, \
 215                                                                 tags = %s WHERE id = %s", entry.title, entry.author,
 216                                                                 entry.published, entry.updated, entry.summary, link,
 217                                                                 feed.tags + tags, res.id)
 218
 219                                                 # Done here
 220                                                 continue
 221
 222                                         # Insert the new post
 223                                         self.db.execute("INSERT INTO blog(title, slug, author, \
 224                                                 published_at, html, link, tags, updated_at, feed_id, foreign_id) \
 225                                                 VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
 226                                                 entry.title, self._make_slug(entry.title), entry.author,
 227                                                 entry.published, entry.summary, link, feed.tags + tags,
 228                                                 entry.updated, feed.id, entry.id)
 229
 230                                 # Mark feed as updated
 231                                 self.db.execute("UPDATE blog_feeds SET last_updated_at = CURRENT_TIMESTAMP \
 232                                         WHERE id = %s" % feed.id)
 233
 234                 # Refresh the search index
 235                 with self.db.transaction():
 236                         self.refresh()
 237
 238
 239 class Post(misc.Object):
 240         def init(self, id, data=None):
 241                 self.id   = id
 242                 self.data = data
 243
 244         # Title
 245
 246         @property
 247         def title(self):
 248                 return self.data.title
 249
 250         @property
 251         def slug(self):
 252                 return self.data.slug
 253
 254         @lazy_property
 255         def author(self):
 256                 if self.data.author_uid:
 257                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 258
 259                 return self.data.author
 260
 261         @property
 262         def created_at(self):
 263                 return self.data.created_at
 264
 265         @property
 266         def lang(self):
 267                 return self.data.lang
 268
 269         # Published?
 270
 271         @property
 272         def published_at(self):
 273                 return self.data.published_at
 274
 275         def is_published(self):
 276                 """
 277                         Returns True if the post is already published
 278                 """
 279                 return self.published_at and self.published_at <= datetime.datetime.now()
 280
 281         def publish(self, when=None):
 282                 if self.is_published():
 283                         return
 284
 285                 self.db.execute("UPDATE blog SET published_at = COALESCE(%s, CURRENT_TIMESTAMP) \
 286                         WHERE id = %s", when, self.id)
 287
 288                 # Update search indices
 289                 self.backend.blog.refresh()
 290
 291         # Updated?
 292
 293         @property
 294         def updated_at(self):
 295                 return self.data.updated_at
 296
 297         # Text
 298
 299         @property
 300         def text(self):
 301                 return self.data.text
 302
 303         # HTML
 304
 305         @lazy_property
 306         def html(self):
 307                 """
 308                         Returns this post as rendered HTML
 309                 """
 310                 return self.data.html or self.backend.blog._render_text(self.text, lang=self.lang)
 311
 312         @lazy_property
 313         def plaintext(self):
 314                 h = html2text.HTML2Text()
 315                 h.ignore_links = True
 316
 317                 return h.handle(self.html)
 318
 319         # Excerpt
 320
 321         @property
 322         def excerpt(self):
 323                 paragraphs = self.plaintext.split("\n\n")
 324
 325                 excerpt = []
 326
 327                 for paragraph in paragraphs:
 328                         excerpt.append(paragraph)
 329
 330                         # Add another paragraph if we encountered a headline
 331                         if paragraph.startswith("#"):
 332                                 continue
 333
 334                         # End if this paragraph was long enough
 335                         if len(paragraph) >= 40:
 336                                 break
 337
 338                 return "\n\n".join(excerpt)
 339
 340         # Tags
 341
 342         @property
 343         def tags(self):
 344                 return self.data.tags
 345
 346         # Link
 347
 348         @property
 349         def link(self):
 350                 return self.data.link
 351
 352         @lazy_property
 353         def release(self):
 354                 return self.backend.releases._get_release("SELECT * FROM releases \
 355                         WHERE published IS NOT NULL AND published <= NOW() AND blog_id = %s", self.id)
 356
 357         def is_editable(self, editor):
 358                 # Authors can edit their own posts
 359                 return self.author == editor
 360
 361         def update(self, title, text, tags=[]):
 362                 """
 363                         Called to update the content of this post
 364                 """
 365                 # Update slug when post isn't published yet
 366                 slug = self.backend.blog._make_slug(title) \
 367                         if not self.is_published() and not self.title == title else self.slug
 368
 369                 # Render and cache HTML
 370                 html = self.backend.blog._render_text(text, lang=self.lang)
 371
 372                 self.db.execute("UPDATE blog SET title = %s, slug = %s, text = %s, html = %s, \
 373                         tags = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s",
 374                         title, slug, text, html, list(tags), self.id)
 375
 376                 # Update cache
 377                 self.data.update({
 378                         "title" : title,
 379                         "slug"  : slug,
 380                         "text"  : text,
 381                         "html"  : html,
 382                         "tags"  : tags,
 383                 })
 384
 385                 # Update search index if post is published
 386                 if self.is_published():
 387                         self.backend.blog.refresh()
 388
 389         def delete(self):
 390                 self.db.execute("DELETE FROM blog WHERE id = %s", self.id)
 391
 392                 # Update search indices
 393                 self.backend.blog.refresh()
 394
 395         async def announce(self):
 396                 # Get people who should receive this message
 397                 group = self.backend.groups.get_by_gid("promotional-consent")
 398                 if not group:
 399                         return
 400
 401                 with self.db.transaction():
 402                         # Generate an email for everybody in this group
 403                         for account in group:
 404                                 self.backend.messages.send_template("blog/messages/announcement",
 405                                         account=account, post=self)
 406
 407                         # Mark this post as announced
 408                         self.db.execute("UPDATE blog SET announced_at = CURRENT_TIMESTAMP \
 409                                 WHERE id = %s", self.id)
 410
 411
 412 class PrettyLinksExtension(markdown.extensions.Extension):
 413         def extendMarkdown(self, md):
 414                 md.preprocessors.register(BugzillaLinksPreprocessor(md), "bugzilla", 10)
 415                 md.preprocessors.register(CVELinksPreprocessor(md), "cve", 10)
 416
 417
 418 class BugzillaLinksPreprocessor(markdown.preprocessors.Preprocessor):
 419         regex = re.compile(r"(?:#(\d{5,}))", re.I)
 420
 421         def run(self, lines):
 422                 for line in lines:
 423                         yield self.regex.sub(r"[#\1](https://bugzilla.ipfire.org/show_bug.cgi?id=\1)", line)
 424
 425
 426 class CVELinksPreprocessor(markdown.preprocessors.Preprocessor):
 427         regex = re.compile(r"(?:CVE)[\s\-](\d{4}\-\d+)")
 428
 429         def run(self, lines):
 430                 for line in lines:
 431                         yield self.regex.sub(r"[CVE-\1](https://cve.mitre.org/cgi-bin/cvename.cgi?name=\1)", line)