src/backend/blog.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import feedparser
   5 import html2text
   6 import re
   7 import textile
   8 import unicodedata
   9
  10 from . import misc
  11 from . import wiki
  12 from .decorators import *
  13
  14 class Blog(misc.Object):
  15         def _get_post(self, query, *args):
  16                 res = self.db.get(query, *args)
  17
  18                 if res:
  19                         return Post(self.backend, res.id, data=res)
  20
  21         def _get_posts(self, query, *args):
  22                 res = self.db.query(query, *args)
  23
  24                 for row in res:
  25                         yield Post(self.backend, row.id, data=row)
  26
  27         def get_by_id(self, id):
  28                 return self._get_post("SELECT * FROM blog \
  29                         WHERE id = %s", id)
  30
  31         def get_by_slug(self, slug):
  32                 return self._get_post("SELECT * FROM blog \
  33                         WHERE slug = %s", slug)
  34
  35         def get_newest(self, limit=None):
  36                 posts = self._get_posts("SELECT * FROM blog \
  37                         WHERE published_at IS NOT NULL \
  38                                 AND published_at <= NOW() \
  39                         ORDER BY published_at DESC LIMIT %s", limit)
  40
  41                 return list(posts)
  42
  43         def get_by_tag(self, tag, limit=None):
  44                 return self._get_posts("SELECT * FROM blog \
  45                         WHERE published_at IS NOT NULL \
  46                                 AND published_at <= NOW() \
  47                                 AND %s = ANY(tags) \
  48                         ORDER BY published_at DESC LIMIT %s", tag, limit)
  49
  50         def get_by_year(self, year):
  51                 return self._get_posts("SELECT * FROM blog \
  52                         WHERE EXTRACT(year FROM published_at) = %s \
  53                                 AND published_at IS NOT NULL \
  54                                 AND published_at <= NOW() \
  55                         ORDER BY published_at DESC", year)
  56
  57         def get_drafts(self, author, limit=None):
  58                 return self._get_posts("SELECT * FROM blog \
  59                         WHERE author_uid = %s \
  60                                 AND (published_at IS NULL OR published_at > NOW()) \
  61                         ORDER BY COALESCE(updated_at, created_at) DESC LIMIT %s",
  62                         author.uid, limit)
  63
  64         def search(self, query, limit=None):
  65                 posts = self._get_posts("SELECT blog.* FROM blog \
  66                         LEFT JOIN blog_search_index search_index ON blog.id = search_index.post_id \
  67                         WHERE search_index.document @@ websearch_to_tsquery('english', %s) \
  68                                 ORDER BY ts_rank(search_index.document, websearch_to_tsquery('english', %s)) DESC \
  69                         LIMIT %s", query, query, limit)
  70
  71                 return list(posts)
  72
  73         def has_had_recent_activity(self, **kwargs):
  74                 t = datetime.timedelta(**kwargs)
  75
  76                 res = self.db.get("SELECT COUNT(*) AS count FROM blog \
  77                         WHERE published_at IS NOT NULL AND published_at BETWEEN NOW() - %s AND NOW()", t)
  78
  79                 if res and res.count > 0:
  80                         return True
  81
  82                 return False
  83
  84         def create_post(self, title, text, author, tags=[], lang="markdown"):
  85                 """
  86                         Creates a new post and returns the resulting Post object
  87                 """
  88                 # Pre-render HTML
  89                 html = self._render_text(text, lang=lang)
  90
  91                 return self._get_post("INSERT INTO blog(title, slug, text, html, lang, author_uid, tags) \
  92                         VALUES(%s, %s, %s, %s, %s, %s, %s) RETURNING *", title, self._make_slug(title), text,
  93                         html, lang, author.uid, list(tags))
  94
  95         def _make_slug(self, s):
  96                 # Remove any non-ASCII characters
  97                 try:
  98                         s = unicodedata.normalize("NFKD", s)
  99                 except TypeError:
 100                         pass
 101
 102                 # Remove excessive whitespace
 103                 s = re.sub(r"[^\w]+", " ", s)
 104
 105                 slug = "-".join(s.split()).lower()
 106
 107                 while True:
 108                         e = self.db.get("SELECT 1 FROM blog WHERE slug = %s", slug)
 109                         if not e:
 110                                 break
 111
 112                         slug += "-"
 113
 114                 return slug
 115
 116         def _render_text(self, text, lang="markdown"):
 117                 if lang == "markdown":
 118                         renderer = wiki.Markdown(
 119                                 self.backend,
 120                                 extensions=[
 121                                         wiki.PrettyLinksExtension(),
 122                                         "codehilite",
 123                                         "fenced_code",
 124                                         "footnotes",
 125                                         "nl2br",
 126                                         "sane_lists",
 127                                         "tables",
 128                                         "toc",
 129                                 ],
 130                         )
 131
 132                         return renderer.convert(text)
 133
 134                 elif lang == "textile":
 135                         return textile.textile(text)
 136
 137                 else:
 138                         return text
 139
 140         def refresh(self):
 141                 """
 142                         Needs to be called after a post has been changed
 143                         and updates the search index.
 144                 """
 145                 self.db.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY blog_search_index")
 146
 147         @property
 148         def years(self):
 149                 res = self.db.query("SELECT DISTINCT EXTRACT(year FROM published_at)::integer AS year \
 150                         FROM blog WHERE published_at IS NOT NULL AND published_at <= NOW() \
 151                         ORDER BY year DESC")
 152
 153                 for row in res:
 154                         yield row.year
 155
 156         async def announce(self):
 157                 posts = self._get_posts("SELECT * FROM blog \
 158                         WHERE (published_at IS NOT NULL AND published_at <= NOW()) \
 159                                 AND announced_at IS NULL")
 160
 161                 for post in posts:
 162                         await post.announce()
 163
 164         async def update_feeds(self):
 165                 """
 166                         Updates all enabled feeds
 167                 """
 168                 for feed in self.db.query("SELECT * FROM blog_feeds WHERE enabled IS TRUE"):
 169                         try:
 170                                 f = feedparser.parse(feed.url)
 171                         except Exception as e:
 172                                 raise e
 173
 174                         with self.db.transaction():
 175                                 # Update name
 176                                 self.db.execute("UPDATE blog_feeds SET name = %s \
 177                                         WHERE id = %s", f.feed.title, feed.id)
 178
 179                                 # Walk through all entries
 180                                 for entry in f.entries:
 181                                         # Skip everything without the "blog.ipfire.org" tag
 182                                         try:
 183                                                 tags = list((t.term for t in entry.tags))
 184
 185                                                 if not "blog.ipfire.org" in tags:
 186                                                         continue
 187                                         except AttributeError:
 188                                                 continue
 189
 190                                         # Get link to the posting site
 191                                         link = entry.links[0].href
 192
 193                                         # Check if the entry has already been imported
 194                                         res = self.db.get("SELECT id, (updated_at < %s) AS needs_update \
 195                                                         FROM blog WHERE feed_id = %s AND foreign_id = %s",
 196                                                         entry.updated, feed.id, entry.id)
 197                                         if res:
 198                                                 # If the post needs to be updated, we do so
 199                                                 if res.needs_update:
 200                                                         self.db.execute("UPDATE blog SET title = %s, author = %s, \
 201                                                                 published_at = %s, updated_at = %s, html = %s, link = %s, \
 202                                                                 tags = %s WHERE id = %s", entry.title, entry.author,
 203                                                                 entry.published, entry.updated, entry.summary, link,
 204                                                                 feed.tags + tags, res.id)
 205
 206                                                 # Done here
 207                                                 continue
 208
 209                                         # Insert the new post
 210                                         self.db.execute("INSERT INTO blog(title, slug, author, \
 211                                                 published_at, html, link, tags, updated_at, feed_id, foreign_id) \
 212                                                 VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
 213                                                 entry.title, self._make_slug(entry.title), entry.author,
 214                                                 entry.published, entry.summary, link, feed.tags + tags,
 215                                                 entry.updated, feed.id, entry.id)
 216
 217                                 # Mark feed as updated
 218                                 self.db.execute("UPDATE blog_feeds SET last_updated_at = CURRENT_TIMESTAMP \
 219                                         WHERE id = %s" % feed.id)
 220
 221                 # Refresh the search index
 222                 with self.db.transaction():
 223                         self.refresh()
 224
 225
 226 class Post(misc.Object):
 227         def init(self, id, data=None):
 228                 self.id   = id
 229                 self.data = data
 230
 231         # Title
 232
 233         @property
 234         def title(self):
 235                 return self.data.title
 236
 237         @property
 238         def slug(self):
 239                 return self.data.slug
 240
 241         @lazy_property
 242         def author(self):
 243                 if self.data.author_uid:
 244                         return self.backend.accounts.get_by_uid(self.data.author_uid)
 245
 246                 return self.data.author
 247
 248         @property
 249         def created_at(self):
 250                 return self.data.created_at
 251
 252         @property
 253         def lang(self):
 254                 return self.data.lang
 255
 256         # Published?
 257
 258         @property
 259         def published_at(self):
 260                 return self.data.published_at
 261
 262         def is_published(self):
 263                 """
 264                         Returns True if the post is already published
 265                 """
 266                 return self.published_at and self.published_at <= datetime.datetime.now()
 267
 268         def publish(self, when=None):
 269                 if self.is_published():
 270                         return
 271
 272                 self.db.execute("UPDATE blog SET published_at = COALESCE(%s, CURRENT_TIMESTAMP) \
 273                         WHERE id = %s", when, self.id)
 274
 275                 # Update search indices
 276                 self.backend.blog.refresh()
 277
 278         # Updated?
 279
 280         @property
 281         def updated_at(self):
 282                 return self.data.updated_at
 283
 284         # Text
 285
 286         @property
 287         def text(self):
 288                 return self.data.text
 289
 290         # HTML
 291
 292         @lazy_property
 293         def html(self):
 294                 """
 295                         Returns this post as rendered HTML
 296                 """
 297                 return self.data.html or self.backend.blog._render_text(self.text, lang=self.lang)
 298
 299         @lazy_property
 300         def plaintext(self):
 301                 h = html2text.HTML2Text()
 302                 h.ignore_links = True
 303
 304                 return h.handle(self.html)
 305
 306         # Excerpt
 307
 308         @property
 309         def excerpt(self):
 310                 paragraphs = self.plaintext.split("\n\n")
 311
 312                 excerpt = []
 313
 314                 for paragraph in paragraphs:
 315                         excerpt.append(paragraph)
 316
 317                         # Add another paragraph if we encountered a headline
 318                         if paragraph.startswith("#"):
 319                                 continue
 320
 321                         # End if this paragraph was long enough
 322                         if len(paragraph) >= 40:
 323                                 break
 324
 325                 return "\n\n".join(excerpt)
 326
 327         # Tags
 328
 329         @property
 330         def tags(self):
 331                 return self.data.tags
 332
 333         # Link
 334
 335         @property
 336         def link(self):
 337                 return self.data.link
 338
 339         @lazy_property
 340         def release(self):
 341                 return self.backend.releases._get_release("SELECT * FROM releases \
 342                         WHERE published IS NOT NULL AND published <= NOW() AND blog_id = %s", self.id)
 343
 344         def is_editable(self, user):
 345                 # Anonymous users cannot do anything
 346                 if not user:
 347                         return False
 348
 349                 # Admins can edit anything
 350                 if user.is_admin():
 351                         return True
 352
 353                 # User must have permission for the blog
 354                 if not user.is_blog_author():
 355                         return False
 356
 357                 # Authors can edit their own posts
 358                 return self.author == user
 359
 360         def update(self, title, text, tags=[]):
 361                 """
 362                         Called to update the content of this post
 363                 """
 364                 # Update slug when post isn't published yet
 365                 slug = self.backend.blog._make_slug(title) \
 366                         if not self.is_published() and not self.title == title else self.slug
 367
 368                 # Render and cache HTML
 369                 html = self.backend.blog._render_text(text, lang=self.lang)
 370
 371                 self.db.execute("UPDATE blog SET title = %s, slug = %s, text = %s, html = %s, \
 372                         tags = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s",
 373                         title, slug, text, html, list(tags), self.id)
 374
 375                 # Update cache
 376                 self.data.update({
 377                         "title" : title,
 378                         "slug"  : slug,
 379                         "text"  : text,
 380                         "html"  : html,
 381                         "tags"  : tags,
 382                 })
 383
 384                 # Update search index if post is published
 385                 if self.is_published():
 386                         self.backend.blog.refresh()
 387
 388         def delete(self):
 389                 self.db.execute("DELETE FROM blog WHERE id = %s", self.id)
 390
 391                 # Update search indices
 392                 self.backend.blog.refresh()
 393
 394         async def announce(self):
 395                 # Send emails
 396                 await self.backend.campaigns.send("blog/messages/announcement", promotional=False, post=self)
 397
 398                 # Mark this post as announced
 399                 self.db.execute("UPDATE blog SET announced_at = CURRENT_TIMESTAMP \
 400                         WHERE id = %s", self.id)