[ipfire.org.git] / src / backend / blog.py

#!/usr/bin/python

import datetime
import feedparser
import html2text
import re
import textile
import unicodedata

from . import misc
from . import wiki
from .decorators import *

class Blog(misc.Object):
	def _get_post(self, query, *args):
		res = self.db.get(query, *args)

		if res:
			return Post(self.backend, res.id, data=res)

	def _get_posts(self, query, *args):
		res = self.db.query(query, *args)

		for row in res:
			yield Post(self.backend, row.id, data=row)

	def get_by_id(self, id):
		return self._get_post("SELECT * FROM blog \
			WHERE id = %s", id)

	def get_by_slug(self, slug):
		return self._get_post("SELECT * FROM blog \
			WHERE slug = %s", slug)

	def get_newest(self, limit=None):
		posts = self._get_posts("SELECT * FROM blog \
			WHERE published_at IS NOT NULL \
				AND published_at <= NOW() \
			ORDER BY published_at DESC LIMIT %s", limit)

		return list(posts)

	def get_by_tag(self, tag, limit=None):
		return self._get_posts("SELECT * FROM blog \
			WHERE published_at IS NOT NULL \
				AND published_at <= NOW() \
				AND %s = ANY(tags) \
			ORDER BY published_at DESC LIMIT %s", tag, limit)

	def get_by_year(self, year):
		return self._get_posts("SELECT * FROM blog \
			WHERE EXTRACT(year FROM published_at) = %s \
				AND published_at IS NOT NULL \
				AND published_at <= NOW() \
			ORDER BY published_at DESC", year)

	def get_drafts(self, author, limit=None):
		return self._get_posts("SELECT * FROM blog \
			WHERE author_uid = %s \
				AND (published_at IS NULL OR published_at > NOW()) \
			ORDER BY COALESCE(updated_at, created_at) DESC LIMIT %s",
			author.uid, limit)

	def search(self, query, limit=None):
		posts = self._get_posts("SELECT blog.* FROM blog \
			LEFT JOIN blog_search_index search_index ON blog.id = search_index.post_id \
			WHERE search_index.document @@ websearch_to_tsquery('english', %s) \
				ORDER BY ts_rank(search_index.document, websearch_to_tsquery('english', %s)) DESC \
			LIMIT %s", query, query, limit)

		return list(posts)

	def has_had_recent_activity(self, **kwargs):
		t = datetime.timedelta(**kwargs)

		res = self.db.get("SELECT COUNT(*) AS count FROM blog \
			WHERE published_at IS NOT NULL AND published_at BETWEEN NOW() - %s AND NOW()", t)

		if res and res.count > 0:
			return True

		return False

	def create_post(self, title, text, author, tags=[], lang="markdown"):
		"""
			Creates a new post and returns the resulting Post object
		"""
		# Pre-render HTML
		html = self._render_text(text, lang=lang)

		return self._get_post("INSERT INTO blog(title, slug, text, html, lang, author_uid, tags) \
			VALUES(%s, %s, %s, %s, %s, %s, %s) RETURNING *", title, self._make_slug(title), text,
			html, lang, author.uid, list(tags))

	def _make_slug(self, s):
		# Remove any non-ASCII characters
		try:
			s = unicodedata.normalize("NFKD", s)
		except TypeError:
			pass

		# Remove excessive whitespace
		s = re.sub(r"[^\w]+", " ", s)

		slug = "-".join(s.split()).lower()

		while True:
			e = self.db.get("SELECT 1 FROM blog WHERE slug = %s", slug)
			if not e:
				break

			slug += "-"

		return slug

	def _render_text(self, text, lang="markdown"):
		if lang == "markdown":
			renderer = wiki.Markdown(
				self.backend,
				extensions=[
					wiki.PrettyLinksExtension(),
					"codehilite",
					"fenced_code",
					"footnotes",
					"nl2br",
					"sane_lists",
					"tables",
					"toc",
				],
			)

			return renderer.convert(text)

		elif lang == "textile":
			return textile.textile(text)

		else:
			return text

	def refresh(self):
		"""
			Needs to be called after a post has been changed
			and updates the search index.
		"""
		self.db.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY blog_search_index")

	@property
	def years(self):
		res = self.db.query("SELECT DISTINCT EXTRACT(year FROM published_at)::integer AS year \
			FROM blog WHERE published_at IS NOT NULL AND published_at <= NOW() \
			ORDER BY year DESC")

		for row in res:
			yield row.year

	async def announce(self):
		posts = self._get_posts("SELECT * FROM blog \
			WHERE (published_at IS NOT NULL AND published_at <= NOW()) \
				AND announced_at IS NULL")

		for post in posts:
			await post.announce()

	async def update_feeds(self):
		"""
			Updates all enabled feeds
		"""
		for feed in self.db.query("SELECT * FROM blog_feeds WHERE enabled IS TRUE"):
			try:
				f = feedparser.parse(feed.url)
			except Exception as e:
				raise e

			with self.db.transaction():
				# Update name
				self.db.execute("UPDATE blog_feeds SET name = %s \
					WHERE id = %s", f.feed.title, feed.id)

				# Walk through all entries
				for entry in f.entries:
					# Skip everything without the "blog.ipfire.org" tag
					try:
						tags = list((t.term for t in entry.tags))

						if not "blog.ipfire.org" in tags:
							continue
					except AttributeError:
						continue

					# Get link to the posting site
					link = entry.links[0].href

					# Check if the entry has already been imported
					res = self.db.get("SELECT id, (updated_at < %s) AS needs_update \
							FROM blog WHERE feed_id = %s AND foreign_id = %s",
							entry.updated, feed.id, entry.id)
					if res:
						# If the post needs to be updated, we do so
						if res.needs_update:
							self.db.execute("UPDATE blog SET title = %s, author = %s, \
								published_at = %s, updated_at = %s, html = %s, link = %s, \
								tags = %s WHERE id = %s", entry.title, entry.author,
								entry.published, entry.updated, entry.summary, link,
								feed.tags + tags, res.id)

						# Done here
						continue

					# Insert the new post
					self.db.execute("INSERT INTO blog(title, slug, author, \
						published_at, html, link, tags, updated_at, feed_id, foreign_id) \
						VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
						entry.title, self._make_slug(entry.title), entry.author,
						entry.published, entry.summary, link, feed.tags + tags,
						entry.updated, feed.id, entry.id)

				# Mark feed as updated
				self.db.execute("UPDATE blog_feeds SET last_updated_at = CURRENT_TIMESTAMP \
					WHERE id = %s" % feed.id)

		# Refresh the search index
		with self.db.transaction():
			self.refresh()


class Post(misc.Object):
	def init(self, id, data=None):
		self.id   = id
		self.data = data

	# Title

	@property
	def title(self):
		return self.data.title

	@property
	def slug(self):
		return self.data.slug

	@lazy_property
	def author(self):
		if self.data.author_uid:
			return self.backend.accounts.get_by_uid(self.data.author_uid)

		return self.data.author

	@property
	def created_at(self):
		return self.data.created_at

	@property
	def lang(self):
		return self.data.lang

	# Published?

	@property
	def published_at(self):
		return self.data.published_at

	def is_published(self):
		"""
			Returns True if the post is already published
		"""
		return self.published_at and self.published_at <= datetime.datetime.now()

	def publish(self, when=None):
		if self.is_published():
			return

		self.db.execute("UPDATE blog SET published_at = COALESCE(%s, CURRENT_TIMESTAMP) \
			WHERE id = %s", when, self.id)

		# Update search indices
		self.backend.blog.refresh()

	# Updated?

	@property
	def updated_at(self):
		return self.data.updated_at

	# Text

	@property
	def text(self):
		return self.data.text

	# HTML

	@lazy_property
	def html(self):
		"""
			Returns this post as rendered HTML
		"""
		return self.data.html or self.backend.blog._render_text(self.text, lang=self.lang)

	@lazy_property
	def plaintext(self):
		h = html2text.HTML2Text()
		h.ignore_links = True

		return h.handle(self.html)

	# Excerpt

	@property
	def excerpt(self):
		paragraphs = self.plaintext.split("\n\n")

		excerpt = []

		for paragraph in paragraphs:
			excerpt.append(paragraph)

			# Add another paragraph if we encountered a headline
			if paragraph.startswith("#"):
				continue

			# End if this paragraph was long enough
			if len(paragraph) >= 40:
				break

		return "\n\n".join(excerpt)

	# Tags

	@property
	def tags(self):
		return self.data.tags

	# Link

	@property
	def link(self):
		return self.data.link

	@lazy_property
	def release(self):
		return self.backend.releases._get_release("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published <= NOW() AND blog_id = %s", self.id)

	def is_editable(self, user):
		# Anonymous users cannot do anything
		if not user:
			return False

		# Admins can edit anything
		if user.is_admin():
			return True

		# User must have permission for the blog
		if not user.is_blog_author():
			return False

		# Authors can edit their own posts
		return self.author == user

	def update(self, title, text, tags=[]):
		"""
			Called to update the content of this post
		"""
		# Update slug when post isn't published yet
		slug = self.backend.blog._make_slug(title) \
			if not self.is_published() and not self.title == title else self.slug

		# Render and cache HTML
		html = self.backend.blog._render_text(text, lang=self.lang)

		self.db.execute("UPDATE blog SET title = %s, slug = %s, text = %s, html = %s, \
			tags = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s",
			title, slug, text, html, list(tags), self.id)

		# Update cache
		self.data.update({
			"title" : title,
			"slug"  : slug,
			"text"  : text,
			"html"  : html,
			"tags"  : tags,
		})

		# Update search index if post is published
		if self.is_published():
			self.backend.blog.refresh()

	def delete(self):
		self.db.execute("DELETE FROM blog WHERE id = %s", self.id)

		# Update search indices
		self.backend.blog.refresh()

	async def announce(self):
		# Send emails
		await self.backend.campaigns.send("blog/messages/announcement", promotional=False, post=self)

		# Mark this post as announced
		self.db.execute("UPDATE blog SET announced_at = CURRENT_TIMESTAMP \
			WHERE id = %s", self.id)
Commit	Line	Data
	1	#!/usr/bin/python
	2
	3	import datetime
	4	import feedparser
	5	import html2text
	6	import re
	7	import textile
	8	import unicodedata
	9
	10	from . import misc
	11	from . import wiki
	12	from .decorators import *
	13
	14	class Blog(misc.Object):
	15	def _get_post(self, query, *args):
	16	res = self.db.get(query, *args)
	17
	18	if res:
	19	return Post(self.backend, res.id, data=res)
	20
	21	def _get_posts(self, query, *args):
	22	res = self.db.query(query, *args)
	23
	24	for row in res:
	25	yield Post(self.backend, row.id, data=row)
	26
	27	def get_by_id(self, id):
	28	return self._get_post("SELECT * FROM blog \
	29	WHERE id = %s", id)
	30
	31	def get_by_slug(self, slug):
	32	return self._get_post("SELECT * FROM blog \
	33	WHERE slug = %s", slug)
	34
	35	def get_newest(self, limit=None):
	36	posts = self._get_posts("SELECT * FROM blog \
	37	WHERE published_at IS NOT NULL \
	38	AND published_at <= NOW() \
	39	ORDER BY published_at DESC LIMIT %s", limit)
	40
	41	return list(posts)
	42
	43	def get_by_tag(self, tag, limit=None):
	44	return self._get_posts("SELECT * FROM blog \
	45	WHERE published_at IS NOT NULL \
	46	AND published_at <= NOW() \
	47	AND %s = ANY(tags) \
	48	ORDER BY published_at DESC LIMIT %s", tag, limit)
	49
	50	def get_by_year(self, year):
	51	return self._get_posts("SELECT * FROM blog \
	52	WHERE EXTRACT(year FROM published_at) = %s \
	53	AND published_at IS NOT NULL \
	54	AND published_at <= NOW() \
	55	ORDER BY published_at DESC", year)
	56
	57	def get_drafts(self, author, limit=None):
	58	return self._get_posts("SELECT * FROM blog \
	59	WHERE author_uid = %s \
	60	AND (published_at IS NULL OR published_at > NOW()) \
	61	ORDER BY COALESCE(updated_at, created_at) DESC LIMIT %s",
	62	author.uid, limit)
	63
	64	def search(self, query, limit=None):
	65	posts = self._get_posts("SELECT blog.* FROM blog \
	66	LEFT JOIN blog_search_index search_index ON blog.id = search_index.post_id \
	67	WHERE search_index.document @@ websearch_to_tsquery('english', %s) \
	68	ORDER BY ts_rank(search_index.document, websearch_to_tsquery('english', %s)) DESC \
	69	LIMIT %s", query, query, limit)
	70
	71	return list(posts)
	72
	73	def has_had_recent_activity(self, **kwargs):
	74	t = datetime.timedelta(**kwargs)
	75
	76	res = self.db.get("SELECT COUNT(*) AS count FROM blog \
	77	WHERE published_at IS NOT NULL AND published_at BETWEEN NOW() - %s AND NOW()", t)
	78
	79	if res and res.count > 0:
	80	return True
	81
	82	return False
	83
	84	def create_post(self, title, text, author, tags=[], lang="markdown"):
	85	"""
	86	Creates a new post and returns the resulting Post object
	87	"""
	88	# Pre-render HTML
	89	html = self._render_text(text, lang=lang)
	90
	91	return self._get_post("INSERT INTO blog(title, slug, text, html, lang, author_uid, tags) \
	92	VALUES(%s, %s, %s, %s, %s, %s, %s) RETURNING *", title, self._make_slug(title), text,
	93	html, lang, author.uid, list(tags))
	94
	95	def _make_slug(self, s):
	96	# Remove any non-ASCII characters
	97	try:
	98	s = unicodedata.normalize("NFKD", s)
	99	except TypeError:
	100	pass
	101
	102	# Remove excessive whitespace
	103	s = re.sub(r"[^\w]+", " ", s)
	104
	105	slug = "-".join(s.split()).lower()
	106
	107	while True:
	108	e = self.db.get("SELECT 1 FROM blog WHERE slug = %s", slug)
	109	if not e:
	110	break
	111
	112	slug += "-"
	113
	114	return slug
	115
	116	def _render_text(self, text, lang="markdown"):
	117	if lang == "markdown":
	118	renderer = wiki.Markdown(
	119	self.backend,
	120	extensions=[
	121	wiki.PrettyLinksExtension(),
	122	"codehilite",
	123	"fenced_code",
	124	"footnotes",
	125	"nl2br",
	126	"sane_lists",
	127	"tables",
	128	"toc",
	129	],
	130	)
	131
	132	return renderer.convert(text)
	133
	134	elif lang == "textile":
	135	return textile.textile(text)
	136
	137	else:
	138	return text
	139
	140	def refresh(self):
	141	"""
	142	Needs to be called after a post has been changed
	143	and updates the search index.
	144	"""
	145	self.db.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY blog_search_index")
	146
	147	@property
	148	def years(self):
	149	res = self.db.query("SELECT DISTINCT EXTRACT(year FROM published_at)::integer AS year \
	150	FROM blog WHERE published_at IS NOT NULL AND published_at <= NOW() \
	151	ORDER BY year DESC")
	152
	153	for row in res:
	154	yield row.year
	155
	156	async def announce(self):
	157	posts = self._get_posts("SELECT * FROM blog \
	158	WHERE (published_at IS NOT NULL AND published_at <= NOW()) \
	159	AND announced_at IS NULL")
	160
	161	for post in posts:
	162	await post.announce()
	163
	164	async def update_feeds(self):
	165	"""
	166	Updates all enabled feeds
	167	"""
	168	for feed in self.db.query("SELECT * FROM blog_feeds WHERE enabled IS TRUE"):
	169	try:
	170	f = feedparser.parse(feed.url)
	171	except Exception as e:
	172	raise e
	173
	174	with self.db.transaction():
	175	# Update name
	176	self.db.execute("UPDATE blog_feeds SET name = %s \
	177	WHERE id = %s", f.feed.title, feed.id)
	178
	179	# Walk through all entries
	180	for entry in f.entries:
	181	# Skip everything without the "blog.ipfire.org" tag
	182	try:
	183	tags = list((t.term for t in entry.tags))
	184
	185	if not "blog.ipfire.org" in tags:
	186	continue
	187	except AttributeError:
	188	continue
	189
	190	# Get link to the posting site
	191	link = entry.links[0].href
	192
	193	# Check if the entry has already been imported
	194	res = self.db.get("SELECT id, (updated_at < %s) AS needs_update \
	195	FROM blog WHERE feed_id = %s AND foreign_id = %s",
	196	entry.updated, feed.id, entry.id)
	197	if res:
	198	# If the post needs to be updated, we do so
	199	if res.needs_update:
	200	self.db.execute("UPDATE blog SET title = %s, author = %s, \
	201	published_at = %s, updated_at = %s, html = %s, link = %s, \
	202	tags = %s WHERE id = %s", entry.title, entry.author,
	203	entry.published, entry.updated, entry.summary, link,
	204	feed.tags + tags, res.id)
	205
	206	# Done here
	207	continue
	208
	209	# Insert the new post
	210	self.db.execute("INSERT INTO blog(title, slug, author, \
	211	published_at, html, link, tags, updated_at, feed_id, foreign_id) \
	212	VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
	213	entry.title, self._make_slug(entry.title), entry.author,
	214	entry.published, entry.summary, link, feed.tags + tags,
	215	entry.updated, feed.id, entry.id)
	216
	217	# Mark feed as updated
	218	self.db.execute("UPDATE blog_feeds SET last_updated_at = CURRENT_TIMESTAMP \
	219	WHERE id = %s" % feed.id)
	220
	221	# Refresh the search index
	222	with self.db.transaction():
	223	self.refresh()
	224
	225
	226	class Post(misc.Object):
	227	def init(self, id, data=None):
	228	self.id = id
	229	self.data = data
	230
	231	# Title
	232
	233	@property
	234	def title(self):
	235	return self.data.title
	236
	237	@property
	238	def slug(self):
	239	return self.data.slug
	240
	241	@lazy_property
	242	def author(self):
	243	if self.data.author_uid:
	244	return self.backend.accounts.get_by_uid(self.data.author_uid)
	245
	246	return self.data.author
	247
	248	@property
	249	def created_at(self):
	250	return self.data.created_at
	251
	252	@property
	253	def lang(self):
	254	return self.data.lang
	255
	256	# Published?
	257
	258	@property
	259	def published_at(self):
	260	return self.data.published_at
	261
	262	def is_published(self):
	263	"""
	264	Returns True if the post is already published
	265	"""
	266	return self.published_at and self.published_at <= datetime.datetime.now()
	267
	268	def publish(self, when=None):
	269	if self.is_published():
	270	return
	271
	272	self.db.execute("UPDATE blog SET published_at = COALESCE(%s, CURRENT_TIMESTAMP) \
	273	WHERE id = %s", when, self.id)
	274
	275	# Update search indices
	276	self.backend.blog.refresh()
	277
	278	# Updated?
	279
	280	@property
	281	def updated_at(self):
	282	return self.data.updated_at
	283
	284	# Text
	285
	286	@property
	287	def text(self):
	288	return self.data.text
	289
	290	# HTML
	291
	292	@lazy_property
	293	def html(self):
	294	"""
	295	Returns this post as rendered HTML
	296	"""
	297	return self.data.html or self.backend.blog._render_text(self.text, lang=self.lang)
	298
	299	@lazy_property
	300	def plaintext(self):
	301	h = html2text.HTML2Text()
	302	h.ignore_links = True
	303
	304	return h.handle(self.html)
	305
	306	# Excerpt
	307
	308	@property
	309	def excerpt(self):
	310	paragraphs = self.plaintext.split("\n\n")
	311
	312	excerpt = []
	313
	314	for paragraph in paragraphs:
	315	excerpt.append(paragraph)
	316
	317	# Add another paragraph if we encountered a headline
	318	if paragraph.startswith("#"):
	319	continue
	320
	321	# End if this paragraph was long enough
	322	if len(paragraph) >= 40:
	323	break
	324
	325	return "\n\n".join(excerpt)
	326
	327	# Tags
	328
	329	@property
	330	def tags(self):
	331	return self.data.tags
	332
	333	# Link
	334
	335	@property
	336	def link(self):
	337	return self.data.link
	338
	339	@lazy_property
	340	def release(self):
	341	return self.backend.releases._get_release("SELECT * FROM releases \
	342	WHERE published IS NOT NULL AND published <= NOW() AND blog_id = %s", self.id)
	343
	344	def is_editable(self, user):
	345	# Anonymous users cannot do anything
	346	if not user:
	347	return False
	348
	349	# Admins can edit anything
	350	if user.is_admin():
	351	return True
	352
	353	# User must have permission for the blog
	354	if not user.is_blog_author():
	355	return False
	356
	357	# Authors can edit their own posts
	358	return self.author == user
	359
	360	def update(self, title, text, tags=[]):
	361	"""
	362	Called to update the content of this post
	363	"""
	364	# Update slug when post isn't published yet
	365	slug = self.backend.blog._make_slug(title) \
	366	if not self.is_published() and not self.title == title else self.slug
	367
	368	# Render and cache HTML
	369	html = self.backend.blog._render_text(text, lang=self.lang)
	370
	371	self.db.execute("UPDATE blog SET title = %s, slug = %s, text = %s, html = %s, \
	372	tags = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s",
	373	title, slug, text, html, list(tags), self.id)
	374
	375	# Update cache
	376	self.data.update({
	377	"title" : title,
	378	"slug" : slug,
	379	"text" : text,
	380	"html" : html,
	381	"tags" : tags,
	382	})
	383
	384	# Update search index if post is published
	385	if self.is_published():
	386	self.backend.blog.refresh()
	387
	388	def delete(self):
	389	self.db.execute("DELETE FROM blog WHERE id = %s", self.id)
	390
	391	# Update search indices
	392	self.backend.blog.refresh()
	393
	394	async def announce(self):
	395	# Send emails
	396	await self.backend.campaigns.send("blog/messages/announcement", promotional=False, post=self)
	397
	398	# Mark this post as announced
	399	self.db.execute("UPDATE blog SET announced_at = CURRENT_TIMESTAMP \
	400	WHERE id = %s", self.id)