[ipfire.org.git] / src / backend / blog.py

#!/usr/bin/python

import datetime
import feedparser
import markdown
import markdown.extensions
import markdown.preprocessors
import re
import textile
import tornado.gen
import unicodedata

from . import misc
from . import util
from .decorators import *

class Blog(misc.Object):
	def _get_post(self, query, *args):
		res = self.db.get(query, *args)

		if res:
			return Post(self.backend, res.id, data=res)

	def _get_posts(self, query, *args):
		res = self.db.query(query, *args)

		for row in res:
			yield Post(self.backend, row.id, data=row)

	def get_by_id(self, id):
		return self._get_post("SELECT * FROM blog \
			WHERE id = %s", id)

	def get_by_slug(self, slug, published=True):
		if published:
			return self._get_post("SELECT * FROM blog \
				WHERE slug = %s AND published_at <= NOW()", slug)

		return self._get_post("SELECT * FROM blog \
			WHERE slug = %s", slug)

	def get_newest(self, limit=None):
		return self._get_posts("SELECT * FROM blog \
			WHERE published_at IS NOT NULL \
				AND published_at <= NOW() \
			ORDER BY published_at DESC LIMIT %s", limit)

	def get_by_tag(self, tag, limit=None):
		return self._get_posts("SELECT * FROM blog \
			WHERE published_at IS NOT NULL \
				AND published_at <= NOW() \
				AND %s = ANY(tags) \
			ORDER BY published_at DESC LIMIT %s", tag, limit)

	def get_by_author(self, author, limit=None):
		return self._get_posts("SELECT * FROM blog \
			WHERE (author = %s OR author_uid = %s) \
				AND published_at IS NOT NULL \
				AND published_at <= NOW() \
			ORDER BY published_at DESC LIMIT %s",
			author.name, author.uid, limit)

	def get_by_year(self, year):
		return self._get_posts("SELECT * FROM blog \
			WHERE EXTRACT(year FROM published_at) = %s \
				AND published_at IS NOT NULL \
				AND published_at <= NOW() \
			ORDER BY published_at DESC", year)

	def get_drafts(self, author=None, limit=None):
		if author:
			return self._get_posts("SELECT * FROM blog \
				WHERE author_uid = %s \
					AND (published_at IS NULL OR published_at > NOW()) \
				ORDER BY COALESCE(updated_at, created_at) DESC LIMIT %s",
				author.uid, limit)

		return self._get_posts("SELECT * FROM blog \
			WHERE (published_at IS NULL OR published_at > NOW()) \
			ORDER BY COALESCE(updated_at, created_at) DESC LIMIT %s", limit)

	def search(self, query, limit=None):
		query = util.parse_search_query(query)

		return self._get_posts("SELECT blog.* FROM blog \
			LEFT JOIN blog_search_index search_index ON blog.id = search_index.post_id \
			WHERE search_index.document @@ to_tsquery('english', %s) \
				ORDER BY ts_rank(search_index.document, to_tsquery('english', %s)) DESC \
			LIMIT %s", query, query, limit)

	def has_had_recent_activity(self, t=None):
		if t is None:
			t = datetime.timedelta(hours=24)

		res = self.db.get("SELECT COUNT(*) AS count FROM blog \
			WHERE published_at IS NOT NULL AND published_at >= NOW() - %s", t)

		if res and res.count > 0:
			return True

		return False

	def create_post(self, title, text, author, tags=[], lang="markdown"):
		"""
			Creates a new post and returns the resulting Post object
		"""
		# Pre-render HTML
		html = self._render_text(text, lang=lang)

		return self._get_post("INSERT INTO blog(title, slug, text, html, lang, author_uid, tags) \
			VALUES(%s, %s, %s, %s, %s, %s, %s) RETURNING *", title, self._make_slug(title), text,
			html, lang, author.uid, list(tags))

	def _make_slug(self, s):
		# Remove any non-ASCII characters
		try:
			s = unicodedata.normalize("NFKD", s)
		except TypeError:
			pass

		# Remove excessive whitespace
		s = re.sub(r"[^\w]+", " ", s)

		slug = "-".join(s.split()).lower()

		while True:
			e = self.db.get("SELECT 1 FROM blog WHERE slug = %s", slug)
			if not e:
				break

			slug += "-"

		return slug

	def _render_text(self, text, lang="markdown"):
		if lang == "markdown":
			return markdown.markdown(text,
				extensions=[
					PrettyLinksExtension(),
					"codehilite",
					"fenced_code",
					"footnotes",
					"nl2br",
					"sane_lists",
					"tables",
					"toc",
				])

		elif lang == "textile":
			return textile.textile(text)

		return text

	def refresh(self):
		"""
			Needs to be called after a post has been changed
			and updates the search index.
		"""
		self.db.execute("REFRESH MATERIALIZED VIEW blog_search_index")

	@property
	def years(self):
		res = self.db.query("SELECT DISTINCT EXTRACT(year FROM published_at)::integer AS year \
			FROM blog WHERE published_at IS NOT NULL AND published_at <= NOW() \
			ORDER BY year DESC")

		for row in res:
			yield row.year

	@tornado.gen.coroutine
	def update_feeds(self):
		"""
			Updates all enabled feeds
		"""
		for feed in self.db.query("SELECT * FROM blog_feeds WHERE enabled IS TRUE"):
			try:
				f = feedparser.parse(feed.url)
			except Exception as e:
				raise e

			with self.db.transaction():
				# Update name
				self.db.execute("UPDATE blog_feeds SET name = %s \
					WHERE id = %s", f.feed.title, feed.id)

				# Walk through all entries
				for entry in f.entries:
					# Skip everything without the "blog.ipfire.org" tag
					try:
						tags = list((t.term for t in entry.tags))

						if not "blog.ipfire.org" in tags:
							continue
					except AttributeError:
						continue

					# Get link to the posting site
					link = entry.links[0].href

					# Check if the entry has already been imported
					res = self.db.get("SELECT id, (updated_at < %s) AS needs_update \
							FROM blog WHERE feed_id = %s AND foreign_id = %s",
							entry.updated, feed.id, entry.id)
					if res:
						# If the post needs to be updated, we do so
						if res.needs_update:
							self.db.execute("UPDATE blog SET title = %s, author = %s, \
								published_at = %s, updated_at = %s, html = %s, link = %s, \
								tags = %s WHERE id = %s", entry.title, entry.author,
								entry.published, entry.updated, entry.summary, link,
								feed.tags + tags, res.id)

						# Done here
						continue

					# Insert the new post
					self.db.execute("INSERT INTO blog(title, slug, author, \
						published_at, html, link, tags, updated_at, feed_id, foreign_id) \
						VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
						entry.title, self._make_slug(entry.title), entry.author,
						entry.published, entry.summary, link, feed.tags + tags,
						entry.updated, feed.id, entry.id)

				# Mark feed as updated
				self.db.execute("UPDATE blog_feeds SET last_updated_at = CURRENT_TIMESTAMP \
					WHERE id = %s" % feed.id)

		# Refresh the search index
		with self.db.transaction():
			self.refresh()


class Post(misc.Object):
	def init(self, id, data=None):
		self.id   = id
		self.data = data

	# Title

	@property
	def title(self):
		return self.data.title

	@property
	def slug(self):
		return self.data.slug

	@lazy_property
	def author(self):
		if self.data.author_uid:
			return self.backend.accounts.get_by_uid(self.data.author_uid)

		return self.data.author

	@property
	def created_at(self):
		return self.data.created_at

	@property
	def lang(self):
		return self.data.lang

	# Published?

	@property
	def published_at(self):
		return self.data.published_at

	def is_published(self):
		"""
			Returns True if the post is already published
		"""
		return self.published_at and self.published_at <= datetime.datetime.now()

	def publish(self, when=None):
		if self.is_published():
			return

		self.db.execute("UPDATE blog SET published_at = COALESCE(%s, CURRENT_TIMESTAMP) \
			WHERE id = %s", when, self.id)

		# Update search indices
		self.backend.blog.refresh()

	# Updated?

	@property
	def updated_at(self):
		return self.data.updated_at

	# Text

	@property
	def text(self):
		return self.data.text

	# HTML

	@lazy_property
	def html(self):
		"""
			Returns this post as rendered HTML
		"""
		return self.data.html or self.backend.blog._render_text(self.text, lang=self.lang)

	# Tags

	@property
	def tags(self):
		return self.data.tags

	# Link

	@property
	def link(self):
		return self.data.link

	@lazy_property
	def release(self):
		return self.backend.releases._get_release("SELECT * FROM releases \
			WHERE published IS NOT NULL AND published <= NOW() AND blog_id = %s", self.id)

	def is_editable(self, editor):
		# Authors can edit their own posts
		return self.author == editor

	def update(self, title, text, tags=[]):
		"""
			Called to update the content of this post
		"""
		# Update slug when post isn't published yet
		slug = self.backend.blog._make_slug(title) \
			if not self.is_published() and not self.title == title else self.slug

		# Render and cache HTML
		html = self.backend.blog._render_text(text, lang=self.lang)

		self.db.execute("UPDATE blog SET title = %s, slug = %s, text = %s, html = %s, \
			tags = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s",
			title, slug, text, html, list(tags), self.id)

		# Update cache
		self.data.update({
			"title" : title,
			"slug"  : slug,
			"text"  : text,
			"html"  : html,
			"tags"  : tags,
		})

		# Update search index if post is published
		if self.is_published():
			self.backend.blog.refresh()

	def delete(self):
		self.db.execute("DELETE FROM blog WHERE id = %s", self.id)

		# Update search indices
		self.backend.blog.refresh()


class PrettyLinksExtension(markdown.extensions.Extension):
	def extendMarkdown(self, md):
		md.preprocessors.register(BugzillaLinksPreprocessor(md), "bugzilla", 10)
		md.preprocessors.register(CVELinksPreprocessor(md), "cve", 10)


class BugzillaLinksPreprocessor(markdown.preprocessors.Preprocessor):
	regex = re.compile(r"(?:#(\d{5,}))", re.I)

	def run(self, lines):
		for line in lines:
			yield self.regex.sub(r"[#\1](https://bugzilla.ipfire.org/show_bug.cgi?id=\1)", line)


class CVELinksPreprocessor(markdown.preprocessors.Preprocessor):
	regex = re.compile(r"(?:CVE)[\s\-](\d{4}\-\d+)")

	def run(self, lines):
		for line in lines:
			yield self.regex.sub(r"[CVE-\1](https://cve.mitre.org/cgi-bin/cvename.cgi?name=\1)", line)
Commit	Line	Data
	1	#!/usr/bin/python
	2
	3	import datetime
	4	import feedparser
	5	import markdown
	6	import markdown.extensions
	7	import markdown.preprocessors
	8	import re
	9	import textile
	10	import tornado.gen
	11	import unicodedata
	12
	13	from . import misc
	14	from . import util
	15	from .decorators import *
	16
	17	class Blog(misc.Object):
	18	def _get_post(self, query, *args):
	19	res = self.db.get(query, *args)
	20
	21	if res:
	22	return Post(self.backend, res.id, data=res)
	23
	24	def _get_posts(self, query, *args):
	25	res = self.db.query(query, *args)
	26
	27	for row in res:
	28	yield Post(self.backend, row.id, data=row)
	29
	30	def get_by_id(self, id):
	31	return self._get_post("SELECT * FROM blog \
	32	WHERE id = %s", id)
	33
	34	def get_by_slug(self, slug, published=True):
	35	if published:
	36	return self._get_post("SELECT * FROM blog \
	37	WHERE slug = %s AND published_at <= NOW()", slug)
	38
	39	return self._get_post("SELECT * FROM blog \
	40	WHERE slug = %s", slug)
	41
	42	def get_newest(self, limit=None):
	43	return self._get_posts("SELECT * FROM blog \
	44	WHERE published_at IS NOT NULL \
	45	AND published_at <= NOW() \
	46	ORDER BY published_at DESC LIMIT %s", limit)
	47
	48	def get_by_tag(self, tag, limit=None):
	49	return self._get_posts("SELECT * FROM blog \
	50	WHERE published_at IS NOT NULL \
	51	AND published_at <= NOW() \
	52	AND %s = ANY(tags) \
	53	ORDER BY published_at DESC LIMIT %s", tag, limit)
	54
	55	def get_by_author(self, author, limit=None):
	56	return self._get_posts("SELECT * FROM blog \
	57	WHERE (author = %s OR author_uid = %s) \
	58	AND published_at IS NOT NULL \
	59	AND published_at <= NOW() \
	60	ORDER BY published_at DESC LIMIT %s",
	61	author.name, author.uid, limit)
	62
	63	def get_by_year(self, year):
	64	return self._get_posts("SELECT * FROM blog \
	65	WHERE EXTRACT(year FROM published_at) = %s \
	66	AND published_at IS NOT NULL \
	67	AND published_at <= NOW() \
	68	ORDER BY published_at DESC", year)
	69
	70	def get_drafts(self, author=None, limit=None):
	71	if author:
	72	return self._get_posts("SELECT * FROM blog \
	73	WHERE author_uid = %s \
	74	AND (published_at IS NULL OR published_at > NOW()) \
	75	ORDER BY COALESCE(updated_at, created_at) DESC LIMIT %s",
	76	author.uid, limit)
	77
	78	return self._get_posts("SELECT * FROM blog \
	79	WHERE (published_at IS NULL OR published_at > NOW()) \
	80	ORDER BY COALESCE(updated_at, created_at) DESC LIMIT %s", limit)
	81
	82	def search(self, query, limit=None):
	83	query = util.parse_search_query(query)
	84
	85	return self._get_posts("SELECT blog.* FROM blog \
	86	LEFT JOIN blog_search_index search_index ON blog.id = search_index.post_id \
	87	WHERE search_index.document @@ to_tsquery('english', %s) \
	88	ORDER BY ts_rank(search_index.document, to_tsquery('english', %s)) DESC \
	89	LIMIT %s", query, query, limit)
	90
	91	def has_had_recent_activity(self, t=None):
	92	if t is None:
	93	t = datetime.timedelta(hours=24)
	94
	95	res = self.db.get("SELECT COUNT(*) AS count FROM blog \
	96	WHERE published_at IS NOT NULL AND published_at >= NOW() - %s", t)
	97
	98	if res and res.count > 0:
	99	return True
	100
	101	return False
	102
	103	def create_post(self, title, text, author, tags=[], lang="markdown"):
	104	"""
	105	Creates a new post and returns the resulting Post object
	106	"""
	107	# Pre-render HTML
	108	html = self._render_text(text, lang=lang)
	109
	110	return self._get_post("INSERT INTO blog(title, slug, text, html, lang, author_uid, tags) \
	111	VALUES(%s, %s, %s, %s, %s, %s, %s) RETURNING *", title, self._make_slug(title), text,
	112	html, lang, author.uid, list(tags))
	113
	114	def _make_slug(self, s):
	115	# Remove any non-ASCII characters
	116	try:
	117	s = unicodedata.normalize("NFKD", s)
	118	except TypeError:
	119	pass
	120
	121	# Remove excessive whitespace
	122	s = re.sub(r"[^\w]+", " ", s)
	123
	124	slug = "-".join(s.split()).lower()
	125
	126	while True:
	127	e = self.db.get("SELECT 1 FROM blog WHERE slug = %s", slug)
	128	if not e:
	129	break
	130
	131	slug += "-"
	132
	133	return slug
	134
	135	def _render_text(self, text, lang="markdown"):
	136	if lang == "markdown":
	137	return markdown.markdown(text,
	138	extensions=[
	139	PrettyLinksExtension(),
	140	"codehilite",
	141	"fenced_code",
	142	"footnotes",
	143	"nl2br",
	144	"sane_lists",
	145	"tables",
	146	"toc",
	147	])
	148
	149	elif lang == "textile":
	150	return textile.textile(text)
	151
	152	return text
	153
	154	def refresh(self):
	155	"""
	156	Needs to be called after a post has been changed
	157	and updates the search index.
	158	"""
	159	self.db.execute("REFRESH MATERIALIZED VIEW blog_search_index")
	160
	161	@property
	162	def years(self):
	163	res = self.db.query("SELECT DISTINCT EXTRACT(year FROM published_at)::integer AS year \
	164	FROM blog WHERE published_at IS NOT NULL AND published_at <= NOW() \
	165	ORDER BY year DESC")
	166
	167	for row in res:
	168	yield row.year
	169
	170	@tornado.gen.coroutine
	171	def update_feeds(self):
	172	"""
	173	Updates all enabled feeds
	174	"""
	175	for feed in self.db.query("SELECT * FROM blog_feeds WHERE enabled IS TRUE"):
	176	try:
	177	f = feedparser.parse(feed.url)
	178	except Exception as e:
	179	raise e
	180
	181	with self.db.transaction():
	182	# Update name
	183	self.db.execute("UPDATE blog_feeds SET name = %s \
	184	WHERE id = %s", f.feed.title, feed.id)
	185
	186	# Walk through all entries
	187	for entry in f.entries:
	188	# Skip everything without the "blog.ipfire.org" tag
	189	try:
	190	tags = list((t.term for t in entry.tags))
	191
	192	if not "blog.ipfire.org" in tags:
	193	continue
	194	except AttributeError:
	195	continue
	196
	197	# Get link to the posting site
	198	link = entry.links[0].href
	199
	200	# Check if the entry has already been imported
	201	res = self.db.get("SELECT id, (updated_at < %s) AS needs_update \
	202	FROM blog WHERE feed_id = %s AND foreign_id = %s",
	203	entry.updated, feed.id, entry.id)
	204	if res:
	205	# If the post needs to be updated, we do so
	206	if res.needs_update:
	207	self.db.execute("UPDATE blog SET title = %s, author = %s, \
	208	published_at = %s, updated_at = %s, html = %s, link = %s, \
	209	tags = %s WHERE id = %s", entry.title, entry.author,
	210	entry.published, entry.updated, entry.summary, link,
	211	feed.tags + tags, res.id)
	212
	213	# Done here
	214	continue
	215
	216	# Insert the new post
	217	self.db.execute("INSERT INTO blog(title, slug, author, \
	218	published_at, html, link, tags, updated_at, feed_id, foreign_id) \
	219	VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
	220	entry.title, self._make_slug(entry.title), entry.author,
	221	entry.published, entry.summary, link, feed.tags + tags,
	222	entry.updated, feed.id, entry.id)
	223
	224	# Mark feed as updated
	225	self.db.execute("UPDATE blog_feeds SET last_updated_at = CURRENT_TIMESTAMP \
	226	WHERE id = %s" % feed.id)
	227
	228	# Refresh the search index
	229	with self.db.transaction():
	230	self.refresh()
	231
	232
	233	class Post(misc.Object):
	234	def init(self, id, data=None):
	235	self.id = id
	236	self.data = data
	237
	238	# Title
	239
	240	@property
	241	def title(self):
	242	return self.data.title
	243
	244	@property
	245	def slug(self):
	246	return self.data.slug
	247
	248	@lazy_property
	249	def author(self):
	250	if self.data.author_uid:
	251	return self.backend.accounts.get_by_uid(self.data.author_uid)
	252
	253	return self.data.author
	254
	255	@property
	256	def created_at(self):
	257	return self.data.created_at
	258
	259	@property
	260	def lang(self):
	261	return self.data.lang
	262
	263	# Published?
	264
	265	@property
	266	def published_at(self):
	267	return self.data.published_at
	268
	269	def is_published(self):
	270	"""
	271	Returns True if the post is already published
	272	"""
	273	return self.published_at and self.published_at <= datetime.datetime.now()
	274
	275	def publish(self, when=None):
	276	if self.is_published():
	277	return
	278
	279	self.db.execute("UPDATE blog SET published_at = COALESCE(%s, CURRENT_TIMESTAMP) \
	280	WHERE id = %s", when, self.id)
	281
	282	# Update search indices
	283	self.backend.blog.refresh()
	284
	285	# Updated?
	286
	287	@property
	288	def updated_at(self):
	289	return self.data.updated_at
	290
	291	# Text
	292
	293	@property
	294	def text(self):
	295	return self.data.text
	296
	297	# HTML
	298
	299	@lazy_property
	300	def html(self):
	301	"""
	302	Returns this post as rendered HTML
	303	"""
	304	return self.data.html or self.backend.blog._render_text(self.text, lang=self.lang)
	305
	306	# Tags
	307
	308	@property
	309	def tags(self):
	310	return self.data.tags
	311
	312	# Link
	313
	314	@property
	315	def link(self):
	316	return self.data.link
	317
	318	@lazy_property
	319	def release(self):
	320	return self.backend.releases._get_release("SELECT * FROM releases \
	321	WHERE published IS NOT NULL AND published <= NOW() AND blog_id = %s", self.id)
	322
	323	def is_editable(self, editor):
	324	# Authors can edit their own posts
	325	return self.author == editor
	326
	327	def update(self, title, text, tags=[]):
	328	"""
	329	Called to update the content of this post
	330	"""
	331	# Update slug when post isn't published yet
	332	slug = self.backend.blog._make_slug(title) \
	333	if not self.is_published() and not self.title == title else self.slug
	334
	335	# Render and cache HTML
	336	html = self.backend.blog._render_text(text, lang=self.lang)
	337
	338	self.db.execute("UPDATE blog SET title = %s, slug = %s, text = %s, html = %s, \
	339	tags = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s",
	340	title, slug, text, html, list(tags), self.id)
	341
	342	# Update cache
	343	self.data.update({
	344	"title" : title,
	345	"slug" : slug,
	346	"text" : text,
	347	"html" : html,
	348	"tags" : tags,
	349	})
	350
	351	# Update search index if post is published
	352	if self.is_published():
	353	self.backend.blog.refresh()
	354
	355	def delete(self):
	356	self.db.execute("DELETE FROM blog WHERE id = %s", self.id)
	357
	358	# Update search indices
	359	self.backend.blog.refresh()
	360
	361
	362	class PrettyLinksExtension(markdown.extensions.Extension):
	363	def extendMarkdown(self, md):
	364	md.preprocessors.register(BugzillaLinksPreprocessor(md), "bugzilla", 10)
	365	md.preprocessors.register(CVELinksPreprocessor(md), "cve", 10)
	366
	367
	368	class BugzillaLinksPreprocessor(markdown.preprocessors.Preprocessor):
	369	regex = re.compile(r"(?:#(\d{5,}))", re.I)
	370
	371	def run(self, lines):
	372	for line in lines:
	373	yield self.regex.sub(r"[#\1](https://bugzilla.ipfire.org/show_bug.cgi?id=\1)", line)
	374
	375
	376	class CVELinksPreprocessor(markdown.preprocessors.Preprocessor):
	377	regex = re.compile(r"(?:CVE)[\s\-](\d{4}\-\d+)")
	378
	379	def run(self, lines):
	380	for line in lines:
	381	yield self.regex.sub(r"[CVE-\1](https://cve.mitre.org/cgi-bin/cvename.cgi?name=\1)", line)