From 966498debe9b3624c932f32bd00f3d70f3469439 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sat, 2 Mar 2013 12:32:42 +0100 Subject: [PATCH] Remove using memcache. Basically we don't need that, but have to optimize our application a little bit. --- backend/arches.py | 13 +------ backend/bugtracker.py | 26 +++---------- backend/builds.py | 90 ++++++++----------------------------------- backend/cache.py | 17 -------- backend/settings.py | 29 +++++--------- backend/users.py | 12 ++---- 6 files changed, 37 insertions(+), 150 deletions(-) diff --git a/backend/arches.py b/backend/arches.py index cbb00b6..2c07e2a 100644 --- a/backend/arches.py +++ b/backend/arches.py @@ -78,18 +78,7 @@ class Arch(base.Object): @property def data(self): if self._data is None: - cache_key = "arch_%s" % self.id - - # Search for the data in the cache. - # If nothing was found, we get everything from the database. - data = self.cache.get(cache_key) - if not data: - data = self.db.get("SELECT * FROM arches WHERE id = %s", self.id) - - # Store the data in the cache. - self.cache.set(cache_key, data) - - self._data = data + self._data = self.db.get("SELECT * FROM arches WHERE id = %s", self.id) assert self._data return self._data diff --git a/backend/bugtracker.py b/backend/bugtracker.py index 793e1e0..9eba8af 100644 --- a/backend/bugtracker.py +++ b/backend/bugtracker.py @@ -25,22 +25,12 @@ class BugzillaBug(base.Object): def id(self): return self.bug_id - @property - def cache_id(self): - return "bug_%s" % self.id - - def clear_cache(self): - self.cache.delete(self.cache_id) - @property def data(self): if self._data is None: - data = self.cache.get(self.cache_id) - if not data: - data = self.call("get", ids=[self.id,])["bugs"][0] - self.cache.set(self.cache_id, data, 120) + self._data = self.call("get", ids=[self.id,])["bugs"][0] + assert self._data - self._data = data return self._data @property @@ -75,7 +65,6 @@ class BugzillaBug(base.Object): kwargs["comment"] = { "body" : comment } self.call("update", ids=[self.id,], **kwargs) - self.clear_cache() @@ -153,14 +142,9 @@ class Bugzilla(base.Object): return bug def find_users(self, pattern): - users = self.cache.get("users_%s" % pattern) - if users is None: - users = self.call("User", "get", match=[pattern,]) - users = users["users"] - - self.cache.set("users_%s" % pattern, users) - - return users + users = self.call("User", "get", match=[pattern,]) + if users: + return users["users"] def find_user(self, pattern): users = self.find_users(pattern) diff --git a/backend/builds.py b/backend/builds.py index fac2b2b..34ade40 100644 --- a/backend/builds.py +++ b/backend/builds.py @@ -196,14 +196,9 @@ class Builds(base.Object): return builds def count(self): - count = self.cache.get("builds_count") - if count is None: - builds = self.db.get("SELECT COUNT(*) AS count FROM builds") - - count = builds.count - self.cache.set("builds_count", count, 3600 / 4) - - return count + builds = self.db.get("SELECT COUNT(*) AS count FROM builds") + if builds: + return builds.count def needs_test(self, threshold, arch, limit=None, randomize=False): query = "SELECT id FROM builds \ @@ -411,16 +406,6 @@ class Build(base.Object): return cmp(self.pkg, other.pkg) - @property - def cache_key(self): - return "build_%s" % self.id - - def clear_cache(self): - """ - Clear the stored data from the cache. - """ - self.cache.delete(self.cache_key) - @classmethod def create(cls, pakfire, pkg, type="release", owner=None, distro=None, public=True): assert type in ("release", "scratch", "test") @@ -502,7 +487,6 @@ class Build(base.Object): # Delete the build itself. self.db.execute("DELETE FROM builds WHERE id = %s", self.id) - self.clear_cache() def __delete_bugs(self): """ @@ -551,13 +535,7 @@ class Build(base.Object): Lazy fetching of data for this object. """ if self._data is None: - data = self.cache.get(self.cache_key) - if not data: - # Fetch the whole row in one call. - data = self.db.get("SELECT * FROM builds WHERE id = %s", self.id) - self.cache.set(self.cache_key, data) - - self._data = data + self._data = self.db.get("SELECT * FROM builds WHERE id = %s", self.id) assert self._data return self._data @@ -652,7 +630,6 @@ class Build(base.Object): def set_depends_on(self, build): self.db.execute("UPDATE builds SET depends_on = %s WHERE id = %s", build.id, self.id) - self.clear_cache() # Update cache. self._depends_on = build @@ -791,7 +768,6 @@ class Build(base.Object): if self._data: self._data["severity"] = severity - self.clear_cache() def get_severity(self): return self.data.severity @@ -808,7 +784,6 @@ class Build(base.Object): if self._data: self._data["message"] = msg - self.clear_cache() def has_perm(self, user): """ @@ -869,7 +844,6 @@ class Build(base.Object): self.db.execute("UPDATE builds SET priority = %s WHERE id = %s", priority, self.id) - self.clear_cache() if self._data: self._data["priority"] = priority @@ -1495,39 +1469,24 @@ class Jobs(base.Object): Returns the average build time of all finished builds from the last 3 months. """ - cache_key = "jobs_avg_build_time" - - build_time = self.cache.get(cache_key) - if not build_time: - result = self.db.get("SELECT AVG(time_finished - time_started) as average \ - FROM jobs WHERE type = 'build' AND state = 'finished' AND \ - time_finished >= DATE_SUB(NOW(), INTERVAL 3 MONTH)") + result = self.db.get("SELECT AVG(time_finished - time_started) as average \ + FROM jobs WHERE type = 'build' AND state = 'finished' AND \ + time_finished >= DATE_SUB(NOW(), INTERVAL 3 MONTH)") - build_time = result.average or 0 - self.cache.set(cache_key, build_time, 3600) - - return build_time + if result: + return result.average def count(self, *states): - states = sorted(states) - - cache_key = "jobs_count_%s" % ("-".join(states) or "all") - - count = self.cache.get(cache_key) - if count is None: - query = "SELECT COUNT(*) AS count FROM jobs" - args = [] - - if states: - query += " WHERE %s" % " OR ".join("state = %s" for s in states) - args += states - - jobs = self.db.get(query, *args) + query = "SELECT COUNT(*) AS count FROM jobs" + args = [] - count = jobs.count - self.cache.set(cache_key, count, 60) + if states: + query += " WHERE state IN %s" + args.append(states) - return count + jobs = self.db.get(query, *args) + if jobs: + return jobs.count def get_queue_length(self): res = self.db.get("SELECT COUNT(*) AS count FROM jobs_queue") @@ -1574,16 +1533,6 @@ class Job(base.Object): assert self.build.distro return self.build.distro - @property - def cache_key(self): - return "job_%s" % self.id - - def clear_cache(self): - """ - Clear the stored data from the cache. - """ - self.cache.delete(self.cache_key) - @classmethod def create(cls, pakfire, build, arch, type="build"): id = pakfire.db.execute("INSERT INTO jobs(uuid, type, build_id, arch_id, time_created) \ @@ -1611,7 +1560,6 @@ class Job(base.Object): # Delete the job itself. self.db.execute("DELETE FROM jobs WHERE id = %s", self.id) - self.clear_cache() def __delete_buildroots(self): """ @@ -1761,7 +1709,6 @@ class Job(base.Object): # Nothing to do if the state remains. if not self.state == state: self.db.execute("UPDATE jobs SET state = %s WHERE id = %s", state, self.id) - self.clear_cache() # Log the event. if log and not state == "new": @@ -1811,7 +1758,6 @@ class Job(base.Object): def update_message(self, msg): self.db.execute("UPDATE jobs SET message = %s WHERE id = %s", msg, self.id) - self.clear_cache() if self._data: self._data["message"] = msg @@ -1837,7 +1783,6 @@ class Job(base.Object): # Update cache. if self._data: self._data["builder_id"] = builder.id - self.clear_cache() self._builder = builder @@ -2027,7 +1972,6 @@ class Job(base.Object): def set_aborted_state(self, state): self.db.execute("UPDATE jobs SET aborted_state = %s WHERE id = %s", state, self.id) - self.clear_cache() if self._data: self._data["aborted_state"] = state diff --git a/backend/cache.py b/backend/cache.py index 08e46e9..6ff8432 100644 --- a/backend/cache.py +++ b/backend/cache.py @@ -43,20 +43,3 @@ class Cache(base.Object): key = "".join((self.key_prefix, key)) return self._memcache.delete(key, time=time) - - -class PermanentCache(base.Object): - __items = {} - - def has_key(self, key): - return self.__items.has_key(key) - - def get(self, key, default=None): - return self.__items.get(key, default) - - def set(self, key, val): - self.__items[key] = val - - def delete(self, key): - if self.__items.has_key(key): - del self.__items[key] diff --git a/backend/settings.py b/backend/settings.py index b617e0d..114bbea 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -7,41 +7,32 @@ class Settings(base.Object): def __init__(self, pakfire): base.Object.__init__(self, pakfire) - # Private cache. - self._cache = cache.PermanentCache(self.pakfire) - def query(self, key): return self.db.get("SELECT * FROM settings WHERE k = %s", key) - def get(self, key, default=None, cacheable=True): - if cacheable and self.cache.has_key(key): - return self.cache.get(key) - + def get(self, key, default=None): result = self.query(key) if not result: return default - result = result.v - - # Put the item into the cache to access it later. - if cacheable: - self.cache.set(key, result) - - return result + return result.v def get_id(self, key): - return self.query(key)["id"] + res = self.query(key) + + if res: + return res.id - def get_int(self, key, default=None, cacheable=True): - value = self.get(key, default, cacheable=cacheable) + def get_int(self, key, default=None): + value = self.get(key, default) if value is None: return None return int(value) - def get_float(self, key, default=None, cacheable=True): - value = self.get(key, default, cacheable=cacheable) + def get_float(self, key, default=None): + value = self.get(key, default) if value is None: return None diff --git a/backend/users.py b/backend/users.py index 56d4e36..d74822c 100644 --- a/backend/users.py +++ b/backend/users.py @@ -173,15 +173,11 @@ class Users(base.Object): return User(self.pakfire, user.id) def count(self): - count = self.cache.get("users_count") - if count is None: - users = self.db.get("SELECT COUNT(*) AS count FROM users \ - WHERE activated = 'Y' AND deleted = 'N'") + users = self.db.get("SELECT COUNT(*) AS count FROM users \ + WHERE activated = 'Y' AND deleted = 'N'") - count = users.count - self.cache.set("users_count", count, 3600) - - return count + if users: + return users.count def search(self, pattern, limit=None): pattern = "%%%s%%" % pattern -- 2.47.3