From: Michael Tremer Date: Thu, 23 Jan 2025 09:39:30 +0000 (+0000) Subject: database: Use joined loading a little bit more often X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3e2385d3fcdb7f7433cf88484197b560e659c50c;p=pbs.git database: Use joined loading a little bit more often Since we have so many objects that have so many very close relationships between each other, we can have lots of roundtrips to the database when we fetch them all individually in an extra query. The joined loads will however increase the size of the result by quite a siginificant amount of data when applied to 1:N relationships with larger Ns. It is pretty much free in a 1:1 relationship. Therefore this commit changes that we fetch lots of stuff in one go so we don't have as many roundtrips to the database. It will however result in larger queries and potentially a little bit more bandwidth usage. Signed-off-by: Michael Tremer --- diff --git a/src/buildservice/builders.py b/src/buildservice/builders.py index 074919ac..0feff43c 100644 --- a/src/buildservice/builders.py +++ b/src/buildservice/builders.py @@ -30,7 +30,7 @@ class BuilderStat(database.Base): # Builder builder = sqlalchemy.orm.relationship( - "Builder", foreign_keys=[builder_id], lazy="selectin", + "Builder", foreign_keys=[builder_id], lazy="joined", ) # Created At @@ -468,7 +468,7 @@ class Builder(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Created By created_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[created_by_id], lazy="selectin", + "User", foreign_keys=[created_by_id], lazy="joined", ) # Deleted By ID @@ -478,7 +478,7 @@ class Builder(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Deleted By deleted_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[deleted_by_id], lazy="selectin", + "User", foreign_keys=[deleted_by_id], lazy="joined", ) def is_online(self): diff --git a/src/buildservice/builds.py b/src/buildservice/builds.py index 3e1b2a30..dfe9c9b7 100644 --- a/src/buildservice/builds.py +++ b/src/buildservice/builds.py @@ -443,7 +443,7 @@ class Build(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Owner - owner = sqlalchemy.orm.relationship("User", foreign_keys=[owner_id], lazy="selectin") + owner = sqlalchemy.orm.relationship("User", foreign_keys=[owner_id], lazy="joined") # Build Repo ID @@ -479,7 +479,7 @@ class Build(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Commit commit = sqlalchemy.orm.relationship( - "SourceCommit", foreign_keys=[commit_id], lazy="selectin", + "SourceCommit", foreign_keys=[commit_id], lazy="joined", ) def has_perm(self, user): @@ -608,7 +608,7 @@ class Build(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Deleted By deleted_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[deleted_by_id], lazy="selectin", + "User", foreign_keys=[deleted_by_id], lazy="joined", ) # Add Points @@ -1043,7 +1043,7 @@ class Build(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Deprecated By deprecated_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[deprecated_by_id], lazy="selectin", + "User", foreign_keys=[deprecated_by_id], lazy="joined", ) # Deprecating Build ID @@ -1178,7 +1178,7 @@ class Build(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Test Group test_group = sqlalchemy.orm.relationship( - "BuildGroup", foreign_keys=[test_group_id], lazy="selectin", + "BuildGroup", foreign_keys=[test_group_id], lazy="joined", ) @@ -1263,7 +1263,7 @@ class BuildGroup(database.Base, database.BackendMixin, database.SoftDeleteMixin) # Tested Build tested_build = sqlalchemy.orm.relationship("Build", - foreign_keys=[tested_build_id], lazy="selectin") + foreign_keys=[tested_build_id], lazy="joined") # Test? @@ -1391,7 +1391,7 @@ class BuildBug(database.Base, database.BackendMixin): # Added By added_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[added_by_id], lazy="selectin", + "User", foreign_keys=[added_by_id], lazy="joined", ) # Removed At @@ -1405,7 +1405,7 @@ class BuildBug(database.Base, database.BackendMixin): # Removed ID removed_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[removed_by_id], lazy="selectin", + "User", foreign_keys=[removed_by_id], lazy="joined", ) diff --git a/src/buildservice/database.py b/src/buildservice/database.py index 98a1fdcd..fd4ea101 100644 --- a/src/buildservice/database.py +++ b/src/buildservice/database.py @@ -331,9 +331,6 @@ class Connection(object): # Execute the statement result = await session.execute(stmt) - # Apply unique filtering - #result = result.unique() - return result async def fetch(self, stmt, batch_size=128): @@ -346,6 +343,9 @@ class Connection(object): if batch_size: result = result.yield_per(batch_size) + # Apply unique filtering + result = result.unique() + # Return all objects for object in result.scalars(): yield object @@ -353,6 +353,9 @@ class Connection(object): async def fetch_one(self, stmt): result = await self.execute(stmt) + # Apply unique filtering + result = result.unique() + # Return exactly one object or none, but fail otherwise return result.scalar_one_or_none() diff --git a/src/buildservice/events.py b/src/buildservice/events.py index 14292d4d..b5c18d4b 100644 --- a/src/buildservice/events.py +++ b/src/buildservice/events.py @@ -988,7 +988,7 @@ class Event(database.Base): # By Build by_build = sqlalchemy.orm.relationship( - "Build", foreign_keys=[by_build_id], lazy="selectin", + "Build", foreign_keys=[by_build_id], lazy="joined", ) # Build Comment ID @@ -998,7 +998,7 @@ class Event(database.Base): # Build Comment build_comment = sqlalchemy.orm.relationship( - "BuildComment", foreign_keys=[build_comment_id], lazy="selectin", + "BuildComment", foreign_keys=[build_comment_id], lazy="joined", ) # Build Group ID @@ -1008,7 +1008,7 @@ class Event(database.Base): # Build Group build_group = sqlalchemy.orm.relationship( - "BuildGroup", foreign_keys=[build_group_id], lazy="selectin", + "BuildGroup", foreign_keys=[build_group_id], lazy="joined", ) # Job ID @@ -1032,7 +1032,7 @@ class Event(database.Base): # Mirror mirror = sqlalchemy.orm.relationship( - "Mirror", foreign_keys=[mirror_id], lazy="selectin", + "Mirror", foreign_keys=[mirror_id], lazy="joined", ) # User ID @@ -1042,7 +1042,7 @@ class Event(database.Base): # User user = sqlalchemy.orm.relationship( - "User", foreign_keys=[user_id], lazy="selectin", + "User", foreign_keys=[user_id], lazy="joined", ) # By User ID @@ -1052,7 +1052,7 @@ class Event(database.Base): # By User by_user = sqlalchemy.orm.relationship( - "User", foreign_keys=[by_user_id], lazy="selectin", + "User", foreign_keys=[by_user_id], lazy="joined", ) # Builder ID @@ -1062,7 +1062,7 @@ class Event(database.Base): # Builder builder = sqlalchemy.orm.relationship( - "Builder", foreign_keys=[builder_id], lazy="selectin", + "Builder", foreign_keys=[builder_id], lazy="joined", ) # Repo ID @@ -1072,7 +1072,7 @@ class Event(database.Base): # Repo repo = sqlalchemy.orm.relationship( - "Repo", foreign_keys=[repo_id], lazy="selectin", + "Repo", foreign_keys=[repo_id], lazy="joined", ) # Release ID @@ -1082,7 +1082,7 @@ class Event(database.Base): # Release release = sqlalchemy.orm.relationship( - "Release", foreign_keys=[release_id], lazy="selectin", + "Release", foreign_keys=[release_id], lazy="joined", ) # Bug diff --git a/src/buildservice/images.py b/src/buildservice/images.py index 9d8a407f..dafcbb09 100644 --- a/src/buildservice/images.py +++ b/src/buildservice/images.py @@ -43,7 +43,7 @@ class Image(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Release release = sqlalchemy.orm.relationship( - "Release", foreign_keys=[release_id], lazy="selectin", + "Release", foreign_keys=[release_id], lazy="joined", ) # Arch diff --git a/src/buildservice/jobs.py b/src/buildservice/jobs.py index 99474e52..9c0042a5 100644 --- a/src/buildservice/jobs.py +++ b/src/buildservice/jobs.py @@ -843,7 +843,7 @@ class Job(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Aborted By aborted_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[aborted_by_id], lazy="selectin", + "User", foreign_keys=[aborted_by_id], lazy="joined", ) # Message @@ -1020,7 +1020,7 @@ class Job(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Builder - builder = sqlalchemy.orm.relationship("Builder", foreign_keys=[builder_id], lazy="selectin") + builder = sqlalchemy.orm.relationship("Builder", foreign_keys=[builder_id], lazy="joined") @property def ccache_enabled(self): diff --git a/src/buildservice/keys.py b/src/buildservice/keys.py index 7d689694..3daa8526 100644 --- a/src/buildservice/keys.py +++ b/src/buildservice/keys.py @@ -65,7 +65,7 @@ class Key(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Created By created_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[created_by_id], lazy="selectin", + "User", foreign_keys=[created_by_id], lazy="joined", ) # Deleted By ID @@ -75,7 +75,7 @@ class Key(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Deleted By deleted_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[deleted_by_id], lazy="selectin", + "User", foreign_keys=[deleted_by_id], lazy="joined", ) # Public Key diff --git a/src/buildservice/mirrors.py b/src/buildservice/mirrors.py index 5516ca03..331116ef 100644 --- a/src/buildservice/mirrors.py +++ b/src/buildservice/mirrors.py @@ -190,7 +190,7 @@ class Mirror(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Created By created_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[created_by_id], lazy="selectin", + "User", foreign_keys=[created_by_id], lazy="joined", ) # Deleted By ID @@ -200,7 +200,7 @@ class Mirror(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Deleted By deleted_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[deleted_by_id], lazy="selectin", + "User", foreign_keys=[deleted_by_id], lazy="joined", ) def has_perm(self, user): diff --git a/src/buildservice/packages.py b/src/buildservice/packages.py index 5f96d750..9868ab8c 100644 --- a/src/buildservice/packages.py +++ b/src/buildservice/packages.py @@ -410,7 +410,7 @@ class Package(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Distro distro = sqlalchemy.orm.relationship("Distro", - foreign_keys=[distro_id], lazy="selectin") + foreign_keys=[distro_id], lazy="joined") # Build ID @@ -620,7 +620,7 @@ class File(database.Base): # Package - package = sqlalchemy.orm.relationship("Package", foreign_keys=[pkg_id], lazy="selectin") + package = sqlalchemy.orm.relationship("Package", foreign_keys=[pkg_id], lazy="joined") # Path diff --git a/src/buildservice/releasemonitoring.py b/src/buildservice/releasemonitoring.py index 0aab0981..556916a2 100644 --- a/src/buildservice/releasemonitoring.py +++ b/src/buildservice/releasemonitoring.py @@ -94,7 +94,7 @@ class MonitoringRelease(database.Base, database.BackendMixin): # Monitoring - monitoring = sqlalchemy.orm.relationship("Monitoring", lazy="selectin") + monitoring = sqlalchemy.orm.relationship("Monitoring", lazy="joined") # Version @@ -559,7 +559,7 @@ class Monitoring(database.Base, database.BackendMixin, database.SoftDeleteMixin) # Distro - distro = sqlalchemy.orm.relationship("Distro", lazy="selectin") + distro = sqlalchemy.orm.relationship("Distro", lazy="joined") # Name @@ -577,7 +577,7 @@ class Monitoring(database.Base, database.BackendMixin, database.SoftDeleteMixin) # Created By created_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[created_by_id], lazy="selectin", + "User", foreign_keys=[created_by_id], lazy="joined", ) # Deleted By ID @@ -587,7 +587,7 @@ class Monitoring(database.Base, database.BackendMixin, database.SoftDeleteMixin) # Deleted By deleted_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[deleted_by_id], lazy="selectin", + "User", foreign_keys=[deleted_by_id], lazy="joined", ) # Project ID @@ -778,7 +778,7 @@ class Monitoring(database.Base, database.BackendMixin, database.SoftDeleteMixin) # Latest Release latest_release = sqlalchemy.orm.relationship("MonitoringRelease", - order_by=MonitoringRelease.created_at.desc(), uselist=False, viewonly=True, lazy="selectin", + order_by=MonitoringRelease.created_at.desc(), uselist=False, viewonly=True, lazy="joined", ) async def _release_exists(self, version): diff --git a/src/buildservice/releases.py b/src/buildservice/releases.py index 92aed353..110fbf59 100644 --- a/src/buildservice/releases.py +++ b/src/buildservice/releases.py @@ -50,7 +50,7 @@ class Release(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Distro - distro = sqlalchemy.orm.relationship("Distro", lazy="selectin") + distro = sqlalchemy.orm.relationship("Distro", lazy="joined") # Name @@ -72,7 +72,7 @@ class Release(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Created By created_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[created_by_id], lazy="selectin", + "User", foreign_keys=[created_by_id], lazy="joined", ) # Deleted By ID @@ -82,7 +82,7 @@ class Release(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Deleted By deleted_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[deleted_by_id], lazy="selectin", + "User", foreign_keys=[deleted_by_id], lazy="joined", ) # Stable? diff --git a/src/buildservice/repos.py b/src/buildservice/repos.py index 3da418f3..dd04bb4e 100644 --- a/src/buildservice/repos.py +++ b/src/buildservice/repos.py @@ -41,7 +41,7 @@ class RepoBuild(database.Base): # Repo - repo = sqlalchemy.orm.relationship("Repo", foreign_keys=[repo_id], lazy="selectin") + repo = sqlalchemy.orm.relationship("Repo", foreign_keys=[repo_id], lazy="joined") # Build ID @@ -49,7 +49,7 @@ class RepoBuild(database.Base): # Build - build = sqlalchemy.orm.relationship("Build", foreign_keys=[build_id], lazy="selectin") + build = sqlalchemy.orm.relationship("Build", foreign_keys=[build_id], lazy="joined") # Added At @@ -63,7 +63,7 @@ class RepoBuild(database.Base): # Added By added_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[added_by_id], lazy="selectin", + "User", foreign_keys=[added_by_id], lazy="joined", ) # Removed At @@ -77,7 +77,7 @@ class RepoBuild(database.Base): # Removed By removed_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[removed_by_id], lazy="selectin", + "User", foreign_keys=[removed_by_id], lazy="joined", ) @@ -215,7 +215,7 @@ class Repo(database.Base, database.BackendMixin, database.SoftDeleteMixin): distro_id = Column(Integer, ForeignKey("distributions.id"), nullable=False) - distro = sqlalchemy.orm.relationship("Distro", lazy="selectin") + distro = sqlalchemy.orm.relationship("Distro", lazy="joined") # Created At @@ -250,7 +250,7 @@ class Repo(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Owner - owner = sqlalchemy.orm.relationship("User", lazy="selectin") + owner = sqlalchemy.orm.relationship("User", lazy="joined") def has_perm(self, user): """ @@ -391,7 +391,7 @@ class Repo(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Key - key = sqlalchemy.orm.relationship("Key", foreign_keys=[key_id], lazy="selectin") + key = sqlalchemy.orm.relationship("Key", foreign_keys=[key_id], lazy="joined") # Architectures diff --git a/src/buildservice/sessions.py b/src/buildservice/sessions.py index b4ab565f..d1ba5e2e 100644 --- a/src/buildservice/sessions.py +++ b/src/buildservice/sessions.py @@ -94,7 +94,7 @@ class Session(database.Base): user_id = Column(Integer, ForeignKey("users.id"), nullable=False) - user = sqlalchemy.orm.relationship("User", back_populates="sessions", lazy="selectin") + user = sqlalchemy.orm.relationship("User", back_populates="sessions", lazy="joined") # Created At diff --git a/src/buildservice/sources.py b/src/buildservice/sources.py index cf2bfdac..4ffabbe9 100644 --- a/src/buildservice/sources.py +++ b/src/buildservice/sources.py @@ -187,7 +187,7 @@ class Source(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Repo repo = sqlalchemy.orm.relationship( - "Repo", foreign_keys=[repo_id], lazy="selectin", + "Repo", foreign_keys=[repo_id], lazy="joined", ) # Distro @@ -208,7 +208,7 @@ class Source(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Created By created_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[created_by_id], lazy="selectin", + "User", foreign_keys=[created_by_id], lazy="joined", ) # Deleted By ID @@ -218,7 +218,7 @@ class Source(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Deleted By deleted_by = sqlalchemy.orm.relationship( - "User", foreign_keys=[deleted_by_id], lazy="selectin", + "User", foreign_keys=[deleted_by_id], lazy="joined", ) # Path @@ -401,7 +401,7 @@ class SourceCommit(database.Base, database.BackendMixin, database.SoftDeleteMixi # Source source = sqlalchemy.orm.relationship( - "Source", foreign_keys=[source_id], lazy="selectin", + "Source", foreign_keys=[source_id], lazy="joined", ) # Revision @@ -622,7 +622,7 @@ class SourceJob(database.Base, database.BackendMixin): # Commit commit = sqlalchemy.orm.relationship( - "SourceCommit", foreign_keys=[commit_id], lazy="selectin", + "SourceCommit", foreign_keys=[commit_id], lazy="joined", ) # Action diff --git a/src/buildservice/uploads.py b/src/buildservice/uploads.py index 9b571d1b..a5394049 100644 --- a/src/buildservice/uploads.py +++ b/src/buildservice/uploads.py @@ -173,7 +173,7 @@ class Upload(database.Base, database.BackendMixin): # Builder - builder = sqlalchemy.orm.relationship("Builder", foreign_keys=[builder_id], lazy="selectin") + builder = sqlalchemy.orm.relationship("Builder", foreign_keys=[builder_id], lazy="joined") # User ID @@ -181,7 +181,7 @@ class Upload(database.Base, database.BackendMixin): # User - user = sqlalchemy.orm.relationship("User", foreign_keys=[user_id], lazy="selectin") + user = sqlalchemy.orm.relationship("User", foreign_keys=[user_id], lazy="joined") # Has Perms? diff --git a/src/buildservice/users.py b/src/buildservice/users.py index 7920e6aa..60eef4cc 100644 --- a/src/buildservice/users.py +++ b/src/buildservice/users.py @@ -1044,7 +1044,7 @@ class UserPushSubscription(database.Base): # User - user = sqlalchemy.orm.relationship("User", lazy="selectin") + user = sqlalchemy.orm.relationship("User", lazy="joined") # UUID