database: Use joined loading a little bit more often

Since we have so many objects that have so many very close relationships
between each other, we can have lots of roundtrips to the database when
we fetch them all individually in an extra query.

The joined loads will however increase the size of the result by quite a
siginificant amount of data when applied to 1:N relationships with
larger Ns. It is pretty much free in a 1:1 relationship.

Therefore this commit changes that we fetch lots of stuff in one go so
we don't have as many roundtrips to the database. It will however result
in larger queries and potentially a little bit more bandwidth usage.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/buildservice/builders.py b/src/buildservice/builders.py
index 074919acb803dc694fcdc460b14cd68bc82e68ca..0feff43c8338036c4f1fcbfd799251b94957fce5 100644 (file)
--- a/src/buildservice/builders.py
+++ b/src/buildservice/builders.py
@@ -30,7 +30,7 @@ class BuilderStat(database.Base):
        # Builder
 
        builder = sqlalchemy.orm.relationship(
-               "Builder", foreign_keys=[builder_id], lazy="selectin",
+               "Builder", foreign_keys=[builder_id], lazy="joined",
        )
 
        # Created At
@@ -468,7 +468,7 @@ class Builder(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Created By
 
        created_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[created_by_id], lazy="selectin",
+               "User", foreign_keys=[created_by_id], lazy="joined",
        )
 
        # Deleted By ID
@@ -478,7 +478,7 @@ class Builder(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Deleted By
 
        deleted_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[deleted_by_id], lazy="selectin",
+               "User", foreign_keys=[deleted_by_id], lazy="joined",
        )
 
        def is_online(self):

diff --git a/src/buildservice/builds.py b/src/buildservice/builds.py
index 3e1b2a30da0a6ff8c64f1d87ec8167887c2d947b..dfe9c9b7d7edbdb4c738a4e7e736d78accf73116 100644 (file)
--- a/src/buildservice/builds.py
+++ b/src/buildservice/builds.py
@@ -443,7 +443,7 @@ class Build(database.Base, database.BackendMixin, database.SoftDeleteMixin):
 
        # Owner
 
-       owner = sqlalchemy.orm.relationship("User", foreign_keys=[owner_id], lazy="selectin")
+       owner = sqlalchemy.orm.relationship("User", foreign_keys=[owner_id], lazy="joined")
 
        # Build Repo ID
 
@@ -479,7 +479,7 @@ class Build(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Commit
 
        commit = sqlalchemy.orm.relationship(
-               "SourceCommit", foreign_keys=[commit_id], lazy="selectin",
+               "SourceCommit", foreign_keys=[commit_id], lazy="joined",
        )
 
        def has_perm(self, user):
@@ -608,7 +608,7 @@ class Build(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Deleted By
 
        deleted_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[deleted_by_id], lazy="selectin",
+               "User", foreign_keys=[deleted_by_id], lazy="joined",
        )
 
        # Add Points
@@ -1043,7 +1043,7 @@ class Build(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Deprecated By
 
        deprecated_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[deprecated_by_id], lazy="selectin",
+               "User", foreign_keys=[deprecated_by_id], lazy="joined",
        )
 
        # Deprecating Build ID
@@ -1178,7 +1178,7 @@ class Build(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Test Group
 
        test_group = sqlalchemy.orm.relationship(
-               "BuildGroup", foreign_keys=[test_group_id], lazy="selectin",
+               "BuildGroup", foreign_keys=[test_group_id], lazy="joined",
        )
 
 
@@ -1263,7 +1263,7 @@ class BuildGroup(database.Base, database.BackendMixin, database.SoftDeleteMixin)
        # Tested Build
 
        tested_build = sqlalchemy.orm.relationship("Build",
-               foreign_keys=[tested_build_id], lazy="selectin")
+               foreign_keys=[tested_build_id], lazy="joined")
 
        # Test?
 
@@ -1391,7 +1391,7 @@ class BuildBug(database.Base, database.BackendMixin):
        # Added By
 
        added_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[added_by_id], lazy="selectin",
+               "User", foreign_keys=[added_by_id], lazy="joined",
        )
 
        # Removed At
@@ -1405,7 +1405,7 @@ class BuildBug(database.Base, database.BackendMixin):
        # Removed ID
 
        removed_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[removed_by_id], lazy="selectin",
+               "User", foreign_keys=[removed_by_id], lazy="joined",
        )
 
 

diff --git a/src/buildservice/database.py b/src/buildservice/database.py
index 98a1fdcdd332ae340169c4ed4f3b685d4051a2d9..fd4ea1015be648f885fcc317158cc742efbb3d8c 100644 (file)
--- a/src/buildservice/database.py
+++ b/src/buildservice/database.py
@@ -331,9 +331,6 @@ class Connection(object):
                # Execute the statement
                result = await session.execute(stmt)
 
-               # Apply unique filtering
-               #result = result.unique()
-
                return result
 
        async def fetch(self, stmt, batch_size=128):
@@ -346,6 +343,9 @@ class Connection(object):
                if batch_size:
                        result = result.yield_per(batch_size)
 
+               # Apply unique filtering
+               result = result.unique()
+
                # Return all objects
                for object in result.scalars():
                        yield object
@@ -353,6 +353,9 @@ class Connection(object):
        async def fetch_one(self, stmt):
                result = await self.execute(stmt)
 
+               # Apply unique filtering
+               result = result.unique()
+
                # Return exactly one object or none, but fail otherwise
                return result.scalar_one_or_none()
 

diff --git a/src/buildservice/events.py b/src/buildservice/events.py
index 14292d4d83fa2fbddfd641ef022cec4066bfb7af..b5c18d4b8cae9b7beb03b21411956cc48f91be8c 100644 (file)
--- a/src/buildservice/events.py
+++ b/src/buildservice/events.py
@@ -988,7 +988,7 @@ class Event(database.Base):
        # By Build
 
        by_build = sqlalchemy.orm.relationship(
-               "Build", foreign_keys=[by_build_id], lazy="selectin",
+               "Build", foreign_keys=[by_build_id], lazy="joined",
        )
 
        # Build Comment ID
@@ -998,7 +998,7 @@ class Event(database.Base):
        # Build Comment
 
        build_comment = sqlalchemy.orm.relationship(
-               "BuildComment", foreign_keys=[build_comment_id], lazy="selectin",
+               "BuildComment", foreign_keys=[build_comment_id], lazy="joined",
        )
 
        # Build Group ID
@@ -1008,7 +1008,7 @@ class Event(database.Base):
        # Build Group
 
        build_group = sqlalchemy.orm.relationship(
-               "BuildGroup", foreign_keys=[build_group_id], lazy="selectin",
+               "BuildGroup", foreign_keys=[build_group_id], lazy="joined",
        )
 
        # Job ID
@@ -1032,7 +1032,7 @@ class Event(database.Base):
        # Mirror
 
        mirror = sqlalchemy.orm.relationship(
-               "Mirror", foreign_keys=[mirror_id], lazy="selectin",
+               "Mirror", foreign_keys=[mirror_id], lazy="joined",
        )
 
        # User ID
@@ -1042,7 +1042,7 @@ class Event(database.Base):
        # User
 
        user = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[user_id], lazy="selectin",
+               "User", foreign_keys=[user_id], lazy="joined",
        )
 
        # By User ID
@@ -1052,7 +1052,7 @@ class Event(database.Base):
        # By User
 
        by_user = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[by_user_id], lazy="selectin",
+               "User", foreign_keys=[by_user_id], lazy="joined",
        )
 
        # Builder ID
@@ -1062,7 +1062,7 @@ class Event(database.Base):
        # Builder
 
        builder = sqlalchemy.orm.relationship(
-               "Builder", foreign_keys=[builder_id], lazy="selectin",
+               "Builder", foreign_keys=[builder_id], lazy="joined",
        )
 
        # Repo ID
@@ -1072,7 +1072,7 @@ class Event(database.Base):
        # Repo
 
        repo = sqlalchemy.orm.relationship(
-               "Repo", foreign_keys=[repo_id], lazy="selectin",
+               "Repo", foreign_keys=[repo_id], lazy="joined",
        )
 
        # Release ID
@@ -1082,7 +1082,7 @@ class Event(database.Base):
        # Release
 
        release = sqlalchemy.orm.relationship(
-               "Release", foreign_keys=[release_id], lazy="selectin",
+               "Release", foreign_keys=[release_id], lazy="joined",
        )
 
        # Bug

diff --git a/src/buildservice/images.py b/src/buildservice/images.py
index 9d8a407f8c6e9d71d88582bd462d0a8cba1cc97e..dafcbb0971e17a408addb5664fa68070ed8fad48 100644 (file)
--- a/src/buildservice/images.py
+++ b/src/buildservice/images.py
@@ -43,7 +43,7 @@ class Image(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Release
 
        release = sqlalchemy.orm.relationship(
-               "Release", foreign_keys=[release_id], lazy="selectin",
+               "Release", foreign_keys=[release_id], lazy="joined",
        )
 
        # Arch

diff --git a/src/buildservice/jobs.py b/src/buildservice/jobs.py
index 99474e520c54443a667b8299e0990874df084baa..9c0042a5faa928becdbb89316cd07d3bf09706b1 100644 (file)
--- a/src/buildservice/jobs.py
+++ b/src/buildservice/jobs.py
@@ -843,7 +843,7 @@ class Job(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Aborted By
 
        aborted_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[aborted_by_id], lazy="selectin",
+               "User", foreign_keys=[aborted_by_id], lazy="joined",
        )
 
        # Message
@@ -1020,7 +1020,7 @@ class Job(database.Base, database.BackendMixin, database.SoftDeleteMixin):
 
        # Builder
 
-       builder = sqlalchemy.orm.relationship("Builder", foreign_keys=[builder_id], lazy="selectin")
+       builder = sqlalchemy.orm.relationship("Builder", foreign_keys=[builder_id], lazy="joined")
 
        @property
        def ccache_enabled(self):

diff --git a/src/buildservice/keys.py b/src/buildservice/keys.py
index 7d689694ac79ee34023b749fa635fa575c1fee93..3daa8526c07ef266f60de33aa457daf5e9955e09 100644 (file)
--- a/src/buildservice/keys.py
+++ b/src/buildservice/keys.py
@@ -65,7 +65,7 @@ class Key(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Created By
 
        created_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[created_by_id], lazy="selectin",
+               "User", foreign_keys=[created_by_id], lazy="joined",
        )
 
        # Deleted By ID
@@ -75,7 +75,7 @@ class Key(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Deleted By
 
        deleted_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[deleted_by_id], lazy="selectin",
+               "User", foreign_keys=[deleted_by_id], lazy="joined",
        )
 
        # Public Key

diff --git a/src/buildservice/mirrors.py b/src/buildservice/mirrors.py
index 5516ca0388486a1371f281913b45e2f0cf3ff768..331116ef5b30adb1d20e18d0224cea388b1b69a9 100644 (file)
--- a/src/buildservice/mirrors.py
+++ b/src/buildservice/mirrors.py
@@ -190,7 +190,7 @@ class Mirror(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Created By
 
        created_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[created_by_id], lazy="selectin",
+               "User", foreign_keys=[created_by_id], lazy="joined",
        )
 
        # Deleted By ID
@@ -200,7 +200,7 @@ class Mirror(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Deleted By
 
        deleted_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[deleted_by_id], lazy="selectin",
+               "User", foreign_keys=[deleted_by_id], lazy="joined",
        )
 
        def has_perm(self, user):

diff --git a/src/buildservice/packages.py b/src/buildservice/packages.py
index 5f96d7507b7ac515a61223bdceb0d7f949badb6f..9868ab8c904a2959a152c4c7e7e5d35607751e43 100644 (file)
--- a/src/buildservice/packages.py
+++ b/src/buildservice/packages.py
@@ -410,7 +410,7 @@ class Package(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Distro
 
        distro = sqlalchemy.orm.relationship("Distro",
-               foreign_keys=[distro_id], lazy="selectin")
+               foreign_keys=[distro_id], lazy="joined")
 
        # Build ID
 
@@ -620,7 +620,7 @@ class File(database.Base):
 
        # Package
 
-       package = sqlalchemy.orm.relationship("Package", foreign_keys=[pkg_id], lazy="selectin")
+       package = sqlalchemy.orm.relationship("Package", foreign_keys=[pkg_id], lazy="joined")
 
        # Path
 

diff --git a/src/buildservice/releasemonitoring.py b/src/buildservice/releasemonitoring.py
index 0aab09814390ed3e3e49d1b66816696d2227bced..556916a2f222244f8569c97dbc9d7af963e8abb2 100644 (file)
--- a/src/buildservice/releasemonitoring.py
+++ b/src/buildservice/releasemonitoring.py
@@ -94,7 +94,7 @@ class MonitoringRelease(database.Base, database.BackendMixin):
 
        # Monitoring
 
-       monitoring = sqlalchemy.orm.relationship("Monitoring", lazy="selectin")
+       monitoring = sqlalchemy.orm.relationship("Monitoring", lazy="joined")
 
        # Version
 
@@ -559,7 +559,7 @@ class Monitoring(database.Base, database.BackendMixin, database.SoftDeleteMixin)
 
        # Distro
 
-       distro = sqlalchemy.orm.relationship("Distro", lazy="selectin")
+       distro = sqlalchemy.orm.relationship("Distro", lazy="joined")
 
        # Name
 
@@ -577,7 +577,7 @@ class Monitoring(database.Base, database.BackendMixin, database.SoftDeleteMixin)
        # Created By
 
        created_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[created_by_id], lazy="selectin",
+               "User", foreign_keys=[created_by_id], lazy="joined",
        )
 
        # Deleted By ID
@@ -587,7 +587,7 @@ class Monitoring(database.Base, database.BackendMixin, database.SoftDeleteMixin)
        # Deleted By
 
        deleted_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[deleted_by_id], lazy="selectin",
+               "User", foreign_keys=[deleted_by_id], lazy="joined",
        )
 
        # Project ID
@@ -778,7 +778,7 @@ class Monitoring(database.Base, database.BackendMixin, database.SoftDeleteMixin)
        # Latest Release
 
        latest_release = sqlalchemy.orm.relationship("MonitoringRelease",
-               order_by=MonitoringRelease.created_at.desc(), uselist=False, viewonly=True, lazy="selectin",
+               order_by=MonitoringRelease.created_at.desc(), uselist=False, viewonly=True, lazy="joined",
        )
 
        async def _release_exists(self, version):

diff --git a/src/buildservice/releases.py b/src/buildservice/releases.py
index 92aed353d448e12dede8410a827f7d4c7d9d3c4c..110fbf5927797d7dc45b422070f4ee76480feae2 100644 (file)
--- a/src/buildservice/releases.py
+++ b/src/buildservice/releases.py
@@ -50,7 +50,7 @@ class Release(database.Base, database.BackendMixin, database.SoftDeleteMixin):
 
        # Distro
 
-       distro = sqlalchemy.orm.relationship("Distro", lazy="selectin")
+       distro = sqlalchemy.orm.relationship("Distro", lazy="joined")
 
        # Name
 
@@ -72,7 +72,7 @@ class Release(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Created By
 
        created_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[created_by_id], lazy="selectin",
+               "User", foreign_keys=[created_by_id], lazy="joined",
        )
 
        # Deleted By ID
@@ -82,7 +82,7 @@ class Release(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Deleted By
 
        deleted_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[deleted_by_id], lazy="selectin",
+               "User", foreign_keys=[deleted_by_id], lazy="joined",
        )
 
        # Stable?

diff --git a/src/buildservice/repos.py b/src/buildservice/repos.py
index 3da418f37f8189bf5421734707f7b0eba4f1691c..dd04bb4e7eca09519394e130c8b5c67aed15d8dd 100644 (file)
--- a/src/buildservice/repos.py
+++ b/src/buildservice/repos.py
@@ -41,7 +41,7 @@ class RepoBuild(database.Base):
 
        # Repo
 
-       repo = sqlalchemy.orm.relationship("Repo", foreign_keys=[repo_id], lazy="selectin")
+       repo = sqlalchemy.orm.relationship("Repo", foreign_keys=[repo_id], lazy="joined")
 
        # Build ID
 
@@ -49,7 +49,7 @@ class RepoBuild(database.Base):
 
        # Build
 
-       build = sqlalchemy.orm.relationship("Build", foreign_keys=[build_id], lazy="selectin")
+       build = sqlalchemy.orm.relationship("Build", foreign_keys=[build_id], lazy="joined")
 
        # Added At
 
@@ -63,7 +63,7 @@ class RepoBuild(database.Base):
        # Added By
 
        added_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[added_by_id], lazy="selectin",
+               "User", foreign_keys=[added_by_id], lazy="joined",
        )
 
        # Removed At
@@ -77,7 +77,7 @@ class RepoBuild(database.Base):
        # Removed By
 
        removed_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[removed_by_id], lazy="selectin",
+               "User", foreign_keys=[removed_by_id], lazy="joined",
        )
 
 
@@ -215,7 +215,7 @@ class Repo(database.Base, database.BackendMixin, database.SoftDeleteMixin):
 
        distro_id = Column(Integer, ForeignKey("distributions.id"), nullable=False)
 
-       distro = sqlalchemy.orm.relationship("Distro", lazy="selectin")
+       distro = sqlalchemy.orm.relationship("Distro", lazy="joined")
 
        # Created At
 
@@ -250,7 +250,7 @@ class Repo(database.Base, database.BackendMixin, database.SoftDeleteMixin):
 
        # Owner
 
-       owner = sqlalchemy.orm.relationship("User", lazy="selectin")
+       owner = sqlalchemy.orm.relationship("User", lazy="joined")
 
        def has_perm(self, user):
                """
@@ -391,7 +391,7 @@ class Repo(database.Base, database.BackendMixin, database.SoftDeleteMixin):
 
        # Key
 
-       key = sqlalchemy.orm.relationship("Key", foreign_keys=[key_id], lazy="selectin")
+       key = sqlalchemy.orm.relationship("Key", foreign_keys=[key_id], lazy="joined")
 
        # Architectures
 

diff --git a/src/buildservice/sessions.py b/src/buildservice/sessions.py
index b4ab565f50210013a7abb0761abde3acf1de54f1..d1ba5e2e5b6180083e5f9c7e2853fc672c25a5d0 100644 (file)
--- a/src/buildservice/sessions.py
+++ b/src/buildservice/sessions.py
@@ -94,7 +94,7 @@ class Session(database.Base):
 
        user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
 
-       user = sqlalchemy.orm.relationship("User", back_populates="sessions", lazy="selectin")
+       user = sqlalchemy.orm.relationship("User", back_populates="sessions", lazy="joined")
 
        # Created At
 

diff --git a/src/buildservice/sources.py b/src/buildservice/sources.py
index cf2bfdacd9232e8775465839e63225963247ad4d..4ffabbe9b074c86eb60adfc1c5fa28ba578fd953 100644 (file)
--- a/src/buildservice/sources.py
+++ b/src/buildservice/sources.py
@@ -187,7 +187,7 @@ class Source(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Repo
 
        repo = sqlalchemy.orm.relationship(
-               "Repo", foreign_keys=[repo_id], lazy="selectin",
+               "Repo", foreign_keys=[repo_id], lazy="joined",
        )
 
        # Distro
@@ -208,7 +208,7 @@ class Source(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Created By
 
        created_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[created_by_id], lazy="selectin",
+               "User", foreign_keys=[created_by_id], lazy="joined",
        )
 
        # Deleted By ID
@@ -218,7 +218,7 @@ class Source(database.Base, database.BackendMixin, database.SoftDeleteMixin):
        # Deleted By
 
        deleted_by = sqlalchemy.orm.relationship(
-               "User", foreign_keys=[deleted_by_id], lazy="selectin",
+               "User", foreign_keys=[deleted_by_id], lazy="joined",
        )
 
        # Path
@@ -401,7 +401,7 @@ class SourceCommit(database.Base, database.BackendMixin, database.SoftDeleteMixi
        # Source
 
        source = sqlalchemy.orm.relationship(
-               "Source", foreign_keys=[source_id], lazy="selectin",
+               "Source", foreign_keys=[source_id], lazy="joined",
        )
 
        # Revision
@@ -622,7 +622,7 @@ class SourceJob(database.Base, database.BackendMixin):
        # Commit
 
        commit = sqlalchemy.orm.relationship(
-               "SourceCommit", foreign_keys=[commit_id], lazy="selectin",
+               "SourceCommit", foreign_keys=[commit_id], lazy="joined",
        )
 
        # Action

diff --git a/src/buildservice/uploads.py b/src/buildservice/uploads.py
index 9b571d1bee46514c4ad1811c74797cd698142303..a5394049d778c0960d6dd6df70e6bc107ab9a5d9 100644 (file)
--- a/src/buildservice/uploads.py
+++ b/src/buildservice/uploads.py
@@ -173,7 +173,7 @@ class Upload(database.Base, database.BackendMixin):
 
        # Builder
 
-       builder = sqlalchemy.orm.relationship("Builder", foreign_keys=[builder_id], lazy="selectin")
+       builder = sqlalchemy.orm.relationship("Builder", foreign_keys=[builder_id], lazy="joined")
 
        # User ID
 
@@ -181,7 +181,7 @@ class Upload(database.Base, database.BackendMixin):
 
        # User
 
-       user = sqlalchemy.orm.relationship("User", foreign_keys=[user_id], lazy="selectin")
+       user = sqlalchemy.orm.relationship("User", foreign_keys=[user_id], lazy="joined")
 
        # Has Perms?
 

diff --git a/src/buildservice/users.py b/src/buildservice/users.py
index 7920e6aa78e3a224d094a4bac1ffe3e32f2889ca..60eef4cc72c52f61354146084cce696fab2b8690 100644 (file)
--- a/src/buildservice/users.py
+++ b/src/buildservice/users.py
@@ -1044,7 +1044,7 @@ class UserPushSubscription(database.Base):
 
        # User
 
-       user = sqlalchemy.orm.relationship("User", lazy="selectin")
+       user = sqlalchemy.orm.relationship("User", lazy="joined")
 
        # UUID