lists: Implement showing false-positives

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/dbl/lists.py b/src/dbl/lists.py
index cbe7308f84123b8de772b20c5c8c6054647e3904..787f77045c1966ff0e4a278a27a28a0ad330136d 100644 (file)
--- a/src/dbl/lists.py
+++ b/src/dbl/lists.py
@@ -906,6 +906,12 @@ class List(sqlmodel.SQLModel, database.BackendMixin, table=True):
                if update_stats:
                        await self.update_stats()
 
+       async def get_false_positives(self):
+               """
+                       Returns a list of all false-positives
+               """
+               return { source : await source.get_false_positives() for source in self.sources }
+
 
 class ListStats(sqlmodel.SQLModel, table=True):
        __tablename__ = "list_stats"

diff --git a/src/dbl/sources.py b/src/dbl/sources.py
index 199c771b1c0b86749dabfb4e90bf9be0938d0835..8f9bfb4af92d87a51ee1d4f0e64ff5b3a2d94456 100644 (file)
--- a/src/dbl/sources.py
+++ b/src/dbl/sources.py
@@ -689,6 +689,21 @@ class Source(sqlmodel.SQLModel, database.BackendMixin, table=True):
                # Store the total number of dead domains
                self.dead_domains = await self.backend.db.fetch_one(stmt)
 
+               # Store the number of false positives
+               self.false_positives = len(
+                       await self.get_false_positives(),
+               )
+
+               # Store the stats history
+               await self.backend.db.insert(
+                       SourceStats,
+                       source          = self,
+                       total_domains   = self.total_domains,
+                       dead_domains    = self.dead_domains,
+                       false_positives = self.false_positives,
+               )
+
+       async def get_false_positives(self):
                whitelisted_domains = (
                        sqlmodel
                        .select(
@@ -707,14 +722,12 @@ class Source(sqlmodel.SQLModel, database.BackendMixin, table=True):
                        .cte("whitelisted_domains")
                )
 
-               stmt = (
+               return await self.backend.db.fetch_as_set(
                        sqlmodel
                        .select(
-                               sqlmodel.func.count(),
-                       )
-                       .select_from(
-                               domains.Domain,
+                               domains.Domain.name,
                        )
+                       .distinct()
                        .where(
                                # Domains must be from this source
                                domains.Domain.source == self,
@@ -739,18 +752,6 @@ class Source(sqlmodel.SQLModel, database.BackendMixin, table=True):
                        )
                )
 
-               # Store the number of false positives
-               self.false_positives = await self.backend.db.fetch_one(stmt)
-
-               # Store the stats history
-               await self.backend.db.insert(
-                       SourceStats,
-                       source          = self,
-                       total_domains   = self.total_domains,
-                       dead_domains    = self.dead_domains,
-                       false_positives = self.false_positives,
-               )
-
 
 class SourceStats(sqlmodel.SQLModel, table=True):
        __tablename__ = "source_stats"

diff --git a/src/scripts/dbl.in b/src/scripts/dbl.in
index 6e2b0aa209b3d89a7d951240b1bcea0741edb618..1a9b4a6c471823bb21d38c2a8701a8f4a1cc86c3 100644 (file)
--- a/src/scripts/dbl.in
+++ b/src/scripts/dbl.in
@@ -213,6 +213,13 @@ class CLI(object):
                                help=_("Notifies moderators about any pending reports"))
                notify.set_defaults(func=self.__notify)
 
+               # False Positives
+               false_positives = subparsers.add_parser("false-positives",
+                       help=_("Shows a list of false-positives for the given list"),
+               )
+               false_positives.add_argument("list", help=_("The name of the list"))
+               false_positives.set_defaults(func=self.__false_positives)
+
                # Parse all arguments
                args = parser.parse_args()
 
@@ -619,6 +626,24 @@ class CLI(object):
                """
                await backend.reports.notify()
 
+       # False Positives
+
+       async def __false_positives(self, backend, args):
+               # Fetch the list
+               list = await self.__get_list(backend, args.list)
+
+               # Fetch all false-positives
+               sources = await list.get_false_positives()
+
+               for source, fps in sorted(sources.items()):
+                       if not fps:
+                               continue
+
+                       print(source)
+
+                       for name in fps:
+                               print ("        * %s" % name)
+
 
 def main():
        c = CLI()