Implement an additional flag for hostile networks safe to drop

This patch implements an additional flag intended for networks and
Autonomous Systems being considered hostile. While libloc does not and
should not be an opinionated database, reality shows it is being used
this way.

Hereby, we assign "XD" (drop) as a custom country code for networks
being flagged this way. According to ISO, "XA" to "XZ" are reserved for
"user-assgined codes" (https://www.iso.org/glossary-for-iso-3166.html),
so this is a safe thing to do.

This patch does not interfere with "A1" to "A3", which we currently
assign outside standardised country code ranges for historical reasons.

Neither does it specify any policy or source for tagging networks with a
"drop" flag. Doing so is beyond the scope of this - technical -
approach.

To avoid confusions with the SQL "DROP" command, "is_drop" will be used
as a column name for database operations.

Thanks to Michael for his remarks and ideas during the run-up.

Cc: Michael Tremer <michael.tremer@ipfire.org>
Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/man/location.txt b/man/location.txt
index df1da53405ad787e26e3fb2da9f47f8aa01d2cad..b38f21cf1ab8d7e5f8ff7bd445fd155d7b2a82f5 100644 (file)
--- a/man/location.txt
+++ b/man/location.txt
@@ -10,7 +10,7 @@ location - Query the location database
 `location list-countries [--show-name] [--show-continent]`
 `location list-networks-by-as ASN`
 `location list-networks-by-cc COUNTRY_CODE`
-`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
+`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast|--drop]`
 `location lookup ADDRESS [ADDRESS...]`
 `location search-as STRING`
 `location update [--cron=daily|weekly|monthly]`
@@ -73,7 +73,7 @@ or countries.
        +
        See above for usage of the '--family' and '--format' parameters.
 
-'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast]'::
+'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast|--drop]'::
        Lists all networks that have a certain flag.
        +
        See above for usage of the '--family' and '--format' parameters.

diff --git a/po/de.po b/po/de.po
index 3b073d6e3e3fd1784389033d91b5d60a7d3a4e00..3cbcdd708637411b2ac76ea009dc39ddc9917ce5 100644 (file)
--- a/po/de.po
+++ b/po/de.po
@@ -152,6 +152,9 @@ msgstr ""
 msgid "Anycasts"
 msgstr ""
 
+msgid "Hostile Networks safe to drop"
+msgstr ""
+
 msgid "Lists all countries"
 msgstr ""
 

diff --git a/src/loc/network.h b/src/loc/network.h
index af3dafd5561939a7d16ac23acae7044ea8ec049e..a30f65347d4c7143550667c75c02b30c66f79374 100644 (file)
--- a/src/loc/network.h
+++ b/src/loc/network.h
@@ -1,7 +1,7 @@
 /*
        libloc - A library to determine the location of someone on the Internet
 
-       Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
+       Copyright (C) 2017-2021 IPFire Development Team <info@ipfire.org>
 
        This library is free software; you can redistribute it and/or
        modify it under the terms of the GNU Lesser General Public
@@ -27,6 +27,7 @@ enum loc_network_flags {
        LOC_NETWORK_FLAG_ANONYMOUS_PROXY    = (1 << 0), // A1
        LOC_NETWORK_FLAG_SATELLITE_PROVIDER = (1 << 1), // A2
        LOC_NETWORK_FLAG_ANYCAST            = (1 << 2), // A3
+       LOC_NETWORK_FLAG_DROP               = (1 << 3), // XD
 };
 
 struct loc_network;

diff --git a/src/perl/Location.xs b/src/perl/Location.xs
index b7676d2667f07b9ff5c7ec53b34c6b6e880037cd..73f85b45ce586f5136b0738f33afa61e95cd5f67 100644 (file)
--- a/src/perl/Location.xs
+++ b/src/perl/Location.xs
@@ -198,6 +198,8 @@ lookup_network_has_flag(db, address, flag)
                        iv |= LOC_NETWORK_FLAG_SATELLITE_PROVIDER;
                else if (strcmp("LOC_NETWORK_FLAG_ANYCAST", flag) == 0)
                        iv |= LOC_NETWORK_FLAG_ANYCAST;
+               else if (strcmp("LOC_NETWORK_FLAG_DROP", flag) == 0)
+                       iv |= LOC_NETWORK_FLAG_DROP;
                else
                        croak("Invalid flag");
 

diff --git a/src/python/export.py b/src/python/export.py
index f0eae2644fc0a8411ae70c28a5666cb91f6fceeb..3b9e1e0262634a0fded1c43a2b7f0dce6ed8fe12 100644 (file)
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -3,7 +3,7 @@
 #                                                                             #
 # libloc - A library to determine the location of someone on the Internet     #
 #                                                                             #
-# Copyright (C) 2020 IPFire Development Team <info@ipfire.org>                #
+# Copyright (C) 2020-2021 IPFire Development Team <info@ipfire.org>           #
 #                                                                             #
 # This library is free software; you can redistribute it and/or               #
 # modify it under the terms of the GNU Lesser General Public                  #
@@ -33,6 +33,7 @@ FLAGS = {
        _location.NETWORK_FLAG_ANONYMOUS_PROXY    : "A1",
        _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
        _location.NETWORK_FLAG_ANYCAST            : "A3",
+       _location.NETWORK_FLAG_DROP               : "XD",
 }
 
 class OutputWriter(object):

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index 3e1f859cb30af6854fe86cc9deb360ad186c005f..aa3b8f77879287df86c0f4056e08e8c0a4b6a260 100644 (file)
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -183,6 +183,7 @@ class CLI(object):
                                );
                                CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
                                        ON autnum_overrides(number);
+                               ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
 
                                CREATE TABLE IF NOT EXISTS network_overrides(
                                        network inet NOT NULL,
@@ -195,6 +196,7 @@ class CLI(object):
                                        ON network_overrides(network);
                                CREATE INDEX IF NOT EXISTS network_overrides_search
                                        ON network_overrides USING GIST(network inet_ops);
+                               ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
                        """)
 
                return db
@@ -302,7 +304,20 @@ class CLI(object):
                                                        WHERE networks.autnum = overrides.number
                                        ),
                                        FALSE
-                               ) AS is_anycast
+                               ) AS is_anycast,
+                               COALESCE(
+                                       (
+                                               SELECT is_drop FROM network_overrides overrides
+                                                       WHERE networks.network <<= overrides.network
+                                                       ORDER BY masklen(overrides.network) DESC
+                                                       LIMIT 1
+                                       ),
+                                       (
+                                               SELECT is_drop FROM autnum_overrides overrides
+                                                       WHERE networks.autnum = overrides.number
+                                       ),
+                                       FALSE
+                               ) AS is_drop
                        FROM (
                                SELECT
                                        known_networks.network AS network,
@@ -351,6 +366,9 @@ class CLI(object):
                        if row.is_anycast:
                                network.set_flag(location.NETWORK_FLAG_ANYCAST)
 
+                       if row.is_drop:
+                               network.set_flag(location.NETWORK_FLAG_DROP)
+
                # Add all countries
                log.info("Writing countries...")
                rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
@@ -981,14 +999,16 @@ class CLI(object):
                                                                        country,
                                                                        is_anonymous_proxy,
                                                                        is_satellite_provider,
-                                                                       is_anycast
-                                                               ) VALUES (%s, %s, %s, %s, %s)
+                                                                       is_anycast,
+                                                                       is_drop
+                                                               ) VALUES (%s, %s, %s, %s, %s, %s)
                                                                ON CONFLICT (network) DO NOTHING""",
                                                                "%s" % network,
                                                                block.get("country"),
                                                                self._parse_bool(block, "is-anonymous-proxy"),
                                                                self._parse_bool(block, "is-satellite-provider"),
                                                                self._parse_bool(block, "is-anycast"),
+                                                               self._parse_bool(block, "drop"),
                                                        )
 
                                                elif type == "aut-num":
@@ -1009,8 +1029,9 @@ class CLI(object):
                                                                        country,
                                                                        is_anonymous_proxy,
                                                                        is_satellite_provider,
-                                                                       is_anycast
-                                                               ) VALUES(%s, %s, %s, %s, %s, %s)
+                                                                       is_anycast,
+                                                                       is_drop
+                                                               ) VALUES(%s, %s, %s, %s, %s, %s, %s)
                                                                ON CONFLICT DO NOTHING""",
                                                                autnum,
                                                                block.get("name"),
@@ -1018,6 +1039,7 @@ class CLI(object):
                                                                self._parse_bool(block, "is-anonymous-proxy"),
                                                                self._parse_bool(block, "is-satellite-provider"),
                                                                self._parse_bool(block, "is-anycast"),
+                                                               self._parse_bool(block, "drop"),
                                                        )
 
                                                else:

diff --git a/src/python/location.in b/src/python/location.in
index e02b4e891d3b0844758267ce7d686369f696d8a4..0c89d75263370952259255a5470aa4884685b1b1 100644 (file)
--- a/src/python/location.in
+++ b/src/python/location.in
@@ -3,7 +3,7 @@
 #                                                                             #
 # libloc - A library to determine the location of someone on the Internet     #
 #                                                                             #
-# Copyright (C) 2017 IPFire Development Team <info@ipfire.org>                #
+# Copyright (C) 2017-2021 IPFire Development Team <info@ipfire.org>           #
 #                                                                             #
 # This library is free software; you can redistribute it and/or               #
 # modify it under the terms of the GNU Lesser General Public                  #
@@ -146,6 +146,9 @@ class CLI(object):
                list_networks_by_flags.add_argument("--anycast",
                        action="store_true", help=_("Anycasts"),
                )
+               list_networks_by_flags.add_argument("--drop",
+                       action="store_true", help=_("Hostile Networks safe to drop"),
+               )
                list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
                list_networks_by_flags.add_argument("--format",
                        choices=location.export.formats.keys(), default="list")
@@ -305,6 +308,12 @@ class CLI(object):
                                        _("Anycast"), _("yes"),
                                ))
 
+                       # Hostile Network
+                       if network.has_flag(location.NETWORK_FLAG_DROP):
+                               print(format % (
+                                       _("Hostile Network safe to drop"), _("yes"),
+                               ))
+
                return ret
 
        def handle_dump(self, db, ns):
@@ -346,6 +355,7 @@ class CLI(object):
                        location.NETWORK_FLAG_ANONYMOUS_PROXY    : "is-anonymous-proxy:",
                        location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
                        location.NETWORK_FLAG_ANYCAST            : "is-anycast:",
+                       location.NETWORK_FLAG_DROP               : "drop:",
                }
 
                # Iterate over all networks
@@ -523,6 +533,9 @@ class CLI(object):
                if ns.anycast:
                        flags |= location.NETWORK_FLAG_ANYCAST
 
+               if ns.drop:
+                       flags |= location.NETWORK_FLAG_DROP
+
                if not flags:
                        raise ValueError(_("You must at least pass one flag"))
 
@@ -551,7 +564,7 @@ class CLI(object):
                                asns.append(object)
 
                        elif location.country_code_is_valid(object) \
-                                       or object in ("A1", "A2", "A3"):
+                                       or object in ("A1", "A2", "A3", "XD"):
                                countries.append(object)
 
                        else:
@@ -560,7 +573,7 @@ class CLI(object):
 
                # Default to exporting all countries
                if not countries and not asns:
-                       countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
+                       countries = ["A1", "A2", "A3", "XD"] + [country.code for country in db.countries]
 
                # Select the output format
                writer = self.__get_output_formatter(ns)

diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c
index 5b72be9b0f9fbfe1df62dc00064d52691a51d1bd..5dd4ec645ccf3854950906980cd7c46db3f66946 100644 (file)
--- a/src/python/locationmodule.c
+++ b/src/python/locationmodule.c
@@ -1,7 +1,7 @@
 /*
        libloc - A library to determine the location of someone on the Internet
 
-       Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
+       Copyright (C) 2017-2021 IPFire Development Team <info@ipfire.org>
 
        This library is free software; you can redistribute it and/or
        modify it under the terms of the GNU Lesser General Public
@@ -169,6 +169,9 @@ PyMODINIT_FUNC PyInit__location(void) {
        if (PyModule_AddIntConstant(m, "NETWORK_FLAG_ANYCAST", LOC_NETWORK_FLAG_ANYCAST))
                return NULL;
 
+       if (PyModule_AddIntConstant(m, "NETWORK_FLAG_DROP", LOC_NETWORK_FLAG_DROP))
+               return NULL;
+
        // Add latest database version
        if (PyModule_AddIntConstant(m, "DATABASE_VERSION_LATEST", LOC_DATABASE_VERSION_LATEST))
                return NULL;