]> git.ipfire.org Git - thirdparty/gnutls.git/commitdiff
x509/name_constraints: introduce a rich comparator
authorAlexander Sosedkin <asosedkin@redhat.com>
Mon, 26 Jan 2026 19:16:36 +0000 (20:16 +0100)
committerAlexander Sosedkin <asosedkin@redhat.com>
Mon, 9 Feb 2026 11:59:26 +0000 (12:59 +0100)
These are preparatory changes before implementing N * log N intersection
over sorted lists of constraints.

Signed-off-by: Alexander Sosedkin <asosedkin@redhat.com>
lib/x509/name_constraints.c

index 81035eef8f1c1d021d6c887c1304562a2da4b723..b5d732d0c58f5091444f936cf6a7c70dad9580f8 100644 (file)
@@ -39,6 +39,9 @@
 #include "ip.h"
 #include "ip-in-cidr.h"
 #include "intprops.h"
+#include "minmax.h"
+
+#include <string.h>
 
 #define MAX_NC_CHECKS (1 << 20)
 
@@ -63,6 +66,282 @@ static struct name_constraints_node_st *
 name_constraints_node_new(gnutls_x509_name_constraints_t nc, unsigned type,
                          const unsigned char *data, unsigned int size);
 
+/* An enum for "rich" comparisons that not only let us sort name constraints,
+ * children-before-parent, but also subsume them during intersection. */
+enum name_constraint_relation {
+       NC_SORTS_BEFORE = -2, /* unrelated constraints */
+       NC_INCLUDED_BY = -1, /* nc1 is included by nc2 / children sort first */
+       NC_EQUAL = 0, /* exact match */
+       NC_INCLUDES = 1, /* nc1 includes nc2 / parents sort last */
+       NC_SORTS_AFTER = 2 /* unrelated constraints */
+};
+
+/* A helper to compare just a pair of strings with this rich comparison */
+static enum name_constraint_relation
+compare_strings(const void *n1, size_t n1_len, const void *n2, size_t n2_len)
+{
+       int r = memcmp(n1, n2, MIN(n1_len, n2_len));
+       if (r < 0)
+               return NC_SORTS_BEFORE;
+       if (r > 0)
+               return NC_SORTS_AFTER;
+       if (n1_len < n2_len)
+               return NC_SORTS_BEFORE;
+       if (n1_len > n2_len)
+               return NC_SORTS_AFTER;
+       return NC_EQUAL;
+}
+
+/* Rich-compare DNS names. Example order/relationships:
+ * z.x.a INCLUDED_BY x.a BEFORE y.a INCLUDED_BY a BEFORE x.b BEFORE y.b */
+static enum name_constraint_relation compare_dns_names(const gnutls_datum_t *n1,
+                                                      const gnutls_datum_t *n2)
+{
+       enum name_constraint_relation rel;
+       unsigned int i, j, i_end, j_end;
+
+       /* start from the end of each name */
+       i = i_end = n1->size;
+       j = j_end = n2->size;
+
+       /* skip the trailing dots for the comparison */
+       while (i && n1->data[i - 1] == '.')
+               i_end = i = i - 1;
+       while (j && n2->data[j - 1] == '.')
+               j_end = j = j - 1;
+
+       while (1) {
+               // rewind back to beginning or an after-dot position
+               while (i && n1->data[i - 1] != '.')
+                       i--;
+               while (j && n2->data[j - 1] != '.')
+                       j--;
+
+               rel = compare_strings(&n1->data[i], i_end - i, &n2->data[j],
+                                     j_end - j);
+               if (rel == NC_SORTS_BEFORE) /* x.a BEFORE y.a */
+                       return NC_SORTS_BEFORE;
+               if (rel == NC_SORTS_AFTER) /* y.a AFTER x.a */
+                       return NC_SORTS_AFTER;
+               if (!i && j) /* x.a INCLUDES z.x.a */
+                       return NC_INCLUDES;
+               if (i && !j) /* z.x.a INCLUDED_BY x.a */
+                       return NC_INCLUDED_BY;
+
+               if (!i && !j) /* r == 0, we ran out of components to compare */
+                       return NC_EQUAL;
+               /* r == 0, i && j: step back past a dot and keep comparing */
+               i_end = i = i - 1;
+               j_end = j = j - 1;
+
+               /* support for non-standard ".gr INCLUDES example.gr" [1] */
+               if (!i && j) /* .a INCLUDES x.a */
+                       return NC_INCLUDES;
+               if (i && !j) /* x.a INCLUDED_BY .a */
+                       return NC_INCLUDED_BY;
+       }
+}
+/* [1] https://mailarchive.ietf.org/arch/msg/saag/Bw6PtreW0G7aEG7SikfzKHES4VA */
+
+/* Rich-compare email name constraints. Example order/relationships:
+ * z@x.a INCLUDED_BY x.a BEFORE y.a INCLUDED_BY a BEFORE x@b BEFORE y@b */
+static enum name_constraint_relation compare_emails(const gnutls_datum_t *n1,
+                                                   const gnutls_datum_t *n2)
+{
+       enum name_constraint_relation domains_rel;
+       unsigned int i, j, i_end, j_end;
+       gnutls_datum_t d1, d2; /* borrow from n1 and n2 */
+
+       /* start from the end of each name */
+       i = i_end = n1->size;
+       j = j_end = n2->size;
+
+       /* rewind to @s to look for domains */
+       while (i && n1->data[i - 1] != '@')
+               i--;
+       d1.size = i_end - i;
+       d1.data = &n1->data[i];
+       while (j && n2->data[j - 1] != '@')
+               j--;
+       d2.size = j_end - j;
+       d2.data = &n2->data[j];
+
+       domains_rel = compare_dns_names(&d1, &d2);
+
+       /* email constraint semantics differ from DNS
+        * DNS: x.a INCLUDED_BY a
+        * Email: x.a INCLUDED_BY .a BEFORE a */
+       if (domains_rel == NC_INCLUDED_BY || domains_rel == NC_INCLUDES) {
+               bool d1_has_dot = (d1.size > 0 && d1.data[0] == '.');
+               bool d2_has_dot = (d2.size > 0 && d2.data[0] == '.');
+               /* a constraint without a dot is exact, excluding subdomains */
+               if (!d2_has_dot && domains_rel == NC_INCLUDED_BY)
+                       domains_rel = NC_SORTS_BEFORE; /* x.a BEFORE a */
+               if (!d1_has_dot && domains_rel == NC_INCLUDES)
+                       domains_rel = NC_SORTS_AFTER; /* a AFTER x.a */
+       }
+
+       if (!i && !j) { /* both are domains-only */
+               return domains_rel;
+       } else if (i && !j) { /* n1 is email, n2 is domain */
+               switch (domains_rel) {
+               case NC_SORTS_AFTER:
+                       return NC_SORTS_AFTER;
+               case NC_SORTS_BEFORE:
+                       return NC_SORTS_BEFORE;
+               case NC_INCLUDES: /* n2 is more specific, a@x.a AFTER z.x.a */
+                       return NC_SORTS_AFTER;
+               case NC_EQUAL: /* subdomains match, z@x.a INCLUDED_BY x.a */
+               case NC_INCLUDED_BY: /* n1 is more specific */
+                       return NC_INCLUDED_BY;
+               }
+       } else if (!i && j) { /* n1 is domain, n2 is email */
+               switch (domains_rel) {
+               case NC_SORTS_AFTER:
+                       return NC_SORTS_AFTER;
+               case NC_SORTS_BEFORE:
+                       return NC_SORTS_BEFORE;
+               case NC_INCLUDES: /* n2 is more specific, a AFTER z@x.a */
+                       return NC_SORTS_AFTER;
+               case NC_EQUAL: /* subdomains match, x.a INCLUDES z@x.a */
+                       return NC_INCLUDES;
+               case NC_INCLUDED_BY: /* n1 is more specific, x.a BEFORE z@a */
+                       return NC_SORTS_BEFORE;
+               }
+       } else if (i && j) { /* both are emails */
+               switch (domains_rel) {
+               case NC_SORTS_AFTER:
+                       return NC_SORTS_AFTER;
+               case NC_SORTS_BEFORE:
+                       return NC_SORTS_BEFORE;
+               case NC_INCLUDES: // n2 is more specific
+                       return NC_SORTS_AFTER;
+               case NC_INCLUDED_BY: // n1 is more specific
+                       return NC_SORTS_BEFORE;
+               case NC_EQUAL: // only case when we need to look before the @
+                       break; // see below for readability
+               }
+       }
+
+       /* i && j, both are emails, domain names match, compare up to @ */
+       return compare_strings(n1->data, i - 1, n2->data, j - 1);
+}
+
+/* Rich-compare IP address constraints. Example order/relationships:
+ * 10.0.0.0/24 INCLUDED_BY 10.0.0.0/16 BEFORE 1::1/128 INCLUDED_BY 1::1/127 */
+static enum name_constraint_relation compare_ip_ncs(const gnutls_datum_t *n1,
+                                                   const gnutls_datum_t *n2)
+{
+       unsigned int len, i;
+       int r;
+       const unsigned char *ip1, *ip2, *mask1, *mask2;
+       unsigned char masked11[16], masked22[16], masked12[16], masked21[16];
+
+       if (n1->size < n2->size)
+               return NC_SORTS_BEFORE;
+       if (n1->size > n2->size)
+               return NC_SORTS_AFTER;
+       len = n1->size / 2; /* 4 for IPv4, 16 for IPv6 */
+
+       /* data is a concatenation of prefix and mask */
+       ip1 = n1->data;
+       ip2 = n2->data;
+       mask1 = n1->data + len;
+       mask2 = n2->data + len;
+       for (i = 0; i < len; i++) {
+               masked11[i] = ip1[i] & mask1[i];
+               masked22[i] = ip2[i] & mask2[i];
+               masked12[i] = ip1[i] & mask2[i];
+               masked21[i] = ip2[i] & mask1[i];
+       }
+
+       r = memcmp(mask1, mask2, len);
+       if (r < 0 && !memcmp(masked11, masked21, len)) /* prefix1 < prefix2 */
+               return NC_INCLUDES; /* ip1 & mask1 == ip2 & mask1 */
+       if (r > 0 && !memcmp(masked12, masked22, len)) /* prefix1 > prefix2 */
+               return NC_INCLUDED_BY; /* ip1 & mask2 == ip2 & mask2 */
+
+       r = memcmp(masked11, masked22, len);
+       if (r < 0)
+               return NC_SORTS_BEFORE;
+       else if (r > 0)
+               return NC_SORTS_AFTER;
+       return NC_EQUAL;
+}
+
+static inline bool is_supported_type(unsigned type)
+{
+       return type == GNUTLS_SAN_DNSNAME || type == GNUTLS_SAN_RFC822NAME ||
+              type == GNUTLS_SAN_IPADDRESS;
+}
+
+/* Universal comparison for name constraint nodes.
+ * Unsupported types sort before supported types to allow early handling.
+ * NULL represents end-of-list and sorts after everything else. */
+static enum name_constraint_relation
+compare_name_constraint_nodes(const struct name_constraints_node_st *n1,
+                             const struct name_constraints_node_st *n2)
+{
+       bool n1_supported, n2_supported;
+
+       if (!n1 && !n2)
+               return NC_EQUAL;
+       if (!n1)
+               return NC_SORTS_AFTER;
+       if (!n2)
+               return NC_SORTS_BEFORE;
+
+       n1_supported = is_supported_type(n1->type);
+       n2_supported = is_supported_type(n2->type);
+
+       /* unsupported types bubble up (sort first). intersect relies on this */
+       if (!n1_supported && n2_supported)
+               return NC_SORTS_BEFORE;
+       if (n1_supported && !n2_supported)
+               return NC_SORTS_AFTER;
+
+       /* next, sort by type */
+       if (n1->type < n2->type)
+               return NC_SORTS_BEFORE;
+       if (n1->type > n2->type)
+               return NC_SORTS_AFTER;
+
+       /* now look deeper */
+       switch (n1->type) {
+       case GNUTLS_SAN_DNSNAME:
+               return compare_dns_names(&n1->name, &n2->name);
+       case GNUTLS_SAN_RFC822NAME:
+               return compare_emails(&n1->name, &n2->name);
+       case GNUTLS_SAN_IPADDRESS:
+               return compare_ip_ncs(&n1->name, &n2->name);
+       default:
+               /* unsupported types: stable lexicographic order */
+               return compare_strings(n1->name.data, n1->name.size,
+                                      n2->name.data, n2->name.size);
+       }
+}
+
+/* qsort-compatible wrapper */
+static int compare_name_constraint_nodes_qsort(const void *a, const void *b)
+{
+       const struct name_constraints_node_st *const *n1 = a;
+       const struct name_constraints_node_st *const *n2 = b;
+       enum name_constraint_relation rel;
+
+       rel = compare_name_constraint_nodes(*n1, *n2);
+       switch (rel) {
+       case NC_SORTS_BEFORE:
+       case NC_INCLUDED_BY:
+               return -1;
+       case NC_SORTS_AFTER:
+       case NC_INCLUDES:
+               return 1;
+       case NC_EQUAL:
+       default:
+               return 0;
+       }
+}
+
 static int
 name_constraints_node_list_add(struct name_constraints_node_list_st *list,
                               struct name_constraints_node_st *node)
@@ -420,9 +699,7 @@ static int name_constraints_node_list_intersect(
                        }
                }
 
-               if (found != NULL && (t->type == GNUTLS_SAN_DNSNAME ||
-                                     t->type == GNUTLS_SAN_RFC822NAME ||
-                                     t->type == GNUTLS_SAN_IPADDRESS)) {
+               if (found != NULL && is_supported_type(t->type)) {
                        /* move node from PERMITTED to REMOVED */
                        ret = name_constraints_node_list_add(&removed, t);
                        if (ret < 0) {
@@ -827,61 +1104,14 @@ cleanup:
        return ret;
 }
 
-static unsigned ends_with(const gnutls_datum_t *str,
-                         const gnutls_datum_t *suffix)
-{
-       unsigned char *tree;
-       unsigned int treelen;
-
-       if (suffix->size >= str->size)
-               return 0;
-
-       tree = suffix->data;
-       treelen = suffix->size;
-       if ((treelen > 0) && (tree[0] == '.')) {
-               tree++;
-               treelen--;
-       }
-
-       if (memcmp(str->data + str->size - treelen, tree, treelen) == 0 &&
-           str->data[str->size - treelen - 1] == '.')
-               return 1; /* match */
-
-       return 0;
-}
-
-static unsigned email_ends_with(const gnutls_datum_t *str,
-                               const gnutls_datum_t *suffix)
-{
-       if (suffix->size >= str->size) {
-               return 0;
-       }
-
-       if (suffix->size > 0 && memcmp(str->data + str->size - suffix->size,
-                                      suffix->data, suffix->size) != 0) {
-               return 0;
-       }
-
-       if (suffix->size > 1 && suffix->data[0] == '.') { /* .domain.com */
-               return 1; /* match */
-       } else if (str->data[str->size - suffix->size - 1] == '@') {
-               return 1; /* match */
-       }
-
-       return 0;
-}
-
 static unsigned dnsname_matches(const gnutls_datum_t *name,
                                const gnutls_datum_t *suffix)
 {
        _gnutls_hard_log("matching %.*s with DNS constraint %.*s\n", name->size,
                         name->data, suffix->size, suffix->data);
 
-       if (suffix->size == name->size &&
-           memcmp(suffix->data, name->data, suffix->size) == 0)
-               return 1; /* match */
-
-       return ends_with(name, suffix);
+       enum name_constraint_relation rel = compare_dns_names(name, suffix);
+       return rel == NC_EQUAL || rel == NC_INCLUDED_BY;
 }
 
 static unsigned email_matches(const gnutls_datum_t *name,
@@ -890,11 +1120,8 @@ static unsigned email_matches(const gnutls_datum_t *name,
        _gnutls_hard_log("matching %.*s with e-mail constraint %.*s\n",
                         name->size, name->data, suffix->size, suffix->data);
 
-       if (suffix->size == name->size &&
-           memcmp(suffix->data, name->data, suffix->size) == 0)
-               return 1; /* match */
-
-       return email_ends_with(name, suffix);
+       enum name_constraint_relation rel = compare_emails(name, suffix);
+       return rel == NC_EQUAL || rel == NC_INCLUDED_BY;
 }
 
 /*-
@@ -918,8 +1145,7 @@ static int name_constraints_intersect_nodes(
        // presume empty intersection
        struct name_constraints_node_st *intersection = NULL;
        const struct name_constraints_node_st *to_copy = NULL;
-       unsigned iplength = 0;
-       unsigned byte;
+       enum name_constraint_relation rel;
 
        *_intersection = NULL;
 
@@ -928,32 +1154,49 @@ static int name_constraints_intersect_nodes(
        }
        switch (node1->type) {
        case GNUTLS_SAN_DNSNAME:
-               if (!dnsname_matches(&node2->name, &node1->name))
+               rel = compare_dns_names(&node1->name, &node2->name);
+               switch (rel) {
+               case NC_EQUAL: // equal means doesn't matter which one
+               case NC_INCLUDES: // node2 is more specific
+                       to_copy = node2;
+                       break;
+               case NC_INCLUDED_BY: // node1 is more specific
+                       to_copy = node1;
+                       break;
+               case NC_SORTS_BEFORE: // no intersection
+               case NC_SORTS_AFTER: // no intersection
                        return GNUTLS_E_SUCCESS;
-               to_copy = node2;
+               }
                break;
        case GNUTLS_SAN_RFC822NAME:
-               if (!email_matches(&node2->name, &node1->name))
+               rel = compare_emails(&node1->name, &node2->name);
+               switch (rel) {
+               case NC_EQUAL: // equal means doesn't matter which one
+               case NC_INCLUDES: // node2 is more specific
+                       to_copy = node2;
+                       break;
+               case NC_INCLUDED_BY: // node1 is more specific
+                       to_copy = node1;
+                       break;
+               case NC_SORTS_BEFORE: // no intersection
+               case NC_SORTS_AFTER: // no intersection
                        return GNUTLS_E_SUCCESS;
-               to_copy = node2;
+               }
                break;
        case GNUTLS_SAN_IPADDRESS:
-               if (node1->name.size != node2->name.size)
+               rel = compare_ip_ncs(&node1->name, &node2->name);
+               switch (rel) {
+               case NC_EQUAL: // equal means doesn't matter which one
+               case NC_INCLUDES: // node2 is more specific
+                       to_copy = node2;
+                       break;
+               case NC_INCLUDED_BY: // node1 is more specific
+                       to_copy = node1;
+                       break;
+               case NC_SORTS_BEFORE: // no intersection
+               case NC_SORTS_AFTER: // no intersection
                        return GNUTLS_E_SUCCESS;
-               iplength = node1->name.size / 2;
-               for (byte = 0; byte < iplength; byte++) {
-                       if (((node1->name.data[byte] ^
-                             node2->name.data[byte]) // XOR of addresses
-                            & node1->name.data[byte +
-                                               iplength] // AND mask from nc1
-                            & node2->name.data[byte +
-                                               iplength]) // AND mask from nc2
-                           != 0) {
-                               // CIDRS do not intersect
-                               return GNUTLS_E_SUCCESS;
-                       }
                }
-               to_copy = node2;
                break;
        default:
                // for other types, we don't know how to do the intersection, assume empty
@@ -970,20 +1213,6 @@ static int name_constraints_intersect_nodes(
                intersection = *_intersection;
 
                assert(intersection->name.data != NULL);
-
-               if (intersection->type == GNUTLS_SAN_IPADDRESS) {
-                       // make sure both IP addresses are correctly masked
-                       _gnutls_mask_ip(intersection->name.data,
-                                       intersection->name.data + iplength,
-                                       iplength);
-                       _gnutls_mask_ip(node1->name.data,
-                                       node1->name.data + iplength, iplength);
-                       // update intersection, if necessary (we already know one is subset of other)
-                       for (byte = 0; byte < 2 * iplength; byte++) {
-                               intersection->name.data[byte] |=
-                                       node1->name.data[byte];
-                       }
-               }
        }
 
        return GNUTLS_E_SUCCESS;