Forbid GEN_OTHERNAME SMTP UTF8 email name constraints.

RFC 9598 States:

Certificate Authorities that wish to issue CA certificates with email
address name constraints MUST use rfc822Name subject alternative names
only. These MUST be IDNA2008-conformant names with no mappings and with
non-ASCII domains encoded in A-labels only.

This appears to be to get around the confusion created if someone
attempts to encode a name constraint for an email address into the
UTF-8 version of the name

Were someone to attempt to support this, not only would you now have
to check two separate sets of name constraints for the same thing, but
would now have to decide what to do if they said different things.

So we just flag any such certficiate as invalid

Reviewed-by: Saša Nedvědický <sashan@openssl.org>
Reviewed-by: Neil Horman <nhorman@openssl.org>
MergeDate: Thu May  7 16:09:44 2026
(Merged from https://github.com/openssl/openssl/pull/30329)

diff --git a/crypto/x509/v3_purp.c b/crypto/x509/v3_purp.c
index df1f0ae2d082fad72f8d6f9a36f66b1a63d951e9..462e2f12c824f284ffd97f4d5f26df8bc7979a41 100644 (file)
--- a/crypto/x509/v3_purp.c
+++ b/crypto/x509/v3_purp.c
@@ -40,6 +40,7 @@ static int no_check_purpose(const X509_PURPOSE *xp, const X509 *x,
     int non_leaf);
 static int check_purpose_ocsp_helper(const X509_PURPOSE *xp, const X509 *x,
     int non_leaf);
+static int check_name_constraints(const NAME_CONSTRAINTS *nc);
 
 static int xp_cmp(const X509_PURPOSE *const *a, const X509_PURPOSE *const *b);
 static void xptable_free(X509_PURPOSE *p);
@@ -728,6 +729,8 @@ int ossl_x509v3_cache_extensions(const X509 *const_x)
     tmp_nc = X509_get_ext_d2i(const_x, NID_name_constraints, &i, NULL);
     if (tmp_nc == NULL && i != -1)
         tmp_ex_flags |= EXFLAG_INVALID;
+    if (!check_name_constraints(tmp_nc))
+        tmp_ex_flags |= EXFLAG_INVALID;
 
     /* Handle CRL distribution point entries */
     res = setup_crldp(const_x, &tmp_crldp);
@@ -1101,6 +1104,45 @@ static int no_check_purpose(const X509_PURPOSE *xp, const X509 *x,
     return 1;
 }
 
+static int check_name_constraints(const NAME_CONSTRAINTS *nc)
+{
+    GENERAL_SUBTREE *sub;
+    int ret = 1;
+
+    if (nc == NULL)
+        goto done;
+
+    for (int i = 0; nc->permittedSubtrees != NULL
+        && i < sk_GENERAL_SUBTREE_num(nc->permittedSubtrees);
+        i++) {
+        sub = sk_GENERAL_SUBTREE_value(nc->permittedSubtrees, i);
+        if (sub->base->type == GEN_OTHERNAME
+            && OBJ_obj2nid(sub->base->d.otherName->type_id)
+                == NID_id_on_SmtpUTF8Mailbox) {
+            /* RFC 9598 prohibits GEN_OTHERNAME email constraints */
+            ERR_raise(ERR_LIB_X509V3, X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE);
+            ret = 0;
+            goto done;
+        }
+    }
+    for (int i = 0; nc->excludedSubtrees != NULL
+        && i < sk_GENERAL_SUBTREE_num(nc->excludedSubtrees);
+        i++) {
+        sub = sk_GENERAL_SUBTREE_value(nc->excludedSubtrees, i);
+        if (sub->base->type == GEN_OTHERNAME
+            && OBJ_obj2nid(sub->base->d.otherName->type_id)
+                == NID_id_on_SmtpUTF8Mailbox) {
+            /* RFC 9598 prohibits GEN_OTHERNAME email constraints */
+            ERR_raise(ERR_LIB_X509V3, X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE);
+            ret = 0;
+            goto done;
+        }
+    }
+
+done:
+    return ret;
+}
+
 /*-
  * Various checks to see if one certificate potentially issued the second.
  * This can be used to prune a set of possible issuer certificates which