Disable sntrup761 if compiler doesn't support VLAs.

The sntrup761 code sourced from supercop uses variable length
arrays.  Although widely supported, they are not part of the ANSI
C89 spec so if the compiler does not support VLAs, disable the
sntrup761x25519-sha512@openssh.com KEX method by replacing the kex
functions with no-op ones similar to what we do in kexecdh.c.

This should allow OpenSSH to build with a plain C89 compiler again.
Spotted by tim@, ok djm@.

diff --git a/configure.ac b/configure.ac
index 35d1aca9fc90d747bc32ceac6b84d392a10823de..0cd1025f6faca6901b5367c5ba4ce7ca1a4c32f1 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -297,6 +297,16 @@ typedef void foo(const char *, ...) __attribute__((format(printf, 1, 2)));]],
         [compiler does not accept __attribute__ on prototype args]) ]
 )
 
+AC_MSG_CHECKING([if compiler supports variable length arrays])
+AC_COMPILE_IFELSE(
+    [AC_LANG_PROGRAM([[#include <stdlib.h>]],
+    [[ int i; for (i=0; i<3; i++){int a[i]; a[i-1]=0;} exit(0); ]])],
+    [ AC_MSG_RESULT([yes])
+      AC_DEFINE(VARIABLE_LENGTH_ARRAYS, [1],
+        [compiler supports variable length arrays]) ],
+    [ AC_MSG_RESULT([no]) ]
+)
+
 if test "x$no_attrib_nonnull" != "x1" ; then
        AC_DEFINE([HAVE_ATTRIBUTE__NONNULL__], [1], [Have attribute nonnull])
 fi

diff --git a/defines.h b/defines.h
index 79dcb507feeb5ecddac31eec853b8d7dd28750fc..d6a1d014cab9d68e2da4b3c8ce1e6e7049fc5232 100644 (file)
--- a/defines.h
+++ b/defines.h
@@ -894,4 +894,11 @@ struct winsize {
 # define USE_SYSTEM_GLOB
 #endif
 
+/*
+ * sntrup761 uses variable length arrays, only enable if the compiler
+ * supports them.
+ */
+#ifdef VARIABLE_LENGTH_ARRAYS
+# define USE_SNTRUP761X25519 1
+#endif
 #endif /* _DEFINES_H */

diff --git a/kex.c b/kex.c
index f08143a5daf5de5b91b77529acaefccee9569834..3269b2c3152f5d44c57a1ef0d1590807f51d8092 100644 (file)
--- a/kex.c
+++ b/kex.c
@@ -110,8 +110,10 @@ static const struct kexalg kexalgs[] = {
 #if defined(HAVE_EVP_SHA256) || !defined(WITH_OPENSSL)
        { KEX_CURVE25519_SHA256, KEX_C25519_SHA256, 0, SSH_DIGEST_SHA256 },
        { KEX_CURVE25519_SHA256_OLD, KEX_C25519_SHA256, 0, SSH_DIGEST_SHA256 },
+#ifdef USE_SNTRUP761X25519
        { KEX_SNTRUP761X25519_SHA512, KEX_KEM_SNTRUP761X25519_SHA512, 0,
            SSH_DIGEST_SHA512 },
+#endif
 #endif /* HAVE_EVP_SHA256 || !WITH_OPENSSL */
        { NULL, 0, -1, -1},
 };

diff --git a/kexsntrup761x25519.c b/kexsntrup761x25519.c
index 3d5c6bdf04a59bde9385dcdd32821917b6884a61..e3007fa291252b17f7872e3bad0e519617030b8c 100644 (file)
--- a/kexsntrup761x25519.c
+++ b/kexsntrup761x25519.c
@@ -25,6 +25,8 @@
 
 #include "includes.h"
 
+#ifdef USE_SNTRUP761X25519
+
 #include <sys/types.h>
 
 #include <stdio.h>
@@ -217,3 +219,33 @@ kex_kem_sntrup761x25519_dec(struct kex *kex,
        sshbuf_free(buf);
        return r;
 }
+
+#else
+
+#include "ssherr.h"
+
+struct kex;
+struct sshbuf;
+struct sshkey;
+
+int
+kex_kem_sntrup761x25519_keypair(struct kex *kex)
+{
+       return SSH_ERR_SIGN_ALG_UNSUPPORTED;
+}
+
+int
+kex_kem_sntrup761x25519_enc(struct kex *kex,
+   const struct sshbuf *client_blob, struct sshbuf **server_blobp,
+   struct sshbuf **shared_secretp)
+{
+       return SSH_ERR_SIGN_ALG_UNSUPPORTED;
+}
+
+int
+kex_kem_sntrup761x25519_dec(struct kex *kex,
+    const struct sshbuf *server_blob, struct sshbuf **shared_secretp)
+{
+       return SSH_ERR_SIGN_ALG_UNSUPPORTED;
+}
+#endif /* USE_SNTRUP761X25519 */

diff --git a/sntrup761.c b/sntrup761.c
index 01f1bc3445a13a137d9b92b37e122ab03524dcda..c63e600fb5b40fc7cefec8313c57af058cf85fa3 100644 (file)
--- a/sntrup761.c
+++ b/sntrup761.c
@@ -10,6 +10,8 @@
 
 #include "includes.h"
 
+#ifdef USE_SNTRUP761X25519
+
 #include <string.h>
 #include "crypto_api.h"
 
@@ -1268,4 +1270,4 @@ int crypto_kem_sntrup761_dec(unsigned char *k,const unsigned char *c,const unsig
   Decap(k,c,sk);
   return 0;
 }
-
+#endif /* USE_SNTRUP761X25519 */