[thirdparty/openssl.git] / test / handshake_helper.c

/*
 * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
 *
 * Licensed under the OpenSSL license (the "License").  You may not use
 * this file except in compliance with the License.  You can obtain a copy
 * in the file LICENSE in the source distribution or at
 * https://www.openssl.org/source/license.html
 */

#include <string.h>

#include <openssl/bio.h>
#include <openssl/x509_vfy.h>
#include <openssl/ssl.h>

#include "handshake_helper.h"

/*
 * Since there appears to be no way to extract the sent/received alert
 * from the SSL object directly, we use the info callback and stash
 * the result in ex_data.
 */
typedef struct handshake_ex_data {
    int alert_sent;
    int alert_received;
    int session_ticket_do_not_call;
} HANDSHAKE_EX_DATA;

static int ex_data_idx;

static void info_callback(const SSL *s, int where, int ret)
{
    if (where & SSL_CB_ALERT) {
        HANDSHAKE_EX_DATA *ex_data =
            (HANDSHAKE_EX_DATA*)(SSL_get_ex_data(s, ex_data_idx));
        if (where & SSL_CB_WRITE) {
            ex_data->alert_sent = ret;
        } else {
            ex_data->alert_received = ret;
        }
    }
}

static int verify_reject_callback(X509_STORE_CTX *ctx, void *arg) {
    X509_STORE_CTX_set_error(ctx, X509_V_ERR_APPLICATION_VERIFICATION);
    return 0;
}

static int verify_accept_callback(X509_STORE_CTX *ctx, void *arg) {
    return 1;
}

static int broken_session_ticket_callback(SSL* s, unsigned char* key_name, unsigned char *iv,
                                          EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc)
{
    return 0;
}

int do_not_call_session_ticket_callback(SSL* s, unsigned char* key_name, unsigned char *iv,
                                        EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc)
{
    HANDSHAKE_EX_DATA *ex_data =
        (HANDSHAKE_EX_DATA*)(SSL_get_ex_data(s, ex_data_idx));
    ex_data->session_ticket_do_not_call = 1;
    return 0;
}

/*
 * Configure callbacks and other properties that can't be set directly
 * in the server/client CONF.
 */
static void configure_handshake_ctx(SSL_CTX *server_ctx, SSL_CTX *client_ctx,
                                    const SSL_TEST_CTX *test_ctx)
{
    switch (test_ctx->client_verify_callback) {
    case SSL_TEST_VERIFY_ACCEPT_ALL:
        SSL_CTX_set_cert_verify_callback(client_ctx, &verify_accept_callback,
                                         NULL);
        break;
    case SSL_TEST_VERIFY_REJECT_ALL:
        SSL_CTX_set_cert_verify_callback(client_ctx, &verify_reject_callback,
                                         NULL);
        break;
    default:
        break;
    }
    if (test_ctx->session_ticket_expected == SSL_TEST_SESSION_TICKET_BROKEN) {
        SSL_CTX_set_tlsext_ticket_key_cb(server_ctx, broken_session_ticket_callback);
    }
}

/*
 * Configure callbacks and other properties that can't be set directly
 * in the server/client CONF.
 */
static void configure_handshake_ssl(SSL *server, SSL *client,
                                    const SSL_TEST_CTX *test_ctx)
{
    SSL_set_tlsext_host_name(client, ssl_servername_name(test_ctx->servername));
}


typedef enum {
    PEER_SUCCESS,
    PEER_RETRY,
    PEER_ERROR
} peer_status_t;

static peer_status_t do_handshake_step(SSL *ssl)
{
    int ret;

    ret = SSL_do_handshake(ssl);

    if (ret == 1) {
        return PEER_SUCCESS;
    } else if (ret == 0) {
        return PEER_ERROR;
    } else {
        int error = SSL_get_error(ssl, ret);
        /* Memory bios should never block with SSL_ERROR_WANT_WRITE. */
        if (error == SSL_ERROR_WANT_READ)
            return PEER_RETRY;
        else
            return PEER_ERROR;
    }
}

typedef enum {
    /* Both parties succeeded. */
    HANDSHAKE_SUCCESS,
    /* Client errored. */
    CLIENT_ERROR,
    /* Server errored. */
    SERVER_ERROR,
    /* Peers are in inconsistent state. */
    INTERNAL_ERROR,
    /* One or both peers not done. */
    HANDSHAKE_RETRY
} handshake_status_t;

/*
 * Determine the handshake outcome.
 * last_status: the status of the peer to have acted last.
 * previous_status: the status of the peer that didn't act last.
 * client_spoke_last: 1 if the client went last.
 */
static handshake_status_t handshake_status(peer_status_t last_status,
                                           peer_status_t previous_status,
                                           int client_spoke_last)
{
    switch (last_status) {
    case PEER_SUCCESS:
        switch (previous_status) {
        case PEER_SUCCESS:
            /* Both succeeded. */
            return HANDSHAKE_SUCCESS;
        case PEER_RETRY:
            /* Let the first peer finish. */
            return HANDSHAKE_RETRY;
        case PEER_ERROR:
            /*
             * Second peer succeeded despite the fact that the first peer
             * already errored. This shouldn't happen.
             */
            return INTERNAL_ERROR;
        }

    case PEER_RETRY:
        if (previous_status == PEER_RETRY) {
            /* Neither peer is done. */
            return HANDSHAKE_RETRY;
        } else {
            /*
             * Deadlock: second peer is waiting for more input while first
             * peer thinks they're done (no more input is coming).
             */
            return INTERNAL_ERROR;
        }
    case PEER_ERROR:
        switch (previous_status) {
        case PEER_SUCCESS:
            /*
             * First peer succeeded but second peer errored.
             * TODO(emilia): we should be able to continue here (with some
             * application data?) to ensure the first peer receives the
             * alert / close_notify.
             */
            return client_spoke_last ? CLIENT_ERROR : SERVER_ERROR;
        case PEER_RETRY:
            /* We errored; let the peer finish. */
            return HANDSHAKE_RETRY;
        case PEER_ERROR:
            /* Both peers errored. Return the one that errored first. */
            return client_spoke_last ? SERVER_ERROR : CLIENT_ERROR;
        }
    }
    /* Control should never reach here. */
    return INTERNAL_ERROR;
}

HANDSHAKE_RESULT do_handshake(SSL_CTX *server_ctx, SSL_CTX *client_ctx,
                              const SSL_TEST_CTX *test_ctx)
{
    SSL *server, *client;
    BIO *client_to_server, *server_to_client;
    HANDSHAKE_EX_DATA server_ex_data, client_ex_data;
    HANDSHAKE_RESULT ret;
    int client_turn = 1;
    peer_status_t client_status = PEER_RETRY, server_status = PEER_RETRY;
    handshake_status_t status = HANDSHAKE_RETRY;
    unsigned char* tick = NULL;
    size_t len = 0;
    SSL_SESSION* sess = NULL;

    configure_handshake_ctx(server_ctx, client_ctx, test_ctx);

    server = SSL_new(server_ctx);
    client = SSL_new(client_ctx);
    OPENSSL_assert(server != NULL && client != NULL);

    configure_handshake_ssl(server, client, test_ctx);

    memset(&server_ex_data, 0, sizeof(server_ex_data));
    memset(&client_ex_data, 0, sizeof(client_ex_data));
    memset(&ret, 0, sizeof(ret));
    ret.result = SSL_TEST_INTERNAL_ERROR;

    client_to_server = BIO_new(BIO_s_mem());
    server_to_client = BIO_new(BIO_s_mem());

    OPENSSL_assert(client_to_server != NULL && server_to_client != NULL);

    /* Non-blocking bio. */
    BIO_set_nbio(client_to_server, 1);
    BIO_set_nbio(server_to_client, 1);

    SSL_set_connect_state(client);
    SSL_set_accept_state(server);

    /* The bios are now owned by the SSL object. */
    SSL_set_bio(client, server_to_client, client_to_server);
    OPENSSL_assert(BIO_up_ref(server_to_client) > 0);
    OPENSSL_assert(BIO_up_ref(client_to_server) > 0);
    SSL_set_bio(server, client_to_server, server_to_client);

    ex_data_idx = SSL_get_ex_new_index(0, "ex data", NULL, NULL, NULL);
    OPENSSL_assert(ex_data_idx >= 0);

    OPENSSL_assert(SSL_set_ex_data(server, ex_data_idx,
                                   &server_ex_data) == 1);
    OPENSSL_assert(SSL_set_ex_data(client, ex_data_idx,
                                   &client_ex_data) == 1);

    SSL_set_info_callback(server, &info_callback);
    SSL_set_info_callback(client, &info_callback);

    /*
     * Half-duplex handshake loop.
     * Client and server speak to each other synchronously in the same process.
     * We use non-blocking BIOs, so whenever one peer blocks for read, it
     * returns PEER_RETRY to indicate that it's the other peer's turn to write.
     * The handshake succeeds once both peers have succeeded. If one peer
     * errors out, we also let the other peer retry (and presumably fail).
     */
    for(;;) {
        if (client_turn) {
            client_status = do_handshake_step(client);
            status = handshake_status(client_status, server_status,
                                      1 /* client went last */);
        } else {
            server_status = do_handshake_step(server);
            status = handshake_status(server_status, client_status,
                                      0 /* server went last */);
        }

        switch (status) {
        case HANDSHAKE_SUCCESS:
            ret.result = SSL_TEST_SUCCESS;
            goto err;
        case CLIENT_ERROR:
            ret.result = SSL_TEST_CLIENT_FAIL;
            goto err;
        case SERVER_ERROR:
            ret.result = SSL_TEST_SERVER_FAIL;
            goto err;
        case INTERNAL_ERROR:
            ret.result = SSL_TEST_INTERNAL_ERROR;
            goto err;
        case HANDSHAKE_RETRY:
            /* Continue. */
            client_turn ^= 1;
            break;
        }
    }
 err:
    ret.server_alert_sent = server_ex_data.alert_sent;
    ret.server_alert_received = client_ex_data.alert_received;
    ret.client_alert_sent = client_ex_data.alert_sent;
    ret.client_alert_received = server_ex_data.alert_received;
    ret.server_protocol = SSL_version(server);
    ret.client_protocol = SSL_version(client);
    ret.servername = ((SSL_get_SSL_CTX(server) == server_ctx)
                      ? SSL_TEST_SERVERNAME_SERVER1
                      : SSL_TEST_SERVERNAME_SERVER2);
    if ((sess = SSL_get0_session(client)) != NULL)
        SSL_SESSION_get0_ticket(sess, &tick, &len);
    if (tick == NULL || len == 0)
        ret.session_ticket = SSL_TEST_SESSION_TICKET_NO;
    else
        ret.session_ticket = SSL_TEST_SESSION_TICKET_YES;
    ret.session_ticket_do_not_call = server_ex_data.session_ticket_do_not_call;

    SSL_free(server);
    SSL_free(client);
    return ret;
}
Commit	Line	Data
453dfd8d EK	1	/*
	2	* Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
	3	*
440e5d80 RS	4	* Licensed under the OpenSSL license (the "License"). You may not use
	5	* this file except in compliance with the License. You can obtain a copy
	6	* in the file LICENSE in the source distribution or at
453dfd8d	7	* https://www.openssl.org/source/license.html
453dfd8d EK	8	*/
	9
	10	#include <string.h>
	11
	12	#include <openssl/bio.h>
a263f320	13	#include <openssl/x509_vfy.h>
453dfd8d EK	14	#include <openssl/ssl.h>
	15
	16	#include "handshake_helper.h"
	17
	18	/*
	19	* Since there appears to be no way to extract the sent/received alert
	20	* from the SSL object directly, we use the info callback and stash
	21	* the result in ex_data.
	22	*/
	23	typedef struct handshake_ex_data {
	24	int alert_sent;
	25	int alert_received;
5c753de6	26	int session_ticket_do_not_call;
453dfd8d EK	27	} HANDSHAKE_EX_DATA;
	28
	29	static int ex_data_idx;
	30
	31	static void info_callback(const SSL *s, int where, int ret)
	32	{
	33	if (where & SSL_CB_ALERT) {
	34	HANDSHAKE_EX_DATA *ex_data =
	35	(HANDSHAKE_EX_DATA*)(SSL_get_ex_data(s, ex_data_idx));
	36	if (where & SSL_CB_WRITE) {
	37	ex_data->alert_sent = ret;
	38	} else {
	39	ex_data->alert_received = ret;
	40	}
	41	}
	42	}
	43
a263f320 EK	44	static int verify_reject_callback(X509_STORE_CTX ctx, void arg) {
	45	X509_STORE_CTX_set_error(ctx, X509_V_ERR_APPLICATION_VERIFICATION);
	46	return 0;
	47	}
	48
	49	static int verify_accept_callback(X509_STORE_CTX ctx, void arg) {
	50	return 1;
	51	}
	52
5c753de6 TS	53	static int broken_session_ticket_callback(SSL* s, unsigned char* key_name, unsigned char *iv,
	54	EVP_CIPHER_CTX ctx, HMAC_CTX hctx, int enc)
	55	{
	56	return 0;
	57	}
	58
	59	int do_not_call_session_ticket_callback(SSL* s, unsigned char* key_name, unsigned char *iv,
	60	EVP_CIPHER_CTX ctx, HMAC_CTX hctx, int enc)
	61	{
	62	HANDSHAKE_EX_DATA *ex_data =
	63	(HANDSHAKE_EX_DATA*)(SSL_get_ex_data(s, ex_data_idx));
	64	ex_data->session_ticket_do_not_call = 1;
	65	return 0;
	66	}
	67
a263f320 EK	68	/*
	69	* Configure callbacks and other properties that can't be set directly
	70	* in the server/client CONF.
	71	*/
5c753de6 TS	72	static void configure_handshake_ctx(SSL_CTX server_ctx, SSL_CTX client_ctx,
5c753de6 TS	73	const SSL_TEST_CTX *test_ctx)
a263f320 EK	74	{
	75	switch (test_ctx->client_verify_callback) {
	76	case SSL_TEST_VERIFY_ACCEPT_ALL:
	77	SSL_CTX_set_cert_verify_callback(client_ctx, &verify_accept_callback,
	78	NULL);
	79	break;
	80	case SSL_TEST_VERIFY_REJECT_ALL:
	81	SSL_CTX_set_cert_verify_callback(client_ctx, &verify_reject_callback,
	82	NULL);
	83	break;
	84	default:
	85	break;
	86	}
5c753de6 TS	87	if (test_ctx->session_ticket_expected == SSL_TEST_SESSION_TICKET_BROKEN) {
	88	SSL_CTX_set_tlsext_ticket_key_cb(server_ctx, broken_session_ticket_callback);
	89	}
	90	}
	91
	92	/*
	93	* Configure callbacks and other properties that can't be set directly
	94	* in the server/client CONF.
	95	*/
	96	static void configure_handshake_ssl(SSL server, SSL client,
	97	const SSL_TEST_CTX *test_ctx)
	98	{
	99	SSL_set_tlsext_host_name(client, ssl_servername_name(test_ctx->servername));
a263f320 EK	100	}
	101
	102
453dfd8d EK	103	typedef enum {
	104	PEER_SUCCESS,
	105	PEER_RETRY,
	106	PEER_ERROR
	107	} peer_status_t;
	108
	109	static peer_status_t do_handshake_step(SSL *ssl)
	110	{
	111	int ret;
	112
	113	ret = SSL_do_handshake(ssl);
	114
	115	if (ret == 1) {
	116	return PEER_SUCCESS;
	117	} else if (ret == 0) {
	118	return PEER_ERROR;
	119	} else {
	120	int error = SSL_get_error(ssl, ret);
	121	/* Memory bios should never block with SSL_ERROR_WANT_WRITE. */
	122	if (error == SSL_ERROR_WANT_READ)
	123	return PEER_RETRY;
	124	else
	125	return PEER_ERROR;
	126	}
	127	}
	128
	129	typedef enum {
	130	/* Both parties succeeded. */
	131	HANDSHAKE_SUCCESS,
	132	/* Client errored. */
	133	CLIENT_ERROR,
	134	/* Server errored. */
	135	SERVER_ERROR,
	136	/* Peers are in inconsistent state. */
	137	INTERNAL_ERROR,
	138	/* One or both peers not done. */
	139	HANDSHAKE_RETRY
	140	} handshake_status_t;
	141
	142	/*
	143	* Determine the handshake outcome.
	144	* last_status: the status of the peer to have acted last.
	145	* previous_status: the status of the peer that didn't act last.
	146	* client_spoke_last: 1 if the client went last.
	147	*/
	148	static handshake_status_t handshake_status(peer_status_t last_status,
	149	peer_status_t previous_status,
	150	int client_spoke_last)
	151	{
	152	switch (last_status) {
	153	case PEER_SUCCESS:
	154	switch (previous_status) {
	155	case PEER_SUCCESS:
	156	/* Both succeeded. */
	157	return HANDSHAKE_SUCCESS;
	158	case PEER_RETRY:
	159	/* Let the first peer finish. */
	160	return HANDSHAKE_RETRY;
	161	case PEER_ERROR:
	162	/*
	163	* Second peer succeeded despite the fact that the first peer
	164	* already errored. This shouldn't happen.
	165	*/
	166	return INTERNAL_ERROR;
167	}
168
169	case PEER_RETRY:
170	if (previous_status == PEER_RETRY) {
171	/* Neither peer is done. */
172	return HANDSHAKE_RETRY;
173	} else {
174	/*
175	* Deadlock: second peer is waiting for more input while first
176	* peer thinks they're done (no more input is coming).
177	*/
178	return INTERNAL_ERROR;
179	}
180	case PEER_ERROR:
181	switch (previous_status) {
182	case PEER_SUCCESS:
183	/*
184	* First peer succeeded but second peer errored.
185	* TODO(emilia): we should be able to continue here (with some
186	* application data?) to ensure the first peer receives the
187	* alert / close_notify.
188	*/
189	return client_spoke_last ? CLIENT_ERROR : SERVER_ERROR;
190	case PEER_RETRY:
191	/* We errored; let the peer finish. */
192	return HANDSHAKE_RETRY;
193	case PEER_ERROR:
194	/* Both peers errored. Return the one that errored first. */
195	return client_spoke_last ? SERVER_ERROR : CLIENT_ERROR;
196	}
197	}
198	/* Control should never reach here. */
199	return INTERNAL_ERROR;
200	}
201
a263f320 EK	202	HANDSHAKE_RESULT do_handshake(SSL_CTX server_ctx, SSL_CTX client_ctx,
a263f320 EK	203	const SSL_TEST_CTX *test_ctx)
453dfd8d EK	204	{
	205	SSL server, client;
	206	BIO client_to_server, server_to_client;
	207	HANDSHAKE_EX_DATA server_ex_data, client_ex_data;
	208	HANDSHAKE_RESULT ret;
	209	int client_turn = 1;
	210	peer_status_t client_status = PEER_RETRY, server_status = PEER_RETRY;
	211	handshake_status_t status = HANDSHAKE_RETRY;
5c753de6 TS	212	unsigned char* tick = NULL;
	213	size_t len = 0;
	214	SSL_SESSION* sess = NULL;
453dfd8d	215
5c753de6	216	configure_handshake_ctx(server_ctx, client_ctx, test_ctx);
a263f320	217
453dfd8d EK	218	server = SSL_new(server_ctx);
	219	client = SSL_new(client_ctx);
	220	OPENSSL_assert(server != NULL && client != NULL);
	221
5c753de6 TS	222	configure_handshake_ssl(server, client, test_ctx);
5c753de6 TS	223
453dfd8d EK	224	memset(&server_ex_data, 0, sizeof(server_ex_data));
	225	memset(&client_ex_data, 0, sizeof(client_ex_data));
	226	memset(&ret, 0, sizeof(ret));
	227	ret.result = SSL_TEST_INTERNAL_ERROR;
	228
	229	client_to_server = BIO_new(BIO_s_mem());
	230	server_to_client = BIO_new(BIO_s_mem());
	231
	232	OPENSSL_assert(client_to_server != NULL && server_to_client != NULL);
	233
	234	/* Non-blocking bio. */
	235	BIO_set_nbio(client_to_server, 1);
	236	BIO_set_nbio(server_to_client, 1);
	237
	238	SSL_set_connect_state(client);
	239	SSL_set_accept_state(server);
	240
	241	/* The bios are now owned by the SSL object. */
	242	SSL_set_bio(client, server_to_client, client_to_server);
	243	OPENSSL_assert(BIO_up_ref(server_to_client) > 0);
	244	OPENSSL_assert(BIO_up_ref(client_to_server) > 0);
	245	SSL_set_bio(server, client_to_server, server_to_client);
	246
	247	ex_data_idx = SSL_get_ex_new_index(0, "ex data", NULL, NULL, NULL);
	248	OPENSSL_assert(ex_data_idx >= 0);
	249
	250	OPENSSL_assert(SSL_set_ex_data(server, ex_data_idx,
	251	&server_ex_data) == 1);
	252	OPENSSL_assert(SSL_set_ex_data(client, ex_data_idx,
	253	&client_ex_data) == 1);
	254
	255	SSL_set_info_callback(server, &info_callback);
	256	SSL_set_info_callback(client, &info_callback);
	257
	258	/*
	259	* Half-duplex handshake loop.
	260	* Client and server speak to each other synchronously in the same process.
	261	* We use non-blocking BIOs, so whenever one peer blocks for read, it
	262	* returns PEER_RETRY to indicate that it's the other peer's turn to write.
	263	* The handshake succeeds once both peers have succeeded. If one peer
	264	* errors out, we also let the other peer retry (and presumably fail).
	265	*/
	266	for(;;) {
	267	if (client_turn) {
	268	client_status = do_handshake_step(client);
	269	status = handshake_status(client_status, server_status,
	270	1 /* client went last */);
	271	} else {
	272	server_status = do_handshake_step(server);
	273	status = handshake_status(server_status, client_status,
	274	0 /* server went last */);
	275	}
	276
	277	switch (status) {
	278	case HANDSHAKE_SUCCESS:
	279	ret.result = SSL_TEST_SUCCESS;
	280	goto err;
	281	case CLIENT_ERROR:
	282	ret.result = SSL_TEST_CLIENT_FAIL;
	283	goto err;
	284	case SERVER_ERROR:
	285	ret.result = SSL_TEST_SERVER_FAIL;
	286	goto err;
	287	case INTERNAL_ERROR:
288	ret.result = SSL_TEST_INTERNAL_ERROR;
289	goto err;
290	case HANDSHAKE_RETRY:
291	/* Continue. */
292	client_turn ^= 1;
293	break;
294	}
295	}
296	err:
297	ret.server_alert_sent = server_ex_data.alert_sent;
298	ret.server_alert_received = client_ex_data.alert_received;
299	ret.client_alert_sent = client_ex_data.alert_sent;
300	ret.client_alert_received = server_ex_data.alert_received;
301	ret.server_protocol = SSL_version(server);
302	ret.client_protocol = SSL_version(client);
5c753de6 TS	303	ret.servername = ((SSL_get_SSL_CTX(server) == server_ctx)
	304	? SSL_TEST_SERVERNAME_SERVER1
	305	: SSL_TEST_SERVERNAME_SERVER2);
	306	if ((sess = SSL_get0_session(client)) != NULL)
	307	SSL_SESSION_get0_ticket(sess, &tick, &len);
	308	if (tick == NULL \|\| len == 0)
	309	ret.session_ticket = SSL_TEST_SESSION_TICKET_NO;
	310	else
	311	ret.session_ticket = SSL_TEST_SESSION_TICKET_YES;
	312	ret.session_ticket_do_not_call = server_ex_data.session_ticket_do_not_call;
453dfd8d EK	313
	314	SSL_free(server);
	315	SSL_free(client);
	316	return ret;
	317	}