From: Jean-Frederic Clere Date: Fri, 12 Jul 2024 15:27:56 +0000 (+0200) Subject: ossl-nghttp3-demo-server.c: Fix compatibility with various clients X-Git-Tag: openssl-3.5.0-alpha1~387 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a31dfb0ee655b1b9a181d05f32975a9a42fd9785;p=thirdparty%2Fopenssl.git ossl-nghttp3-demo-server.c: Fix compatibility with various clients Fixes openssl/project#752 Reviewed-by: Viktor Dukhovni Reviewed-by: Tomas Mraz (Merged from https://github.com/openssl/openssl/pull/24946) --- diff --git a/demos/http3/ossl-nghttp3-demo-server.c b/demos/http3/ossl-nghttp3-demo-server.c index ccf98e7e5fe..c511fb88bf7 100644 --- a/demos/http3/ossl-nghttp3-demo-server.c +++ b/demos/http3/ossl-nghttp3-demo-server.c @@ -19,19 +19,30 @@ sizeof((VALUE)) - 1, NGHTTP3_NV_FLAG_NONE } #define nghttp3_arraylen(A) (sizeof(A) / sizeof(*(A))) -/* CURL according to trace has 2 more streams 7 and 11 */ +/* 3 streams created by the server and 4 by the client (one is bidi) */ struct ssl_id { - SSL *s; - uint64_t id; + SSL *s; /* the stream openssl uses in SSL_read(), SSL_write etc */ + uint64_t id; /* the stream identifier the nghttp3 uses */ + int status; /* 0, CLIENTUNIOPEN or CLIENTUNIOPEN|CLIENTCLOSED (for the moment) */ }; +/* status and origin of the streams the possible values are: */ +#define CLIENTUNIOPEN 0x01 /* unidirectional open by the client (2, 6 and 10) */ +#define CLIENTCLOSED 0x02 /* closed by the client */ +#define CLIENTBIDIOPEN 0x04 /* bidirectional open by the client (something like 0, 4, 8 ...) */ +#define SERVERUNIOPEN 0x08 /* unidirectional open by the server (3, 7 and 11) */ +#define SERVERCLOSED 0x10 /* closed by the server (us) */ #define MAXSSL_IDS 20 struct h3ssl { struct ssl_id ssl_ids[MAXSSL_IDS]; - int end_headers_received; - int datadone; - int has_uni; - int done; + int end_headers_received; /* h3 header received call back called */ + int datadone; /* h3 has given openssl all the data of the response */ + int has_uni; /* we have the 3 uni directional stream needed */ + int close_done; /* connection begins terminating EVENT_EC */ + int done; /* connection terminated EVENT_ECD, after EVENT_EC */ + int received_from_two; /* workaround for -607 on nghttp3_conn_read_stream on stream 2 */ + int restart; /* new request/response cycle started */ + uint64_t id_bidi; /* the id of the stream used to read request and send response */ }; static void init_ids(struct h3ssl *h3ssl) @@ -43,11 +54,15 @@ static void init_ids(struct h3ssl *h3ssl) for (i = 0; i < MAXSSL_IDS; i++) { ssl_ids[i].s = NULL; ssl_ids[i].id = -1; + ssl_ids[i].status = 0; } h3ssl->end_headers_received = 0; h3ssl->datadone = 0; h3ssl->has_uni = 0; + h3ssl->close_done = 0; h3ssl->done = 0; + h3ssl->received_from_two = 0; + h3ssl->restart = 0; } static void add_id(uint64_t id, SSL *ssl, struct h3ssl *h3ssl) @@ -67,6 +82,38 @@ static void add_id(uint64_t id, SSL *ssl, struct h3ssl *h3ssl) exit(1); } +static void set_id_status(uint64_t id, int status, struct h3ssl *h3ssl) +{ + struct ssl_id *ssl_ids; + int i; + + ssl_ids = h3ssl->ssl_ids; + for (i = 0; i < MAXSSL_IDS; i++) { + if (ssl_ids[i].id == id) { + printf("set_id_status: %llu to %d\n", (unsigned long long) ssl_ids[i].id, status); + ssl_ids[i].status = status; + return; + } + } + printf("Oops can't set status, can't find stream!!!\n"); + assert(0); +} + +static int are_all_clientid_closed(struct h3ssl *h3ssl) +{ + struct ssl_id *ssl_ids; + int i; + + ssl_ids = h3ssl->ssl_ids; + for (i = 0; i < MAXSSL_IDS; i++) { + if (ssl_ids[i].status == CLIENTUNIOPEN) { + printf("are_all_clientid_closed: %llu open\n", (unsigned long long) ssl_ids[i].id); + return 0; + } + } + return 1; +} + static void h3close(struct h3ssl *h3ssl, uint64_t id) { struct ssl_id *ssl_ids; @@ -75,9 +122,10 @@ static void h3close(struct h3ssl *h3ssl, uint64_t id) ssl_ids = h3ssl->ssl_ids; for (i = 0; i < MAXSSL_IDS; i++) { if (ssl_ids[i].id == id) { - if (!SSL_stream_conclude(ssl_ids[i].s, 0)) + if (!SSL_stream_conclude(ssl_ids[i].s, 0)) { fprintf(stderr, "h3close: SSL_stream_conclude on %llu failed\n", (unsigned long long) id); - SSL_shutdown(ssl_ids[i].s); + ERR_print_errors_fp(stderr); + } } } } @@ -130,15 +178,127 @@ static int on_end_stream(nghttp3_conn *h3conn, int64_t stream_id, return 0; } -static int read_from_ssl_ids(nghttp3_conn *h3conn, struct h3ssl *h3ssl) +/* Read from the stream and push to the h3conn */ +static int quic_server_read(nghttp3_conn *h3conn, SSL *stream, uint64_t id, struct h3ssl *h3ssl) { + int ret, r; uint8_t msg2[16000]; + size_t l = sizeof(msg2); + + if (!SSL_has_pending(stream)) + return 0; /* Nothing to read */ + + ret = SSL_read(stream, msg2, l); + if (ret <= 0) { + fprintf(stderr, "SSL_read %d on %llu failed\n", + SSL_get_error(stream, ret), + (unsigned long long) id); + if (SSL_get_error(stream, ret) == SSL_ERROR_WANT_READ) { + return 0; /* retry we need more data */ + } + ERR_print_errors_fp(stderr); + return -1; + } + + /* XXX: work around nghttp3_conn_read_stream returning -607 on stream 2 */ + if (!h3ssl->received_from_two && id != 2 ) { + r = nghttp3_conn_read_stream(h3conn, id, msg2, ret, 0); + } else { + r = ret; /* ignore it for the moment ... */ + } + + printf("nghttp3_conn_read_stream used %d of %d on %llu\n", r, + ret, (unsigned long long) id); + if (r != ret) { + /* chrome returns -607 on stream 2 */ + if (!nghttp3_err_is_fatal(r)) { + printf("nghttp3_conn_read_stream used %d of %d (not fatal) on %llu\n", r, + ret, (unsigned long long) id); + if (id == 2) { + h3ssl->received_from_two = 1; + } + return 1; + } + return -1; + } + return 1; +} + + +/* + * creates the control stream, the encoding and decoding streams. + * nghttp3_conn_bind_control_stream() is for the control stream. + */ +static int quic_server_h3streams(nghttp3_conn *h3conn, struct h3ssl *h3ssl) +{ + SSL *rstream; + SSL *pstream; + SSL *cstream; + uint64_t r_streamid, p_streamid, c_streamid; + struct ssl_id *ssl_ids = h3ssl->ssl_ids; + + rstream = SSL_new_stream(ssl_ids[0].s, SSL_STREAM_FLAG_UNI); + if (rstream != NULL) { + fprintf(stderr, "=> Opened on %llu\n", + (unsigned long long)SSL_get_stream_id(rstream)); + fflush(stderr); + } else { + fprintf(stderr, "=> Stream == NULL!\n"); + fflush(stderr); + return -1; + } + pstream = SSL_new_stream(ssl_ids[0].s, SSL_STREAM_FLAG_UNI); + if (pstream != NULL) { + fprintf(stderr, "=> Opened on %llu\n", + (unsigned long long)SSL_get_stream_id(pstream)); + fflush(stderr); + } else { + fprintf(stderr, "=> Stream == NULL!\n"); + fflush(stderr); + return -1; + } + cstream = SSL_new_stream(ssl_ids[0].s, SSL_STREAM_FLAG_UNI); + if (cstream != NULL) { + fprintf(stderr, "=> Opened on %llu\n", + (unsigned long long)SSL_get_stream_id(cstream)); + fflush(stderr); + } else { + fprintf(stderr, "=> Stream == NULL!\n"); + fflush(stderr); + return -1; + } + r_streamid = SSL_get_stream_id(rstream); + p_streamid = SSL_get_stream_id(pstream); + c_streamid = SSL_get_stream_id(cstream); + if (nghttp3_conn_bind_qpack_streams(h3conn, p_streamid, r_streamid)) { + fprintf(stderr, "nghttp3_conn_bind_qpack_streams failed!\n"); + return -1; + } + if (nghttp3_conn_bind_control_stream(h3conn, c_streamid)) { + fprintf(stderr, "nghttp3_conn_bind_qpack_streams failed!\n"); + return -1; + } + printf("control: %llu enc %llu dec %llu\n", + (unsigned long long)c_streamid, + (unsigned long long)p_streamid, + (unsigned long long)r_streamid); + add_id(SSL_get_stream_id(rstream), rstream, h3ssl); + add_id(SSL_get_stream_id(pstream), pstream, h3ssl); + add_id(SSL_get_stream_id(cstream), cstream, h3ssl); + + return 0; +} + +/* Try to read from the streams we have */ +static int read_from_ssl_ids(nghttp3_conn *h3conn, struct h3ssl *h3ssl) +{ int hassomething = 0, i; struct ssl_id *ssl_ids = h3ssl->ssl_ids; SSL_POLL_ITEM items[MAXSSL_IDS] = {0}, *item = items; static const struct timeval nz_timeout = {0, 0}; size_t result_count = SIZE_MAX; int numitem = 0, ret; + uint64_t processed_event = 0; /* * Process all the streams @@ -153,12 +313,23 @@ static int read_from_ssl_ids(nghttp3_conn *h3conn, struct h3ssl *h3ssl) item++; } } + + /* + * SSL_POLL_FLAG_NO_HANDLE_EVENTS would require to use: + * SSL_get_event_timeout on the connection stream + * select/wait using the timeout value (which could be no wait time) + * SSL_handle_events + * SSL_poll + * for the moment we let SSL_poll to performs ticking internally + * on an automatic basis. + */ ret = SSL_poll(items, numitem, sizeof(SSL_POLL_ITEM), &nz_timeout, 0, &result_count); if (!ret) { fprintf(stderr, "SSL_poll failed\n"); return -1; /* something is wrong */ } + printf("read_from_ssl_ids %ld events\n", (unsigned long)result_count); if (result_count == 0) { /* Timeout may be something somewhere */ return 0; @@ -170,165 +341,136 @@ static int read_from_ssl_ids(nghttp3_conn *h3conn, struct h3ssl *h3ssl) if ((item->revents & SSL_POLL_EVENT_ISB) || (item->revents & SSL_POLL_EVENT_ISU)) { SSL *stream = SSL_accept_stream(ssl_ids[0].s, 0); + uint64_t id; + int r; if (stream == NULL) { return -1; /* something is wrong */ } - printf("=> Received connection on %lld\n", - (unsigned long long)SSL_get_stream_id(stream)); - add_id(SSL_get_stream_id(stream), stream, h3ssl); - printf("read_from_ssl_ids %ld events\n", (unsigned long)result_count); - if (result_count == 1) { - return 1; /* loop until we have all the streams */ + id = SSL_get_stream_id(stream); + printf("=> Received connection on %lld %d\n", (unsigned long long) id, + SSL_get_stream_type(stream)); + add_id(id, stream, h3ssl); + if (SSL_get_stream_type(stream) == SSL_STREAM_TYPE_BIDI) { + /* bidi that is the id where we have to send the response */ + printf("=> Received connection on %lld ISBIDI\n", + (unsigned long long) id); + h3ssl->id_bidi = id; + + /* XXX use it to restart to end_headers_received */ + h3ssl->end_headers_received = 0; + h3ssl->datadone = 0; + h3ssl->close_done = 0; + h3ssl->done = 0; + h3ssl->restart = 1; /* Checked in wait_close loop */ + } else { + set_id_status(id, CLIENTUNIOPEN, h3ssl); } + + r = quic_server_read(h3conn, stream, id, h3ssl); + if (r == -1) { + return -1; /* something is wrong */ + } + if (r == 1) { + hassomething++; + } + if (item->revents & SSL_POLL_EVENT_ISB) + processed_event = processed_event + SSL_POLL_EVENT_ISB; + if (item->revents & SSL_POLL_EVENT_ISU) + processed_event = processed_event + SSL_POLL_EVENT_ISU; } - /* Create new streams when allowed */ if (item->revents & SSL_POLL_EVENT_OSB) { + /* Create new streams when allowed */ /* at least one bidi */ + processed_event = processed_event + SSL_POLL_EVENT_OSB; printf("Create bidi?\n"); } if (item->revents & SSL_POLL_EVENT_OSU) { /* at least one uni */ - printf("Create uni\n"); /* we have 4 streams from the client 2, 6 , 10 and 0 */ - /* need 2 streams to the client */ + /* need 3 streams to the client */ + printf("Create uni?\n"); + processed_event = processed_event + SSL_POLL_EVENT_OSU; if (!h3ssl->has_uni) { - SSL *rstream; - SSL *pstream; - uint64_t r_streamid, p_streamid; - - printf("Create uni beacause !h3ssl->has_uni\n"); - rstream = SSL_new_stream(ssl_ids[0].s, SSL_STREAM_FLAG_UNI); - if (rstream != NULL) { - fprintf(stderr, "=> Opened on %llu\n", - (unsigned long long)SSL_get_stream_id(rstream)); - fflush(stderr); - } else { - fprintf(stderr, "=> Stream == NULL!\n"); - fflush(stderr); - return -1; - } - pstream = SSL_new_stream(ssl_ids[0].s, SSL_STREAM_FLAG_UNI); - if (pstream != NULL) { - fprintf(stderr, "=> Opened on %llu\n", - (unsigned long long)SSL_get_stream_id(pstream)); - fflush(stderr); - } else { - fprintf(stderr, "=> Stream == NULL!\n"); - fflush(stderr); + printf("Create uni\n"); + ret = quic_server_h3streams(h3conn, h3ssl); + if (ret == -1) { + fprintf(stderr, "quic_server_h3streams failed!\n"); return -1; } - r_streamid = SSL_get_stream_id(rstream); - p_streamid = SSL_get_stream_id(pstream); - if (nghttp3_conn_bind_qpack_streams(h3conn, p_streamid, - r_streamid)) { - fprintf(stderr, "nghttp3_conn_bind_qpack_streams failed!\n"); - return -1; - } - printf("control: NONE enc %llu dec %llu\n", - (unsigned long long)p_streamid, - (unsigned long long)r_streamid); - add_id(SSL_get_stream_id(rstream), rstream, h3ssl); - add_id(SSL_get_stream_id(pstream), pstream, h3ssl); h3ssl->has_uni = 1; - if (result_count == 1) { - printf("read_from_ssl_ids 1 event only!\n"); - return 0; /* one event only so we are done */ - } + hassomething++; + } + } + if (item->revents & SSL_POLL_EVENT_EC) { + /* the connection begins terminating */ + printf("Connection terminating\n"); + if (!h3ssl->close_done) { + h3ssl->close_done = 1; + } else { + h3ssl->done = 1; } + hassomething++; + processed_event = processed_event + SSL_POLL_EVENT_EC; + } + if (item->revents & SSL_POLL_EVENT_ECD) { + /* the connection is terminated */ + printf("Connection terminated\n"); + h3ssl->done = 1; + hassomething++; + processed_event = processed_event + SSL_POLL_EVENT_ECD; + } + if (item->revents != processed_event) { + /* we missed something we need to figure out */ + printf("Missed revent %llu (%d) on %llu\n", + (unsigned long long)item->revents, SSL_POLL_EVENT_W, + (unsigned long long)ssl_ids[i].id); + } + if (result_count == 1 && !processed_event) { + printf("read_from_ssl_ids 1 event only!\n"); + return hassomething; /* one event only so we are done */ } /* Well trying... */ if (numitem <= 1) { - return 0; /* Assume nothing for the moment */ + return hassomething; } /* Process the other stream */ for (i = 1; i < numitem; i++) { item++; + processed_event = 0; if (item->revents & SSL_POLL_EVENT_R) { /* try to read */ - size_t l = sizeof(msg2) - 1; int r; - if (!SSL_net_read_desired(ssl_ids[i].s)) { + printf("revent READ on %llu\n", + (unsigned long long)ssl_ids[i].id); + r = quic_server_read(h3conn, ssl_ids[i].s, ssl_ids[i].id, h3ssl); + if (r == 0) { continue; } - ret = SSL_read(ssl_ids[i].s, msg2, l); - if (ret <= 0) { - fprintf(stderr, "SSL_read on %llu failed\n", - (unsigned long long)ssl_ids[i].id); - continue; /* TODO */ + if (r == -1) { + return -1; } - r = nghttp3_conn_read_stream(h3conn, ssl_ids[i].id, msg2, - ret, 0); - - printf("reading something %d on %llu\n", ret, - (unsigned long long)ssl_ids[i].id); - printf("nghttp3_conn_read_stream used %d of %d on %llu\n", r, - ret, (unsigned long long)ssl_ids[i].id); hassomething++; - } else { - /* Figure out ??? */ - printf("revent %llu (%d) on %llu\n", - (unsigned long long)item->revents, SSL_POLL_EVENT_W, - (unsigned long long)ssl_ids[i].id); - } - } - return hassomething; -} - -static int nothing_from_ssl_ids(nghttp3_conn *h3conn, struct h3ssl *h3ssl) -{ - int hassomething = 0; - struct ssl_id *ssl_ids; - SSL_POLL_ITEM items[MAXSSL_IDS] = {0}, *item = items; - static const struct timeval nz_timeout = {0, 0}; - size_t result_count = SIZE_MAX; - int numitem = 0, i, ret; - - ssl_ids = h3ssl->ssl_ids; - /* Process all the streams */ - for (i = 0; i < MAXSSL_IDS; i++) { - if (ssl_ids[i].s) { - item->desc = SSL_as_poll_descriptor(ssl_ids[i].s); - item->events = UINT64_MAX; /* TODO adjust to the event we need process */ - item->revents = UINT64_MAX; /* TODO adjust to the event we need process */ - numitem++; - item++; + processed_event = processed_event + SSL_POLL_EVENT_R; } - } - ret = SSL_poll(items, numitem, sizeof(SSL_POLL_ITEM), &nz_timeout, 0, - &result_count); - if (!ret) { - fprintf(stderr, "SSL_poll failed\n"); - return -1; /* something is wrong */ - } - if (result_count == 0) { - /* Timeout may be something somewhere */ - return 0; - } - - /* We have something */ - item = items; - for (i = 0; i < numitem; i++) { - item++; - if (item->revents == SSL_POLL_EVENT_NONE) { - continue; - } else if (item->revents == SSL_POLL_EVENT_W) { - printf("revent %llu (%d) on %llu\n", - (unsigned long long)item->revents, SSL_POLL_EVENT_W, - (unsigned long long)ssl_ids[i].id); - } else if (item->revents == SSL_POLL_EVENT_ER) { - printf("revent %llu (%d) on %llu\n", - (unsigned long long)item->revents, SSL_POLL_EVENT_ER, + if (item->revents & SSL_POLL_EVENT_ER) { + /* mark it closed */ + printf("revent exception READ on %llu\n", (unsigned long long)ssl_ids[i].id); - } else { + if (ssl_ids[i].status == CLIENTUNIOPEN) { + ssl_ids[i].status = ssl_ids[i].status | CLIENTCLOSED; + hassomething++; + } + processed_event = processed_event + SSL_POLL_EVENT_ER; + } + if (item->revents != processed_event) { /* Figure out ??? */ printf("revent %llu (%d) on %llu\n", - (unsigned long long)item->revents, SSL_POLL_EVENT_NONE, + (unsigned long long)item->revents, SSL_POLL_EVENT_W, (unsigned long long)ssl_ids[i].id); - hassomething++; } } return hassomething; @@ -481,11 +623,12 @@ static int waitsocket(int fd, int sec) printf("waitsocket for %d\n", sec); ret = select(fdmax + 1, &read_fds, NULL, NULL, &tv); } else { + printf("waitsocket for ever\n"); ret = select(fdmax + 1, &read_fds, NULL, NULL, NULL); } if (ret == -1) { fprintf(stderr, "waitsocket failed\n"); - exit(1); + return -2; } else if (ret) { printf("waitsocket %d\n", FD_ISSET(fd, &read_fds)); return 0; @@ -497,6 +640,7 @@ static int waitsocket(int fd, int sec) static int run_quic_server(SSL_CTX *ctx, int fd) { int ok = 0; + int hassomething = 0; SSL *listener = NULL, *conn = NULL; /* Create a new QUIC listener. */ @@ -525,16 +669,25 @@ static int run_quic_server(SSL_CTX *ctx, int fd) nghttp3_callbacks callbacks = {0}; struct h3ssl h3ssl; const nghttp3_mem *mem = nghttp3_mem_default(); - int hassomething = 0; nghttp3_nv resp[] = { MAKE_NV(":status", "200"), MAKE_NV("content-length", "20"), }; nghttp3_data_reader dr; - - fprintf(stderr, "waiting on socket\n"); - fflush(stderr); - waitsocket(fd, 0); + int ret; + int numtimeout; + + if (!hassomething) { + fprintf(stderr, "waiting on socket\n"); + fflush(stderr); + ret = waitsocket(fd, 0); + if (ret == -2) { + SSL_free(conn); + printf("waitsocket tells -2\n"); + fflush(stdout); + goto err; + } + } fprintf(stderr, "before SSL_accept_connection\n"); fflush(stderr); @@ -546,7 +699,9 @@ static int run_quic_server(SSL_CTX *ctx, int fd) fflush(stderr); if (conn == NULL) { fprintf(stderr, "error while accepting connection\n"); + hassomething = 0; continue; + /* goto err; */ } /* set the incoming stream policy to accept */ @@ -584,12 +739,24 @@ static int run_quic_server(SSL_CTX *ctx, int fd) fflush(stdout); /* wait until we have received the headers */ +restart: + numtimeout = 0; while (!h3ssl.end_headers_received) { - if (!hassomething) - if (waitsocket(fd, 5)) { - printf("read_from_ssl_ids timeout\n"); - goto err; + if (!hassomething) { + /* + * XXX: 25 is TOO BIG. + * Probably something wrong when waiting for the close on + * the previous request/response + */ + if (waitsocket(fd, 1)) { + printf("waiting for end_headers_received timeout %d\n", numtimeout); + numtimeout++; + if (numtimeout == 25) + goto err; + } else { + printf("waiting for end_headers_received done\n"); } + } hassomething = read_from_ssl_ids(h3conn, &h3ssl); if (hassomething == -1) { fprintf(stderr, "read_from_ssl_ids hassomething failed\n"); @@ -597,19 +764,38 @@ static int run_quic_server(SSL_CTX *ctx, int fd) } else if (hassomething == 0) { printf("read_from_ssl_ids hassomething nothing...\n"); } else { + numtimeout = 0; printf("read_from_ssl_ids hassomething %d...\n", hassomething); + if (h3ssl.close_done) { + /* Other side has closed */ + break; + } + h3ssl.restart = 0; } } + if (h3ssl.close_done) { + printf("Other side close without request\n"); + goto wait_close; + } printf("end_headers_received!!!\n"); + if (!h3ssl.has_uni) { + /* time to create those otherwise we can't push anything to the client */ + printf("Create uni\n"); + if (quic_server_h3streams(h3conn, &h3ssl) == -1) { + fprintf(stderr, "quic_server_h3streams failed!\n"); + goto err; + } + h3ssl.has_uni = 1; + } /* we have receive the request build the response and send it */ /* XXX add MAKE_NV("connection", "close"), to resp[] and recheck */ dr.read_data = step_read_data; - if (nghttp3_conn_submit_response(h3conn, 0, resp, 2, &dr)) { + if (nghttp3_conn_submit_response(h3conn, h3ssl.id_bidi, resp, 2, &dr)) { fprintf(stderr, "nghttp3_conn_submit_response failed!\n"); goto err; } - printf("nghttp3_conn_submit_response...\n"); + printf("nghttp3_conn_submit_response on %llu...\n", (unsigned long long) h3ssl.id_bidi); for (;;) { nghttp3_vec vec[256]; nghttp3_ssize sveccnt; @@ -619,9 +805,20 @@ static int run_quic_server(SSL_CTX *ctx, int fd) sveccnt = nghttp3_conn_writev_stream(h3conn, &streamid, &fin, vec, nghttp3_arraylen(vec)); if (sveccnt <= 0) { - printf("nghttp3_conn_writev_stream done: %ld\n", - (long int)sveccnt); - break; + printf("nghttp3_conn_writev_stream done: %ld stream: %llu fin %d\n", + (long int)sveccnt, + (unsigned long long)streamid, + fin); + if (streamid != -1 && fin) { + printf("Sending end data on %llu fin %d\n", + (unsigned long long) streamid, fin); + nghttp3_conn_add_write_offset(h3conn, streamid, 0); + continue; + } + if (!h3ssl.datadone) + goto err; + else + break; /* Done */ } printf("nghttp3_conn_writev_stream: %ld fin: %d\n", (long int)sveccnt, fin); for (i = 0; i < sveccnt; i++) { @@ -653,26 +850,48 @@ static int run_quic_server(SSL_CTX *ctx, int fd) * All the data was sent. * close stream zero */ - h3close(&h3ssl, 0); + if (!h3ssl.close_done) { + h3close(&h3ssl, h3ssl.id_bidi); + } } /* wait until closed */ +wait_close: for (;;) { int hasnothing; - if (waitsocket(fd, 5)) { + if (waitsocket(fd, 60)) { printf("hasnothing timeout\n"); - goto err; + /* XXX probably not always OK */ + break; } - hasnothing = nothing_from_ssl_ids(h3conn, &h3ssl); + hasnothing = read_from_ssl_ids(h3conn, &h3ssl); if (hasnothing == -1) { printf("hasnothing failed\n"); - goto err; + break; + /* goto err; well in fact not */ } else if (hasnothing == 0) { printf("hasnothing nothing...\n"); - break; + continue; } else { - printf("hasnothing something...\n"); + printf("hasnothing something\n"); + if (h3ssl.done) { + printf("hasnothing something... DONE\n"); + /* we might already have the next connection to accept */ + hassomething = 1; + break; + } + if (h3ssl.restart) { + printf("hasnothing something... RESTART\n"); + h3ssl.restart = 0; + goto restart; + } + if (are_all_clientid_closed(&h3ssl)) { + printf("hasnothing something... DONE other side closed\n"); + /* there might 2 or 3 message we will ignore */ + hassomething = 0; + break; + } } }