demos/http3/ossl-nghttp3.c

   1 #include "ossl-nghttp3.h"
   2 #include <openssl/err.h>
   3 #include <assert.h>
   4
   5 #define ARRAY_LEN(x) (sizeof(x)/sizeof((x)[0]))
   6
   7 enum {
   8     H3_STREAM_TYPE_CTRL_SEND,
   9     H3_STREAM_TYPE_QPACK_ENC_SEND,
  10     H3_STREAM_TYPE_QPACK_DEC_SEND,
  11     H3_STREAM_TYPE_REQ,
  12 };
  13
  14 #define BUF_SIZE    4096
  15
  16 struct h3_stream_st {
  17     uint64_t            id;             /* QUIC stream ID */
  18     SSL                 *s;             /* QUIC stream SSL object */
  19     int                 done_recv_fin;  /* Received FIN */
  20     void                *user_data;
  21
  22     uint8_t             buf[BUF_SIZE];
  23     size_t              buf_cur, buf_total;
  24 };
  25
  26 DEFINE_LHASH_OF_EX(H3_STREAM);
  27
  28 static void h3_stream_free(H3_STREAM *s)
  29 {
  30     if (s == NULL)
  31         return;
  32
  33     SSL_free(s->s);
  34     OPENSSL_free(s);
  35 }
  36
  37 static unsigned long h3_stream_hash(const H3_STREAM *s)
  38 {
  39     return (unsigned long)s->id;
  40 }
  41
  42 static int h3_stream_eq(const H3_STREAM *a, const H3_STREAM *b)
  43 {
  44     if (a->id < b->id) return -1;
  45     if (a->id > b->id) return 1;
  46     return 0;
  47 }
  48
  49 void *H3_STREAM_get_user_data(const H3_STREAM *s)
  50 {
  51     return s->user_data;
  52 }
  53
  54 struct h3_conn_st {
  55     SSL                 *qconn;         /* QUIC connection SSL object */
  56     BIO                 *qconn_bio;     /* BIO wrapping QCSO */
  57     nghttp3_conn        *h3conn;        /* HTTP/3 connection object */
  58     LHASH_OF(H3_STREAM) *streams;       /* map of stream IDs to H3_STREAMs */
  59     void                *user_data;     /* opaque user data pointer */
  60
  61     int                 pump_res;
  62     size_t              consumed_app_data;
  63
  64     /* Forwarding callbacks */
  65     nghttp3_recv_data           recv_data_cb;
  66     nghttp3_stream_close        stream_close_cb;
  67     nghttp3_stop_sending        stop_sending_cb;
  68     nghttp3_reset_stream        reset_stream_cb;
  69     nghttp3_deferred_consume    deferred_consume_cb;
  70 };
  71
  72 void H3_CONN_free(H3_CONN *conn)
  73 {
  74     if (conn == NULL)
  75         return;
  76
  77     lh_H3_STREAM_doall(conn->streams, h3_stream_free);
  78
  79     nghttp3_conn_del(conn->h3conn);
  80     BIO_free_all(conn->qconn_bio);
  81     lh_H3_STREAM_free(conn->streams);
  82     OPENSSL_free(conn);
  83 }
  84
  85 static H3_STREAM *h3_conn_create_stream(H3_CONN *conn, int type)
  86 {
  87     H3_STREAM *s;
  88     uint64_t flags = SSL_STREAM_FLAG_ADVANCE;
  89
  90     if ((s = OPENSSL_zalloc(sizeof(H3_STREAM))) == NULL)
  91         return NULL;
  92
  93     if (type != H3_STREAM_TYPE_REQ)
  94         flags |= SSL_STREAM_FLAG_UNI;
  95
  96     if ((s->s = SSL_new_stream(conn->qconn, flags)) == NULL) {
  97         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
  98                        "could not create QUIC stream object");
  99         goto err;
 100     }
 101
 102     s->id   = SSL_get_stream_id(s->s);
 103     lh_H3_STREAM_insert(conn->streams, s);
 104     return s;
 105
 106 err:
 107     OPENSSL_free(s);
 108     return NULL;
 109 }
 110
 111 static H3_STREAM *h3_conn_accept_stream(H3_CONN *conn, SSL *qstream)
 112 {
 113     H3_STREAM *s;
 114
 115     if ((s = OPENSSL_zalloc(sizeof(H3_STREAM))) == NULL)
 116         return NULL;
 117
 118     s->id   = SSL_get_stream_id(qstream);
 119     s->s    = qstream;
 120     lh_H3_STREAM_insert(conn->streams, s);
 121     return s;
 122 }
 123
 124 static void h3_conn_remove_stream(H3_CONN *conn, H3_STREAM *s)
 125 {
 126     if (s == NULL)
 127         return;
 128
 129     lh_H3_STREAM_delete(conn->streams, s);
 130     h3_stream_free(s);
 131 }
 132
 133 static int h3_conn_recv_data(nghttp3_conn *h3conn, int64_t stream_id,
 134                              const uint8_t *data, size_t datalen,
 135                              void *conn_user_data, void *stream_user_data)
 136 {
 137     H3_CONN *conn = conn_user_data;
 138
 139     conn->consumed_app_data += datalen;
 140     if (conn->recv_data_cb == NULL)
 141         return 0;
 142
 143     return conn->recv_data_cb(h3conn, stream_id, data, datalen,
 144                               conn_user_data, stream_user_data);
 145 }
 146
 147 static int h3_conn_stream_close(nghttp3_conn *h3conn, int64_t stream_id,
 148                                 uint64_t app_error_code,
 149                                 void *conn_user_data, void *stream_user_data)
 150 {
 151     int ret = 0;
 152     H3_CONN *conn = conn_user_data;
 153     H3_STREAM *stream = stream_user_data;
 154
 155     if (conn->stream_close_cb != NULL)
 156         ret = conn->stream_close_cb(h3conn, stream_id, app_error_code,
 157                                     conn_user_data, stream_user_data);
 158
 159     h3_conn_remove_stream(conn, stream);
 160     return ret;
 161 }
 162
 163 static int h3_conn_stop_sending(nghttp3_conn *h3conn, int64_t stream_id,
 164                                 uint64_t app_error_code,
 165                                 void *conn_user_data, void *stream_user_data)
 166 {
 167     int ret = 0;
 168     H3_CONN *conn = conn_user_data;
 169     H3_STREAM *stream = stream_user_data;
 170
 171     if (conn->stop_sending_cb != NULL)
 172         ret = conn->stop_sending_cb(h3conn, stream_id, app_error_code,
 173                                     conn_user_data, stream_user_data);
 174
 175     SSL_free(stream->s);
 176     stream->s = NULL;
 177     return ret;
 178 }
 179
 180 static int h3_conn_reset_stream(nghttp3_conn *h3conn, int64_t stream_id,
 181                                 uint64_t app_error_code,
 182                                 void *conn_user_data, void *stream_user_data)
 183 {
 184     int ret = 0;
 185     H3_CONN *conn = conn_user_data;
 186     H3_STREAM *stream = stream_user_data;
 187     SSL_STREAM_RESET_ARGS args = {0};
 188
 189     if (conn->reset_stream_cb != NULL)
 190         ret = conn->reset_stream_cb(h3conn, stream_id, app_error_code,
 191                                    conn_user_data, stream_user_data);
 192
 193     if (stream->s != NULL) {
 194         args.quic_error_code = app_error_code;
 195
 196         if (!SSL_stream_reset(stream->s, &args, sizeof(args)))
 197             return 1;
 198     }
 199
 200     return ret;
 201 }
 202
 203 static int h3_conn_deferred_consume(nghttp3_conn *h3conn, int64_t stream_id,
 204                                     size_t consumed,
 205                                     void *conn_user_data, void *stream_user_data)
 206 {
 207     int ret = 0;
 208     H3_CONN *conn = conn_user_data;
 209
 210     if (conn->deferred_consume_cb != NULL)
 211         ret = conn->deferred_consume_cb(h3conn, stream_id, consumed,
 212                                         conn_user_data, stream_user_data);
 213
 214     conn->consumed_app_data += consumed;
 215     return ret;
 216 }
 217
 218 H3_CONN *H3_CONN_new_for_conn(BIO *qconn_bio,
 219                               const nghttp3_callbacks *callbacks,
 220                               const nghttp3_settings *settings,
 221                               void *user_data)
 222 {
 223     int ec;
 224     H3_CONN *conn;
 225     H3_STREAM *s_ctl_send = NULL, *s_qpenc_send = NULL, *s_qpdec_send = NULL;
 226     nghttp3_settings dsettings = {0};
 227     nghttp3_callbacks intl_callbacks = {0};
 228     static const unsigned char alpn[] = {2, 'h', '3'};
 229
 230     if (qconn_bio == NULL) {
 231         ERR_raise_data(ERR_LIB_USER, ERR_R_PASSED_NULL_PARAMETER,
 232                        "QUIC connection BIO must be provided");
 233         return NULL;
 234     }
 235
 236     if ((conn = OPENSSL_zalloc(sizeof(H3_CONN))) == NULL)
 237         return NULL;
 238
 239     conn->qconn_bio = qconn_bio;
 240     conn->user_data = user_data;
 241
 242     if (BIO_get_ssl(qconn_bio, &conn->qconn) == 0) {
 243         ERR_raise_data(ERR_LIB_USER, ERR_R_PASSED_INVALID_ARGUMENT,
 244                        "BIO must be an SSL BIO");
 245         goto err;
 246     }
 247
 248     if ((conn->streams = lh_H3_STREAM_new(h3_stream_hash, h3_stream_eq)) == NULL)
 249         goto err;
 250
 251     if (SSL_in_before(conn->qconn))
 252         if (SSL_set_alpn_protos(conn->qconn, alpn, sizeof(alpn))) {
 253             /* SSL_set_alpn_protos returns 1 on failure */
 254             ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
 255                            "failed to configure ALPN");
 256             goto err;
 257         }
 258
 259     BIO_set_nbio(conn->qconn_bio, 1);
 260
 261     if (!SSL_set_default_stream_mode(conn->qconn, SSL_DEFAULT_STREAM_MODE_NONE)) {
 262         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
 263                        "failed to configure default stream mode");
 264         goto err;
 265     }
 266
 267     if ((s_ctl_send = h3_conn_create_stream(conn, H3_STREAM_TYPE_CTRL_SEND)) == NULL)
 268         goto err;
 269
 270     if ((s_qpenc_send = h3_conn_create_stream(conn, H3_STREAM_TYPE_QPACK_ENC_SEND)) == NULL)
 271         goto err;
 272
 273     if ((s_qpdec_send = h3_conn_create_stream(conn, H3_STREAM_TYPE_QPACK_DEC_SEND)) == NULL)
 274         goto err;
 275
 276     if (settings == NULL) {
 277         nghttp3_settings_default(&dsettings);
 278         settings = &dsettings;
 279     }
 280
 281     if (callbacks != NULL)
 282         intl_callbacks = *callbacks;
 283
 284     conn->recv_data_cb          = intl_callbacks.recv_data;
 285     conn->stream_close_cb       = intl_callbacks.stream_close;
 286     conn->stop_sending_cb       = intl_callbacks.stop_sending;
 287     conn->reset_stream_cb       = intl_callbacks.reset_stream;
 288     conn->deferred_consume_cb   = intl_callbacks.deferred_consume;
 289
 290     intl_callbacks.recv_data        = h3_conn_recv_data;
 291     intl_callbacks.stream_close     = h3_conn_stream_close;
 292     intl_callbacks.stop_sending     = h3_conn_stop_sending;
 293     intl_callbacks.reset_stream     = h3_conn_reset_stream;
 294     intl_callbacks.deferred_consume = h3_conn_deferred_consume;
 295
 296     ec = nghttp3_conn_client_new(&conn->h3conn, &intl_callbacks, settings,
 297                                  NULL, conn);
 298     if (ec < 0) {
 299         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
 300                        "cannot create nghttp3 connection: %s (%d)",
 301                        nghttp3_strerror(ec), ec);
 302         goto err;
 303     }
 304
 305     ec = nghttp3_conn_bind_control_stream(conn->h3conn, s_ctl_send->id);
 306     if (ec < 0) {
 307         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
 308                        "cannot bind nghttp3 control stream: %s (%d)",
 309                        nghttp3_strerror(ec), ec);
 310         goto err;
 311     }
 312
 313     ec = nghttp3_conn_bind_qpack_streams(conn->h3conn,
 314                                          s_qpenc_send->id,
 315                                          s_qpdec_send->id);
 316     if (ec < 0) {
 317         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
 318                        "cannot bind nghttp3 QPACK streams: %s (%d)",
 319                        nghttp3_strerror(ec), ec);
 320         goto err;
 321     }
 322
 323     return conn;
 324
 325 err:
 326     nghttp3_conn_del(conn->h3conn);
 327     h3_stream_free(s_ctl_send);
 328     h3_stream_free(s_qpenc_send);
 329     h3_stream_free(s_qpdec_send);
 330     lh_H3_STREAM_free(conn->streams);
 331     OPENSSL_free(conn);
 332     return NULL;
 333 }
 334
 335 H3_CONN *H3_CONN_new_for_addr(SSL_CTX *ctx, const char *addr,
 336                               const nghttp3_callbacks *callbacks,
 337                               const nghttp3_settings *settings,
 338                               void *user_data)
 339 {
 340     BIO *qconn_bio = NULL;
 341     SSL *qconn = NULL;
 342     H3_CONN *conn = NULL;
 343     const char *bare_hostname;
 344
 345     /* QUIC connection setup */
 346     if ((qconn_bio = BIO_new_ssl_connect(ctx)) == NULL)
 347         goto err;
 348
 349     if (BIO_set_conn_hostname(qconn_bio, addr) == 0)
 350         goto err;
 351
 352     bare_hostname = BIO_get_conn_hostname(qconn_bio);
 353     if (bare_hostname == NULL)
 354         goto err;
 355
 356     if (BIO_get_ssl(qconn_bio, &qconn) == 0)
 357         goto err;
 358
 359     if (SSL_set1_host(qconn, bare_hostname) <= 0)
 360         goto err;
 361
 362     conn = H3_CONN_new_for_conn(qconn_bio, callbacks, settings, user_data);
 363     if (conn == NULL)
 364         goto err;
 365
 366     return conn;
 367
 368 err:
 369     BIO_free_all(qconn_bio);
 370     return NULL;
 371 }
 372
 373 int H3_CONN_connect(H3_CONN *conn)
 374 {
 375     return SSL_connect(H3_CONN_get0_connection(conn));
 376 }
 377
 378 void *H3_CONN_get_user_data(const H3_CONN *conn)
 379 {
 380     return conn->user_data;
 381 }
 382
 383 SSL *H3_CONN_get0_connection(const H3_CONN *conn)
 384 {
 385     return conn->qconn;
 386 }
 387
 388 /* Pumps received data to the HTTP/3 stack for a single stream. */
 389 static void h3_conn_pump_stream(H3_STREAM *s, void *conn_)
 390 {
 391     int ec;
 392     H3_CONN *conn = conn_;
 393     size_t num_bytes, consumed;
 394     uint64_t aec;
 395
 396     if (!conn->pump_res)
 397         return;
 398
 399     for (;;) {
 400         if (s->s == NULL
 401             || SSL_get_stream_read_state(s->s) == SSL_STREAM_STATE_WRONG_DIR
 402             || s->done_recv_fin)
 403             break;
 404
 405         /*
 406          * Pump data from OpenSSL QUIC to the HTTP/3 stack by calling SSL_peek
 407          * to get received data and passing it to nghttp3 using
 408          * nghttp3_conn_read_stream. Note that this function is confusingly
 409          * named and inputs data to the HTTP/3 stack.
 410          */
 411         if (s->buf_cur == s->buf_total) {
 412             /* Need more data. */
 413             ec = SSL_read_ex(s->s, s->buf, sizeof(s->buf), &num_bytes);
 414             if (ec <= 0) {
 415                 num_bytes = 0;
 416                 if (SSL_get_error(s->s, ec) == SSL_ERROR_ZERO_RETURN) {
 417                     /* Stream concluded normally. Pass FIN to HTTP/3 stack. */
 418                     ec = nghttp3_conn_read_stream(conn->h3conn, s->id, NULL, 0,
 419                                                   /*fin=*/1);
 420                     if (ec < 0) {
 421                         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
 422                                        "cannot pass FIN to nghttp3: %s (%d)",
 423                                        nghttp3_strerror(ec), ec);
 424                         goto err;
 425                     }
 426
 427                     s->done_recv_fin = 1;
 428                 } else if (SSL_get_stream_read_state(s->s)
 429                             == SSL_STREAM_STATE_RESET_REMOTE) {
 430                     /* Stream was reset by peer. */
 431                     if (!SSL_get_stream_read_error_code(s->s, &aec))
 432                         goto err;
 433
 434                     ec = nghttp3_conn_close_stream(conn->h3conn, s->id, aec);
 435                     if (ec < 0) {
 436                         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
 437                                        "cannot mark stream as reset: %s (%d)",
 438                                        nghttp3_strerror(ec), ec);
 439                         goto err;
 440                     }
 441
 442                     s->done_recv_fin = 1;
 443                 } else {
 444                     /* Other error. */
 445                     goto err;
 446                 }
 447             }
 448
 449             s->buf_cur      = 0;
 450             s->buf_total    = num_bytes;
 451         }
 452
 453         if (s->buf_cur == s->buf_total)
 454             break;
 455
 456         assert(conn->consumed_app_data == 0);
 457         ec = nghttp3_conn_read_stream(conn->h3conn, s->id, s->buf + s->buf_cur,
 458                                       s->buf_total - s->buf_cur, /*fin=*/0);
 459         if (ec < 0) {
 460             ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
 461                            "nghttp3 failed to process incoming data: %s (%d)",
 462                            nghttp3_strerror(ec), ec);
 463             goto err;
 464         }
 465
 466         consumed = ec + conn->consumed_app_data;
 467         assert(consumed <= s->buf_total - s->buf_cur);
 468         s->buf_cur += consumed;
 469         conn->consumed_app_data = 0;
 470     }
 471
 472     return;
 473 err:
 474     conn->pump_res = 0;
 475 }
 476
 477 int H3_CONN_handle_events(H3_CONN *conn)
 478 {
 479     int ec, fin;
 480     size_t i, num_vecs, written, total_written, total_len;
 481     int64_t stream_id;
 482     nghttp3_vec vecs[8] = {0};
 483     H3_STREAM key, *s;
 484     SSL *snew;
 485
 486     if (conn == NULL)
 487         return 0;
 488
 489     /* Check for new incoming streams */
 490     for (;;) {
 491         if ((snew = SSL_accept_stream(conn->qconn, SSL_ACCEPT_STREAM_NO_BLOCK)) == NULL)
 492             break;
 493
 494         if (h3_conn_accept_stream(conn, snew) == NULL) {
 495             SSL_free(snew);
 496             return 0;
 497         }
 498     }
 499
 500     /* Pump outgoing data from HTTP/3 engine to QUIC. */
 501     for (;;) {
 502         /* Get a number of send vectors from the HTTP/3 engine. */
 503         ec = nghttp3_conn_writev_stream(conn->h3conn, &stream_id, &fin,
 504                                         vecs, ARRAY_LEN(vecs));
 505         if (ec < 0)
 506             return 0;
 507         if (ec == 0)
 508             break;
 509
 510         /* For each of the vectors returned, pass it to OpenSSL QUIC. */
 511         key.id = stream_id;
 512         if ((s = lh_H3_STREAM_retrieve(conn->streams, &key)) == NULL) {
 513             ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
 514                            "no stream for ID %zd", stream_id);
 515             return 0;
 516         }
 517
 518         num_vecs = ec;
 519         total_len = nghttp3_vec_len(vecs, num_vecs);
 520         total_written = 0;
 521         for (i = 0; i < num_vecs; ++i) {
 522             if (vecs[i].len == 0)
 523                 continue;
 524
 525             if (s->s == NULL) {
 526                 written = vecs[i].len;
 527             } else if (!SSL_write_ex(s->s, vecs[i].base, vecs[i].len, &written)) {
 528                 if (SSL_get_error(s->s, 0) == SSL_ERROR_WANT_WRITE) {
 529                     written = 0;
 530                     nghttp3_conn_block_stream(conn->h3conn, stream_id);
 531                 } else {
 532                     ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
 533                                    "writing HTTP/3 data to network failed");
 534                     return 0;
 535                 }
 536             } else {
 537                 nghttp3_conn_unblock_stream(conn->h3conn, stream_id);
 538             }
 539
 540             total_written += written;
 541             if (written > 0) {
 542                 ec = nghttp3_conn_add_write_offset(conn->h3conn, stream_id, written);
 543                 if (ec < 0)
 544                     return 0;
 545
 546                 ec = nghttp3_conn_add_ack_offset(conn->h3conn, stream_id, written);
 547                 if (ec < 0)
 548                     return 0;
 549             }
 550         }
 551
 552         if (fin && total_written == total_len) {
 553             SSL_stream_conclude(s->s, 0);
 554             if (total_len == 0) {
 555                 ec = nghttp3_conn_add_write_offset(conn->h3conn, stream_id, 0);
 556                 if (ec < 0)
 557                     return 0;
 558             }
 559         }
 560     }
 561
 562     /* Pump incoming data from QUIC to HTTP/3 engine. */
 563     conn->pump_res = 1;
 564     lh_H3_STREAM_doall_arg(conn->streams, h3_conn_pump_stream, conn);
 565     if (!conn->pump_res)
 566         return 0;
 567
 568     return 1;
 569 }
 570
 571 int H3_CONN_submit_request(H3_CONN *conn, const nghttp3_nv *nva, size_t nvlen,
 572                            const nghttp3_data_reader *dr,
 573                            void *user_data)
 574 {
 575     int ec;
 576     H3_STREAM *s_req = NULL;
 577
 578     if (conn == NULL) {
 579         ERR_raise_data(ERR_LIB_USER, ERR_R_PASSED_NULL_PARAMETER,
 580                        "connection must be specified");
 581         return 0;
 582     }
 583
 584     if ((s_req = h3_conn_create_stream(conn, H3_STREAM_TYPE_REQ)) == NULL)
 585         goto err;
 586
 587     s_req->user_data = user_data;
 588
 589     ec = nghttp3_conn_submit_request(conn->h3conn, s_req->id, nva, nvlen,
 590                                      dr, s_req);
 591     if (ec < 0) {
 592         ERR_raise_data(ERR_LIB_USER, ERR_R_INTERNAL_ERROR,
 593                        "cannot submit HTTP/3 request: %s (%d)",
 594                        nghttp3_strerror(ec), ec);
 595         goto err;
 596     }
 597
 598     return 1;
 599
 600 err:
 601     h3_conn_remove_stream(conn, s_req);
 602     return 0;
 603 }