MEDIUM: mux-h2: implement the emission of DATA frames from HTX DATA blocks

At the moment the way it's done is not optimal. We should aggregate multiple
blocks into a single DATA frame, and we should merge the ES flag with the
last one when we already know we've reached the end. For now and for an
easier tracking of the HTX stream, an individual empty DATA frame is sent
with the ES bit when EOM is met.

The DATA function is called for DATA, EOD and EOM since these stats indicate
that a previous frame was already produced without the ES flag (typically a
headers frame or another DATA frame). Thus it makes sense to handle all these
blocks there.

There's still an uncertainty on the way the EOD and EOM HTX blocks must be
accounted for, as they're counted as one byte in the HTX stream, but if we
count that byte off when parsing these blocks, we end up sending too much
and desynchronizing the HTX stream. Maybe it hides an issue somewhere else.

At least it's possible to reliably retrieve payloads up to 1 GB over H2/HTX
now. It's still unclear why larger ones are interrupted at 1 GB.

diff --git a/src/mux_h2.c b/src/mux_h2.c
index f7b6517f909c774cf2385da877ae67e15e073405..d475369f418a40f6023464f86d7c99c4becf311c 100644 (file)
--- a/src/mux_h2.c
+++ b/src/mux_h2.c
@@ -3742,6 +3742,191 @@ static size_t h2s_htx_frt_make_resp_headers(struct h2s *h2s, struct htx *htx)
        goto end;
 }
 
+/* Try to send a DATA frame matching HTTP response present in HTX structure
+ * <htx>, for stream <h2s>. Returns the number of bytes sent. The caller must
+ * check the stream's status to detect any error which might have happened
+ * subsequently to a successful send. Returns the number of data bytes
+ * consumed, or zero if nothing done. Note that EOD/EOM count for 1 byte.
+ */
+static size_t h2s_htx_frt_make_resp_data(struct h2s *h2s, struct htx *htx)
+{
+       struct h2c *h2c = h2s->h2c;
+       struct buffer outbuf;
+       size_t total = 0;
+       int es_now = 0;
+       int bsize; /* htx block size */
+       int fsize; /* h2 frame size  */
+       struct htx_blk *blk;
+       enum htx_blk_type type;
+       int idx;
+
+       if (h2c_mux_busy(h2c, h2s)) {
+               h2s->flags |= H2_SF_BLK_MBUSY;
+               goto end;
+       }
+
+       if (!h2_get_buf(h2c, &h2c->mbuf)) {
+               h2c->flags |= H2_CF_MUX_MALLOC;
+               h2s->flags |= H2_SF_BLK_MROOM;
+               goto end;
+       }
+
+       /* We only come here with HTX_BLK_DATA or HTX_BLK_EOD blocks. However,
+        * while looping, we can meet an HTX_BLK_EOM block that we'll leave to
+        * the caller to handle.
+        */
+
+ new_frame:
+       if (htx_is_empty(htx))
+               goto end;
+
+       idx   = htx_get_head(htx);
+       blk   = htx_get_blk(htx, idx);
+       type  = htx_get_blk_type(blk); // DATA or EOD or EOM
+       bsize = htx_get_blksz(blk);
+       fsize = bsize;
+
+       if (type == HTX_BLK_EOD) {
+               /* if we have an EOD, we're dealing with chunked data. We may
+                * have a set of trailers after us that the caller will want to
+                * deal with. Let's simply remove the EOD and return.
+                */
+               htx_remove_blk(htx, blk);
+               // FIXME, it seems we must not return it in the total bytes count?
+               //total++; // EOD counts as one byte
+               goto end;
+       }
+
+       /* for DATA and EOM we'll have to emit a frame, even if empty */
+
+       while (1) {
+               outbuf.area  = b_tail(&h2c->mbuf);
+               outbuf.size = b_contig_space(&h2c->mbuf);
+               outbuf.data = 0;
+
+               if (outbuf.size >= 9 || !b_space_wraps(&h2c->mbuf))
+                       break;
+       realign_again:
+               b_slow_realign(&h2c->mbuf, trash.area, b_data(&h2c->mbuf));
+       }
+
+       if (outbuf.size < 9) {
+               h2c->flags |= H2_CF_MUX_MFULL;
+               h2s->flags |= H2_SF_BLK_MROOM;
+               goto end;
+       }
+
+       /* len: 0x000000 (fill later), type: 0(DATA), flags: none=0 */
+       memcpy(outbuf.area, "\x00\x00\x00\x00\x00", 5);
+       write_n32(outbuf.area + 5, h2s->id); // 4 bytes
+       outbuf.data = 9;
+
+       /* we have in <fsize> the exact number of bytes we need to copy from
+        * the HTX buffer. We need to check this against the connection's and
+        * the stream's send windows, and to ensure that this fits in the max
+        * frame size and in the buffer's available space minus 9 bytes (for
+        * the frame header). The connection's flow control is applied last so
+        * that we can use a separate list of streams which are immediately
+        * unblocked on window opening. Note: we don't implement padding.
+        */
+
+       /* EOM is presented with bsize==1 but would lead to the emission of an
+        * empty frame, thus we force it to zero here.
+        */
+       if (type == HTX_BLK_EOM)
+               bsize = fsize = 0;
+
+       if (!fsize)
+               goto send_empty;
+
+       if (h2s->mws <= 0) {
+               h2s->flags |= H2_SF_BLK_SFCTL;
+               if (h2s->send_wait) {
+                       LIST_DEL(&h2s->list);
+                       LIST_INIT(&h2s->list);
+               }
+               goto end;
+       }
+
+       if (fsize > h2s->mws)
+               fsize = h2s->mws; // >0
+
+       if (h2c->mfs && fsize > h2c->mfs)
+               fsize = h2c->mfs; // >0
+
+       if (fsize + 9 > outbuf.size) {
+               /* we have an opportunity for enlarging the too small
+                * available space, let's try.
+                * FIXME: is this really interesting to do? Maybe we'll
+                * spend lots of time realigning instead of using two
+                * frames.
+                */
+               if (b_space_wraps(&h2c->mbuf))
+                       goto realign_again;
+               fsize = outbuf.size - 9;
+
+               if (fsize <= 0) {
+                       /* no need to send an empty frame here */
+                       h2c->flags |= H2_CF_MUX_MFULL;
+                       h2s->flags |= H2_SF_BLK_MROOM;
+                       goto end;
+               }
+       }
+
+       if (h2c->mws <= 0) {
+               h2s->flags |= H2_SF_BLK_MFCTL;
+               goto end;
+       }
+
+       if (fsize > h2c->mws)
+               fsize = h2c->mws;
+
+       /* now let's copy this this into the output buffer */
+       memcpy(outbuf.area + 9, htx_get_blk_ptr(htx, blk), fsize);
+
+ send_empty:
+       /* update the frame's size */
+       h2_set_frame_size(outbuf.area, fsize);
+
+       /* FIXME: for now we only set the ES flag on empty DATA frames, once
+        * meeting EOM. We should optimize this later.
+        */
+       if (type == HTX_BLK_EOM) {
+               // FIXME, it seems we must not return it in the total bytes count?
+               // total++; // EOM counts as one byte
+               es_now = 1;
+       }
+
+       if (es_now)
+               outbuf.area[4] |= H2_F_DATA_END_STREAM;
+
+       /* commit the H2 response */
+       b_add(&h2c->mbuf, fsize + 9);
+
+       /* consume incoming HTX block, including EOM */
+       total += fsize;
+       if (fsize == bsize) {
+               htx_remove_blk(htx, blk);
+               if (fsize)
+                       goto new_frame;
+       } else {
+               /* we've truncated this block */
+               htx_cut_data_blk(htx, blk, fsize);
+       }
+
+       if (es_now) {
+               if (h2s->st == H2_SS_OPEN)
+                       h2s->st = H2_SS_HLOC;
+               else
+                       h2s_close(h2s);
+
+               h2s->flags |= H2_SF_ES_SENT;
+       }
+
+ end:
+       return total;
+}
+
 /* Called from the upper layer, to subscribe to events, such as being able to send */
 static int h2_subscribe(struct conn_stream *cs, int event_type, void *param)
 {
@@ -3930,6 +4115,19 @@ static size_t h2_snd_buf(struct conn_stream *cs, struct buffer *buf, size_t coun
                                }
                                break;
 
+                       case HTX_BLK_DATA:
+                       case HTX_BLK_EOD:
+                       case HTX_BLK_EOM:
+                               /* all these cause the emission of a DATA frame (possibly empty) */
+                               ret = h2s_htx_frt_make_resp_data(h2s, htx);
+                               if (ret > 0) {
+                                       total += ret;
+                                       count -= ret;
+                                       if (ret < bsize)
+                                               goto done;
+                               }
+                               break;
+
                        default:
                                htx_remove_blk(htx, blk);
                                total += bsize;