From e3f335148adc6742728ff8fef4a011d42fa5c7b1 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Mon, 3 Oct 2022 17:40:02 +0200 Subject: [PATCH] websockets: remodeled API to support 63 bit frame sizes curl_ws_recv() now receives data to fill up the provided buffer, but can return a partial fragment. The function now also get a pointer to a curl_ws_frame struct with metadata that also mentions the offset and total size of the fragment (of which you might be receiving a smaller piece). This way, large incoming fragments will be "streamed" to the application. When the curl_ws_frame struct field 'bytesleft' is 0, the final fragment piece has been delivered. curl_ws_recv() was also adjusted to work with a buffer size smaller than the fragment size. (Possibly needless to say as the fragment size can now be 63 bit large). curl_ws_send() now supports sending a piece of a fragment, in a streaming manner, in addition to sending the entire fragment in a single call if it is small enough. To send a huge fragment, curl_ws_send() can be used to send it in many small calls by first telling libcurl about the total expected fragment size, and then send the payload in N number of separate invokes and libcurl will stream those over the wire. The struct curl_ws_meta() returns is now called 'curl_ws_frame' and it has been extended with two new fields: *offset* and *bytesleft*. To help describe the passed on data chunk when a fragment is delivered in many smaller pieces. The documentation has been updated accordingly. Closes #9636 --- docs/libcurl/curl_ws_meta.3 | 51 +++-- docs/libcurl/curl_ws_recv.3 | 28 ++- docs/libcurl/curl_ws_send.3 | 27 ++- docs/libcurl/symbols-in-versions | 2 +- include/curl/websockets.h | 22 ++- lib/easy.c | 4 +- lib/easyif.h | 2 +- lib/http.h | 13 +- lib/ws.c | 325 ++++++++++++++++++++----------- lib/ws.h | 19 ++ tests/libtest/lib2301.c | 2 +- tests/libtest/lib2302.c | 4 +- 12 files changed, 326 insertions(+), 173 deletions(-) diff --git a/docs/libcurl/curl_ws_meta.3 b/docs/libcurl/curl_ws_meta.3 index 8536702bda..73107ddc45 100644 --- a/docs/libcurl/curl_ws_meta.3 +++ b/docs/libcurl/curl_ws_meta.3 @@ -29,19 +29,21 @@ curl_ws_meta - meta data WebSocket information .nf #include -struct curl_ws_metadata { - int age; /* zero */ - int recvflags; /* See the CURLWS_* defines */ +struct curl_ws_frame { + int age; /* zero */ + int flags; /* See the CURLWS_* defines */ + curl_off_t offset; /* the offset of this data into the frame */ + curl_off_t bytesleft; /* number of pending bytes left of the payload */ }; -struct curl_ws_metadata *curl_ws_meta(CURL *curl); +struct curl_ws_frame *curl_ws_meta(CURL *curl); .fi .SH DESCRIPTION This function call is EXPERIMENTAL. When the write callback (\fICURLOPT_WRITEFUNCTION(3)\fP) is invoked on received WebSocket traffic, \fIcurl_ws_meta(3)\fP can be called from within -the callback to provide additional information about the data. +the callback to provide additional information about the current frame. This function only works from within the callback, and only when receiving WebSocket data. @@ -54,9 +56,30 @@ to the callback by libcurl itself, applications that want to use .SH "struct fields" .IP age This field specify the age of this struct. It is always zero for now. -.IP recvflags -This is a bitmask with the exact same meaning as the \fBrecvflags\fP -documented for \fIcurl_ws_recv(3)\fP. +.IP flags +This is a bitmask with individual bits set that describes the WebSocket +data. See the list below. +.IP offset +When this frame is a continuation of fragment data already delivered, this is +the offset into the final fragment where this piece belongs. +.IP bytesleft +If this is not a complete fragment, the \fIbytesleft\fP field informs about +how many additional bytes are expected to arrive before this fragment is +complete. +.SH FLAGS +.IP CURLWS_TEXT +The buffer contains text data. Note that this makes a difference to WebSocket +but libcurl itself will not make any verification of the content or +precautions that you actually receive valid UTF-8 content. +.IP CURLWS_BINARY +This is binary data. +.IP CURLWS_CONT +This is not the final fragment of the message, it implies that there will be +another fragment coming as part of the same message. +.IP CURLWS_CLOSE +This transfer is now closed. +.IP CURLWS_PING +This as an incoming ping message, that expects a pong response. .SH EXAMPLE .nf @@ -70,9 +93,9 @@ static size_t writecb(unsigned char *buffer, size_t size, size_t nitems, void *p) { struct customdata *c = (struct customdata *)p; - struct curl_ws_metadata *m = curl_ws_meta(c->easy); + struct curl_ws_frame *m = curl_ws_meta(c->easy); - /* m->recvflags tells us about the traffic */ + /* m->flags tells us about the traffic */ } { @@ -86,10 +109,10 @@ static size_t writecb(unsigned char *buffer, .SH AVAILABILITY Added in 7.86.0. .SH RETURN VALUE -This function returns a pointer to a metadata struct with information that is -valid for this specific callback invocation. If it cannot return this -information, or if the function is called in the wrong context, it returns -NULL. +This function returns a pointer to a \fIcurl_ws_frame\fP struct with +information that is valid for this specific callback invocation. If it cannot +return this information, or if the function is called in the wrong context, it +returns NULL. .SH "SEE ALSO" .BR curl_easy_setopt "(3), " .BR curl_easy_getinfo "(3), " diff --git a/docs/libcurl/curl_ws_recv.3 b/docs/libcurl/curl_ws_recv.3 index 86254e0ffe..3cbb5e6f97 100644 --- a/docs/libcurl/curl_ws_recv.3 +++ b/docs/libcurl/curl_ws_recv.3 @@ -30,28 +30,22 @@ curl_ws_recv - receive WebSocket data #include CURLcode curl_ws_recv(CURL *curl, void *buffer, size_t buflen, - size_t *recv, unsigned int *flags); + size_t *recv, struct curl_ws_frame **meta); .fi .SH DESCRIPTION This function call is EXPERIMENTAL. Retrieves as much as possible of a received WebSocket data fragment into the -\fBbuffer\fP, but not more than \fBbuflen\fP bytes. The provide -\fIflags\fP argument gets bits set to help characterize the fragment. -.SH FLAGS -.IP CURLWS_TEXT -The buffer contains text data. Note that this makes a difference to WebSocket -but libcurl itself will not make any verification of the content or -precautions that you actually receive valid UTF-8 content. -.IP CURLWS_BINARY -This is binary data. -.IP CURLWS_CONT -This is not the final fragment of the message, it implies that there will be -another fragment coming as part of the same message. -.IP CURLWS_CLOSE -This transfer is now closed. -.IP CURLWS_PING -This as an incoming ping message, that expects a pong response. +\fBbuffer\fP, but not more than \fBbuflen\fP bytes. \fIrecv\fP is set to the +number of bytes actually stored. + +If there is more fragment data to deliver than what fits in the provided +\fIbuffer\fP, libcurl returns a full buffer and the application needs to call +this function again to continue draining the buffer. + +The \fImeta\fP pointer gets set to point to a \fIstruct curl_ws_frame\fP that +contains information about the received data. See the \fIcurl_ws_meta(3)\fP +for details on that struct. .SH EXAMPLE .nf diff --git a/docs/libcurl/curl_ws_send.3 b/docs/libcurl/curl_ws_send.3 index 73ebc25403..f7c6dea91d 100644 --- a/docs/libcurl/curl_ws_send.3 +++ b/docs/libcurl/curl_ws_send.3 @@ -24,18 +24,31 @@ .\" .TH curl_ws_send 3 "12 Jun 2022" "libcurl 7.85.0" "libcurl Manual" .SH NAME -curl_ws_send - receive WebSocket data +curl_ws_send - send WebSocket data .SH SYNOPSIS .nf #include CURLcode curl_ws_send(CURL *curl, const void *buffer, size_t buflen, - size_t *sent, unsigned int flags); + size_t *sent, curl_off_t framesize, + unsigned int flags); .fi .SH DESCRIPTION This function call is EXPERIMENTAL. -Send the specific message fragment over the established WebSocket connection. +Send the specific message fragment over an established WebSocket +connection. The \fIbuffer\fP holds the data to send and it is \fIbuflen\fP +number of payload bytes in that memory area. + +\fIsent\fP is returned as the number of payload bytes actually sent. + +To send a (huge) fragment using multiple calls with partial content per +invoke, set the \fICURLWS_OFFSET\fP bit and the \fIframesize\fP argument as +the total expected size for the first part, then set the \fICURLWS_OFFSET\fP +with a zero \fItotalsize\fP for the following parts. + +If not sending a partial fragment or if this is raw mode, \fIframsize\fP +should be set to zero. If \fBCURLWS_RAW_MODE\fP is enabled in \fICURLOPT_WS_OPTIONS(3)\fP, the \fBflags\fP argument should be set to 0. @@ -47,8 +60,6 @@ but libcurl itself will not make any verification of the content or precautions that you actually send valid UTF-8 content. .IP CURLWS_BINARY This is binary data. -.IP CURLWS_NOCOMPRESS -No-op if there’s no compression anyway. .IP CURLWS_CONT This is not the final fragment of the message, which implies that there will be another fragment coming as part of the same message where this bit is not @@ -59,6 +70,12 @@ Close this transfer. This as a ping. .IP CURLWS_PONG This as a pong. +.IP CURLWS_OFFSET +The provided data is only a partial fragment and there will be more in a +following call to \fIcurl_ws_send()\fP. When sending only a piece of the +fragment like this, the \fIframesize\fP must be provided with the total +expected frame size in the first call and it needs to be zero in subsequent +calls. .SH EXAMPLE .nf diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions index ed2b010e94..df7d4eeb2a 100644 --- a/docs/libcurl/symbols-in-versions +++ b/docs/libcurl/symbols-in-versions @@ -1111,7 +1111,7 @@ CURLWARNING 7.66.0 CURLWS_BINARY 7.86.0 CURLWS_CLOSE 7.86.0 CURLWS_CONT 7.86.0 -CURLWS_NOCOMPRESS 7.86.0 +CURLWS_OFFSET 7.86.0 CURLWS_PING 7.86.0 CURLWS_PONG 7.86.0 CURLWS_RAW_MODE 7.86.0 diff --git a/include/curl/websockets.h b/include/curl/websockets.h index e128526f4e..4d57f91e56 100644 --- a/include/curl/websockets.h +++ b/include/curl/websockets.h @@ -28,12 +28,20 @@ extern "C" { #endif -/* generic in/out flag bits */ +struct curl_ws_frame { + int age; /* zero */ + int flags; /* See the CURLWS_* defines */ + curl_off_t offset; /* the offset of this data into the frame */ + curl_off_t bytesleft; /* number of pending bytes left of the payload */ +}; + +/* flag bits */ #define CURLWS_TEXT (1<<0) #define CURLWS_BINARY (1<<1) #define CURLWS_CONT (1<<2) #define CURLWS_CLOSE (1<<3) #define CURLWS_PING (1<<4) +#define CURLWS_OFFSET (1<<5) /* * NAME curl_ws_recv() @@ -44,10 +52,10 @@ extern "C" { * curl_easy_perform() with CURLOPT_CONNECT_ONLY option. */ CURL_EXTERN CURLcode curl_ws_recv(CURL *curl, void *buffer, size_t buflen, - size_t *recv, unsigned int *recvflags); + size_t *recv, + struct curl_ws_frame **metap); /* sendflags for curl_ws_send() */ -#define CURLWS_NOCOMPRESS (1<<5) #define CURLWS_PONG (1<<6) /* @@ -60,17 +68,13 @@ CURL_EXTERN CURLcode curl_ws_recv(CURL *curl, void *buffer, size_t buflen, */ CURL_EXTERN CURLcode curl_ws_send(CURL *curl, const void *buffer, size_t buflen, size_t *sent, + curl_off_t framesize, unsigned int sendflags); /* bits for the CURLOPT_WS_OPTIONS bitmask: */ #define CURLWS_RAW_MODE (1<<0) -struct curl_ws_metadata { - int age; /* zero */ - int recvflags; /* See the CURLWS_* defines */ -}; - -CURL_EXTERN struct curl_ws_metadata *curl_ws_meta(CURL *curl); +CURL_EXTERN struct curl_ws_frame *curl_ws_meta(CURL *curl); #ifdef __cplusplus } diff --git a/lib/easy.c b/lib/easy.c index 93e8acc8df..15f8a6d0d6 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -1235,7 +1235,7 @@ CURLcode curl_easy_recv(struct Curl_easy *data, void *buffer, size_t buflen, * This is the private internal version of curl_easy_send() */ CURLcode Curl_senddata(struct Curl_easy *data, const void *buffer, - size_t buflen, size_t *n) + size_t buflen, ssize_t *n) { curl_socket_t sfd; CURLcode result; @@ -1279,7 +1279,7 @@ CURLcode curl_easy_send(struct Curl_easy *data, const void *buffer, if(Curl_is_in_callback(data)) return CURLE_RECURSIVE_API_CALL; - return Curl_senddata(data, buffer, buflen, n); + return Curl_senddata(data, buffer, buflen, (ssize_t *)n); } /* diff --git a/lib/easyif.h b/lib/easyif.h index a8289e75aa..205382cd07 100644 --- a/lib/easyif.h +++ b/lib/easyif.h @@ -28,7 +28,7 @@ * Prototypes for library-wide functions provided by easy.c */ CURLcode Curl_senddata(struct Curl_easy *data, const void *buffer, - size_t buflen, size_t *n); + size_t buflen, ssize_t *n); #ifdef CURLDEBUG CURL_EXTERN CURLcode curl_easy_perform_ev(struct Curl_easy *easy); diff --git a/lib/http.h b/lib/http.h index 4ba4b2bbb4..f7cbb34244 100644 --- a/lib/http.h +++ b/lib/http.h @@ -202,17 +202,6 @@ struct h3out; /* see ngtcp2 */ #endif /* _WIN32 */ #endif /* USE_MSH3 */ -struct websockets { - bool contfragment; /* set TRUE if the previous fragment sent was not final */ - unsigned char mask[4]; /* 32 bit mask for this connection */ - struct Curl_easy *data; /* used for write callback handling */ - struct dynbuf buf; - size_t usedbuf; /* number of leading bytes in 'buf' the most recent complete - websocket frame uses */ - struct curl_ws_metadata handout; /* the struct storage used for - curl_ws_meta() returns */ -}; - /**************************************************************************** * HTTP unique setup ***************************************************************************/ @@ -240,7 +229,7 @@ struct HTTP { } sending; #ifdef USE_WEBSOCKETS - struct websockets ws; + struct websocket ws; #endif #ifndef CURL_DISABLE_HTTP diff --git a/lib/ws.c b/lib/ws.c index cec3162d9b..7e383704fb 100644 --- a/lib/ws.c +++ b/lib/ws.c @@ -120,6 +120,7 @@ CURLcode Curl_ws_accept(struct Curl_easy *data) struct SingleRequest *k = &data->req; struct HTTP *ws = data->req.p.http; struct connectdata *conn = data->conn; + struct websocket *wsp = &data->req.p.http->ws; CURLcode result; /* Verify the Sec-WebSocket-Accept response. @@ -146,7 +147,7 @@ CURLcode Curl_ws_accept(struct Curl_easy *data) if(result) return result; - infof(data, "Received 101, switch to WebSockets; mask %02x%02x%02x%02x", + infof(data, "Received 101, switch to WebSocket; mask %02x%02x%02x%02x", ws->ws.mask[0], ws->ws.mask[1], ws->ws.mask[2], ws->ws.mask[3]); k->upgr101 = UPGR101_RECEIVED; @@ -154,6 +155,7 @@ CURLcode Curl_ws_accept(struct Curl_easy *data) /* switch off non-blocking sockets */ (void)curlx_nonblock(conn->sock[FIRSTSOCKET], FALSE); + wsp->oleft = 0; return result; } @@ -172,7 +174,7 @@ CURLcode Curl_ws_accept(struct Curl_easy *data) now been delivered to the application */ static void ws_decode_clear(struct Curl_easy *data) { - struct websockets *wsp = &data->req.p.http->ws; + struct websocket *wsp = &data->req.p.http->ws; size_t spent = wsp->usedbuf; size_t len = Curl_dyn_len(&wsp->buf); size_t keep = len - spent; @@ -186,6 +188,7 @@ static void ws_decode_clear(struct Curl_easy *data) ilen - the size of the provided data, perhaps too little, perhaps too much out - stored pointed to extracted data olen - stored length of the extracted data + oleft - number of unread bytes pending to that belongs to this frame endp - stored pointer to data immediately following the parsed data, if there is more data in there. NULL if there's no more data. flags - stored bitmask about the frame @@ -198,16 +201,17 @@ static void ws_decode_clear(struct Curl_easy *data) static CURLcode ws_decode(struct Curl_easy *data, unsigned char *wpkt, size_t ilen, unsigned char **out, size_t *olen, + curl_off_t *oleft, unsigned char **endp, unsigned int *flags) { bool fin; unsigned char opcode; - size_t total; + curl_off_t total; size_t dataindex = 2; - size_t plen; /* size of data in the buffer */ - size_t payloadssize; - struct websockets *wsp = &data->req.p.http->ws; + curl_off_t plen; /* size of data in the buffer */ + curl_off_t payloadsize; + struct websocket *wsp = &data->req.p.http->ws; unsigned char *p; CURLcode result; @@ -266,36 +270,52 @@ static CURLcode ws_decode(struct Curl_easy *data, failf(data, "WS: masked input frame"); return CURLE_RECV_ERROR; } - payloadssize = p[1]; - if(payloadssize == 126) { + payloadsize = p[1]; + if(payloadsize == 126) { if(plen < 4) { infof(data, "WS:%d plen == %u, EAGAIN", __LINE__, (int)plen); return CURLE_AGAIN; /* not enough data available */ } - payloadssize = (p[2] << 8) | p[3]; + payloadsize = (p[2] << 8) | p[3]; dataindex += 2; } - else if(payloadssize == 127) { - failf(data, "WS: too large frame received"); - return CURLE_RECV_ERROR; + else if(payloadsize == 127) { + /* 64 bit payload size */ + if(plen < 10) + return CURLE_AGAIN; + if(p[2] & 80) { + failf(data, "WS: too large frame"); + return CURLE_RECV_ERROR; + } + dataindex += 8; + payloadsize = ((curl_off_t)p[2] << 56) | + (curl_off_t)p[3] << 48 | + (curl_off_t)p[4] << 40 | + (curl_off_t)p[5] << 32 | + (curl_off_t)p[6] << 24 | + (curl_off_t)p[7] << 16 | + (curl_off_t)p[8] << 8 | + p[9]; } - total = dataindex + payloadssize; + total = dataindex + payloadsize; if(total > plen) { - /* not enough data in buffer yet */ - infof(data, "WS:%d plen == %u (%u), EAGAIN", __LINE__, (int)plen, - (int)total); - return CURLE_AGAIN; + /* deliver a partial frame */ + *oleft = total - dataindex; + payloadsize = total - dataindex; } + else + *oleft = 0; /* point to the payload */ *out = &p[dataindex]; /* return the payload length */ - *olen = payloadssize; + *olen = payloadsize; wsp->usedbuf = total; /* number of bytes "used" from the buffer */ *endp = &p[total]; - infof(data, "WS: received %zu bytes payload", payloadssize); + infof(data, "WS: received %zu bytes payload (%zu left)", + payloadsize, *oleft); return CURLE_OK; } @@ -312,40 +332,52 @@ size_t Curl_ws_writecb(char *buffer, size_t size /* 1 */, if(data->set.ws_raw_mode) return data->set.fwrite_func(buffer, size, nitems, writebody_ptr); else if(nitems) { - unsigned char *wsp; - size_t wslen; + unsigned char *frame; + size_t flen; unsigned int recvflags; CURLcode result; unsigned char *endp; + curl_off_t oleft; + decode: - result = ws_decode(data, (unsigned char *)buffer, nitems, - &wsp, &wslen, &endp, &recvflags); - if(result == CURLE_AGAIN) - /* insufficient amount of data, keep it for later */ - return nitems; - else if(result) { - infof(data, "WS: decode error %d", (int)result); - return nitems - 1; + + oleft = ws->ws.frame.bytesleft; + if(!oleft) { + result = ws_decode(data, (unsigned char *)buffer, nitems, + &frame, &flen, &oleft, &endp, &recvflags); + if(result == CURLE_AGAIN) + /* insufficient amount of data, keep it for later */ + return nitems; + else if(result) { + infof(data, "WS: decode error %d", (int)result); + return nitems - 1; + } + /* Store details about the frame to be reachable with curl_ws_meta() + from within the write callback */ + ws->ws.frame.age = 0; + ws->ws.frame.offset = 0; + ws->ws.frame.flags = recvflags; + ws->ws.frame.bytesleft = oleft; + } + else { + ws->ws.frame.bytesleft -= nitems; } /* auto-respond to PINGs */ - if(recvflags & CURLWS_PING) { + if((recvflags & CURLWS_PING) && !oleft) { size_t bytes; infof(data, "WS: auto-respond to PING with a PONG"); /* send back the exact same content as a PONG */ - result = curl_ws_send(data, wsp, wslen, &bytes, CURLWS_PONG); + result = curl_ws_send(data, frame, flen, &bytes, 0, CURLWS_PONG); if(result) return result; } else { - /* Store details about the frame to be reachable with curl_ws_meta() - from within the write callback */ - ws->ws.handout.age = 0; - ws->ws.handout.recvflags = recvflags; - /* deliver the decoded frame to the user callback */ - if(data->set.fwrite_func((char *)wsp, 1, wslen, writebody_ptr) != wslen) + if(data->set.fwrite_func((char *)frame, 1, flen, writebody_ptr) != flen) return 0; } + if(oleft) + ws->ws.frame.offset += flen; /* the websocket frame has been delivered */ ws_decode_clear(data); if(endp) { @@ -359,43 +391,70 @@ size_t Curl_ws_writecb(char *buffer, size_t size /* 1 */, CURL_EXTERN CURLcode curl_ws_recv(struct Curl_easy *data, void *buffer, - size_t buflen, - size_t *nread, unsigned int *recvflags) + size_t buflen, size_t *nread, + struct curl_ws_frame **metap) { size_t bytes; CURLcode result; + struct websocket *wsp = &data->req.p.http->ws; *nread = 0; - *recvflags = 0; + *metap = NULL; /* get a download buffer */ result = Curl_preconnect(data); if(result) return result; do { - result = curl_easy_recv(data, data->state.buffer, - data->set.buffer_size, &bytes); - if(result) - return result; - + bool drain = FALSE; /* if there is pending buffered data to drain */ + char *inbuf = data->state.buffer; + bytes = wsp->stillbuffer; + if(!bytes) { + result = curl_easy_recv(data, data->state.buffer, + data->set.buffer_size, &bytes); + if(result) + return result; + } + else { + /* the pending bytes can be found here */ + inbuf = wsp->stillb; + drain = TRUE; + } if(bytes) { unsigned char *out; size_t olen; unsigned char *endp; + unsigned int recvflags; + curl_off_t oleft = wsp->frame.bytesleft; + infof(data, "WS: got %u websocket bytes to decode", (int)bytes); - result = ws_decode(data, (unsigned char *)data->state.buffer, - bytes, &out, &olen, &endp, recvflags); - if(result == CURLE_AGAIN) - /* a packet fragment only */ - break; - else if(result) - return result; + if(!oleft && !drain) { + result = ws_decode(data, (unsigned char *)inbuf, bytes, + &out, &olen, &oleft, &endp, &recvflags); + if(result == CURLE_AGAIN) + /* a packet fragment only */ + break; + else if(result) + return result; + wsp->frame.offset = 0; + wsp->frame.bytesleft = oleft; + wsp->frame.flags = recvflags; + } + else { + olen = oleft; + recvflags = wsp->frame.flags; + if((curl_off_t)buflen < oleft) + /* there is still data left after this */ + wsp->frame.bytesleft -= buflen; + else + wsp->frame.bytesleft = 0; + } /* auto-respond to PINGs */ - if(*recvflags & CURLWS_PING) { + if((recvflags & CURLWS_PING) && !oleft) { infof(data, "WS: auto-respond to PING with a PONG"); /* send back the exact same content as a PONG */ - result = curl_ws_send(data, out, olen, &bytes, CURLWS_PONG); + result = curl_ws_send(data, out, olen, &bytes, 0, CURLWS_PONG); if(result) return result; } @@ -404,24 +463,44 @@ CURL_EXTERN CURLcode curl_ws_recv(struct Curl_easy *data, void *buffer, /* copy the payload to the user buffer */ memcpy(buffer, out, olen); *nread = olen; + if(!oleft) + /* websocket frame has been delivered */ + ws_decode_clear(data); } else { - /* Received a larger websocket frame than what could fit in the user - provided buffer! */ - infof(data, "WS: too large websocket frame received"); - return CURLE_RECV_ERROR; + /* copy a partial payload */ + memcpy(buffer, out, buflen); + *nread = buflen; + /* remember what is left and where */ + wsp->stillbuffer = olen - buflen; + wsp->stillb = (char *)buffer + buflen; } + wsp->frame.offset += *nread; } - /* the websocket frame has been delivered */ - ws_decode_clear(data); } else *nread = bytes; break; } while(1); + *metap = &wsp->frame; return CURLE_OK; } +static void ws_xor(struct Curl_easy *data, + const unsigned char *source, + unsigned char *dest, + size_t len) +{ + struct websocket *wsp = &data->req.p.http->ws; + size_t i; + /* append payload after the mask, XOR appropriately */ + for(i = 0; i < len; i++) { + dest[i] = source[i] ^ wsp->mask[wsp->xori]; + wsp->xori++; + wsp->xori &= 3; + } +} + /*** RFC 6455 Section 5.2 @@ -445,17 +524,14 @@ CURL_EXTERN CURLcode curl_ws_recv(struct Curl_easy *data, void *buffer, +---------------------------------------------------------------+ */ -static size_t ws_packet(struct Curl_easy *data, - const unsigned char *payload, size_t len, - unsigned int flags) +static size_t ws_packethead(struct Curl_easy *data, + size_t len, unsigned int flags) { struct HTTP *ws = data->req.p.http; unsigned char *out = (unsigned char *)data->state.ulbuf; unsigned char firstbyte = 0; int outi; unsigned char opcode; - unsigned int xori; - unsigned int i; if(flags & CURLWS_TEXT) { opcode = WSBIT_OPCODE_TEXT; infof(data, "WS: send OPCODE TEXT"); @@ -491,8 +567,13 @@ static size_t ws_packet(struct Curl_easy *data, ws->ws.contfragment = TRUE; } out[0] = firstbyte; + if(len > 65535) { + out[1] = 127 | WSBIT_MASK; + out[2] = (len >> 8) & 0xff; + out[3] = len & 0xff; + outi = 10; + } if(len > 126) { - /* no support for > 16 bit fragment sizes */ out[1] = 126 | WSBIT_MASK; out[2] = (len >> 8) & 0xff; out[3] = len & 0xff; @@ -517,112 +598,138 @@ static size_t ws_packet(struct Curl_easy *data, /* pass over the mask */ outi += 4; - /* append payload after the mask, XOR appropriately */ - for(i = 0, xori = 0; i < len; i++, outi++) { - out[outi] = payload[i] ^ ws->ws.mask[xori]; - xori++; - xori &= 3; - } - + ws->ws.xori = 0; /* return packet size */ return outi; } CURL_EXTERN CURLcode curl_ws_send(struct Curl_easy *data, const void *buffer, size_t buflen, size_t *sent, + curl_off_t totalsize, unsigned int sendflags) { - size_t bytes; CURLcode result; - size_t plen; + size_t headlen; char *out; - - if(buflen > MAX_WS_SIZE) { - failf(data, "too large packet"); - return CURLE_BAD_FUNCTION_ARGUMENT; - } + ssize_t written; + struct websocket *wsp = &data->req.p.http->ws; if(!data->set.ws_raw_mode) { result = Curl_get_upload_buffer(data); if(result) return result; } + else { + if(totalsize || sendflags) + return CURLE_BAD_FUNCTION_ARGUMENT; + } - if(Curl_is_in_callback(data)) { - ssize_t written; - if(data->set.ws_raw_mode) { - /* raw mode sends exactly what was requested, and this is from within - the write callback */ + if(data->set.ws_raw_mode) { + if(!buflen) + /* nothing to do */ + return CURLE_OK; + /* raw mode sends exactly what was requested, and this is from within + the write callback */ + if(Curl_is_in_callback(data)) result = Curl_write(data, data->conn->writesockfd, buffer, buflen, &written); - infof(data, "WS: wanted to send %u bytes, sent %u bytes", - (int)buflen, (int)written); + else + result = Curl_senddata(data, buffer, buflen, &written); + + infof(data, "WS: wanted to send %zu bytes, sent %zu bytes", + buflen, written); + *sent = written; + return result; + } + + if(buflen > (data->set.upload_buffer_size - 10)) + /* don't do more than this in one go */ + buflen = data->set.upload_buffer_size - 10; + + if(sendflags & CURLWS_OFFSET) { + if(totalsize) { + /* a frame series 'totalsize' bytes big, this is the first */ + headlen = ws_packethead(data, totalsize, sendflags); + wsp->sleft = totalsize - buflen; } else { - plen = ws_packet(data, buffer, buflen, sendflags); - out = data->state.ulbuf; - result = Curl_write(data, data->conn->writesockfd, out, plen, - &written); - infof(data, "WS: wanted to send %u bytes, sent %u bytes", - (int)plen, (int)written); + headlen = 0; + if((curl_off_t)buflen > wsp->sleft) { + infof(data, "WS: unaligned frame size (sending %zu instead of %zu)", + buflen, wsp->sleft); + wsp->sleft = 0; + } + else + wsp->sleft -= buflen; } - bytes = written; - } - else { - plen = ws_packet(data, buffer, buflen, sendflags); - - out = data->state.ulbuf; - result = Curl_senddata(data, out, plen, &bytes); - (void)sendflags; } - *sent = bytes; + else + headlen = ws_packethead(data, buflen, sendflags); + + /* headlen is the size of the frame header */ + out = data->state.ulbuf; + if(buflen) + /* for PING and PONG etc there might not be a payload */ + ws_xor(data, buffer, (unsigned char *)out + headlen, buflen - headlen); + if(Curl_is_in_callback(data)) + result = Curl_write(data, data->conn->writesockfd, out, + buflen + headlen, &written); + else + result = Curl_senddata(data, out, buflen + headlen, &written); + + infof(data, "WS: wanted to send %zu bytes, sent %zu bytes", + headlen + buflen, written); + *sent = written; return result; } void Curl_ws_done(struct Curl_easy *data) { - struct websockets *wsp = &data->req.p.http->ws; + struct websocket *wsp = &data->req.p.http->ws; DEBUGASSERT(wsp); Curl_dyn_free(&wsp->buf); } -CURL_EXTERN struct curl_ws_metadata *curl_ws_meta(struct Curl_easy *data) +CURL_EXTERN struct curl_ws_frame *curl_ws_meta(struct Curl_easy *data) { - /* we only return something for websockets, called from within the callback + /* we only return something for websocket, called from within the callback when not using raw mode */ if(GOOD_EASY_HANDLE(data) && Curl_is_in_callback(data) && data->req.p.http && !data->set.ws_raw_mode) - return &data->req.p.http->ws.handout; + return &data->req.p.http->ws.frame; return NULL; } #else CURL_EXTERN CURLcode curl_ws_recv(CURL *curl, void *buffer, size_t buflen, - size_t *nread, unsigned int *recvflags) + size_t *nread, + struct curl_ws_frame **metap) { (void)curl; (void)buffer; (void)buflen; (void)nread; - (void)recvflags; + (void)metap; return CURLE_OK; } CURL_EXTERN CURLcode curl_ws_send(CURL *curl, const void *buffer, size_t buflen, size_t *sent, + curl_off_t framesize, unsigned int sendflags) { (void)curl; (void)buffer; (void)buflen; (void)sent; + (void)framesize; (void)sendflags; return CURLE_OK; } -CURL_EXTERN struct curl_ws_metadata *curl_ws_meta(struct Curl_easy *data) +CURL_EXTERN struct curl_ws_frame *curl_ws_meta(struct Curl_easy *data) { (void)data; return NULL; diff --git a/lib/ws.h b/lib/ws.h index 2af5362cb6..341242e50e 100644 --- a/lib/ws.h +++ b/lib/ws.h @@ -36,6 +36,25 @@ /* this is the largest single fragment size we support */ #define MAX_WS_SIZE 65535 +/* part of 'struct HTTP', when used in the 'struct SingleRequest' in the + Curl_easy struct */ +struct websocket { + bool contfragment; /* set TRUE if the previous fragment sent was not final */ + unsigned char mask[4]; /* 32 bit mask for this connection */ + struct Curl_easy *data; /* used for write callback handling */ + struct dynbuf buf; + size_t usedbuf; /* number of leading bytes in 'buf' the most recent complete + websocket frame uses */ + struct curl_ws_frame frame; /* the struct used for frame state */ + curl_off_t oleft; /* outstanding number of payload bytes left from the + server */ + curl_off_t stillbuffer; /* number of bytes left in the buffer to deliver in + the next curl_ws_recv() call */ + char *stillb; /* the stillbuffer pending bytes are here */ + curl_off_t sleft; /* outstanding number of payload bytes left to send */ + unsigned int xori; /* xor index */ +}; + CURLcode Curl_ws_request(struct Curl_easy *data, REQTYPE *req); CURLcode Curl_ws_accept(struct Curl_easy *data); diff --git a/tests/libtest/lib2301.c b/tests/libtest/lib2301.c index 95247933af..90f240e86e 100644 --- a/tests/libtest/lib2301.c +++ b/tests/libtest/lib2301.c @@ -110,7 +110,7 @@ static size_t writecb(char *b, size_t size, size_t nitems, void *p) if(buffer[0] == 0x89) { CURLcode result; fprintf(stderr, "send back a simple PONG\n"); - result = curl_ws_send(easy, pong, 2, &sent, 0); + result = curl_ws_send(easy, pong, 2, &sent, 0, 0); if(result) nitems = 0; } diff --git a/tests/libtest/lib2302.c b/tests/libtest/lib2302.c index 920f83f99f..cb4ac35316 100644 --- a/tests/libtest/lib2302.c +++ b/tests/libtest/lib2302.c @@ -97,7 +97,7 @@ static size_t writecb(char *buffer, size_t size, size_t nitems, void *p) CURL *easy = p; size_t i; size_t incoming = nitems; - struct curl_ws_metadata *meta; + struct curl_ws_frame *meta; (void)size; for(i = 0; i < nitems; i++) printf("%02x ", (unsigned char)buffer[i]); @@ -105,7 +105,7 @@ static size_t writecb(char *buffer, size_t size, size_t nitems, void *p) meta = curl_ws_meta(easy); if(meta) - printf("RECFLAGS: %x\n", meta->recvflags); + printf("RECFLAGS: %x\n", meta->flags); else fprintf(stderr, "RECFLAGS: NULL\n"); -- 2.47.3