From: rousskov <> Date: Wed, 3 Jun 1998 03:38:02 +0000 (+0000) Subject: - Added basic support for Range requests. X-Git-Tag: SQUID_3_0_PRE1~3197 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=d192d11fc9afd7fc368f54b50ad7b977ef74614e;p=thirdparty%2Fsquid.git - Added basic support for Range requests. --- diff --git a/ChangeLog b/ChangeLog index d5735b7196..c71a0a27d0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,12 @@ + - Added basic support for Range requests. For most cachable requests, + Squid replies with an "Accept-Ranges" header. Upon receiving a + potentially cachable Range request for a not cached object, Squid + requests the whole object from origin server and then replies with + specified range(s) to the client. Multi-range requests are + supported. Limitations: If-Range header is not supported and + ignored; Multi-range requests with out of order or overlapping + ranges are not supported. + Changes to squid-1.2.beta22 (June 1, 1998): - do not cut off "; parameter" from "digitized" Content-Type diff --git a/doc/Programming-Guide/prog-guide.sgml b/doc/Programming-Guide/prog-guide.sgml index 52f30b557f..a4a266f21f 100644 --- a/doc/Programming-Guide/prog-guide.sgml +++ b/doc/Programming-Guide/prog-guide.sgml @@ -429,6 +429,233 @@ Squid consists of the following major components (the ``standard'' mode), or only specific request headers will be allowed (the ``paranoid'' mode). + + +HTTP Headers +

+ + General remarks + +

+ + Most operations on Life cycle + +

+ + /* declare */ + HttpHeader hdr; + + /* initialize (as an HTTP Request header) */ + httpHeaderInit(&hdr, hoRequest); + + /* do something */ + ... + + /* cleanup */ + httpHeaderClean(&hdr); + + +

+ Prior to use, an + Once initialized, the + Note that there are no methods for "creating" or "destroying" a + "dynamic" Header Manipulation. + +

+ The mostly common operations on HTTP headers are testing for a particular + header-field ( + + + Special care must be taken when several header-fields with the same id are + preset in the header. If HTTP protocol allows only one copy of the specified + field per header (e.g. "Content-Length"), + It is prohibited to ask for a List of values when only one value is permitted, + and visa-versa. This restriction prevents a programmer from processing one + value of an header-field while ignoring other valid values. + +

+ + The value being put using one of the + Example: + + + /* add our own Age field if none was added before */ + int age = ... + if (!httpHeaderHas(hdr, HDR_AGE)) + httpHeaderPutInt(hdr, HDR_AGE, age); + + +

+ There are two ways to delete a field from a header. To delete a "known" field + (a field with "id" other than + + The + /* delete all fields with a given name */ + HttpHeaderPos pos = HttpHeaderInitPos; + HttpHeaderEntry *e; + while ((e = httpHeaderGetEntry(hdr, &pos))) { + if (!strCaseCmp(e->name, name)) + ... /* delete entry */ + } + + + Note that I/O and Headers. + +

+ To store a header in a file or socket, pack it using Adding new header-field ids. + +

+ Adding new ids is simple. First add new HDR_ entry to the http_hdr_type + enumeration in enums.h. Then describe a new header-field attributes in + the HeadersAttrs array located in + Finally, add new id to one of the following arrays: + + Also, if the new field is a "list" header, add it to the + + In most cases, if you forget to include a new field id in one of the required + arrays, you will get a run-time assertion. For rarely used fields, however, it + may take a long time for an assertion to be triggered. + +

+ There is virtually no limit on the number of fields supported by Squid. If + current mask sizes cannot fit all the ids (you will get an assertion if that + happens), simply enlarge HttpHeaderMask type in A Word on Efficiency. + +

+ + Adding new fields is somewhat expensive if they require complex conversions to + a string. + +

+ Deleting existing fields requires scan of all the entries and comparing their + "id"s (faster) or "names" (slower) with the one specified for deletion. + +

+ Most of the operations are faster than their "ascii string" equivalents. + + + Internet Cache Protocol

elength); } + +void +httpHdrContRangeSet(HttpHdrContRange *cr, HttpHdrRangeSpec spec, size_t ent_len) +{ + assert(cr && ent_len >= 0); + cr->spec = spec; + cr->elength = ent_len; +} diff --git a/src/HttpHdrRange.cc b/src/HttpHdrRange.cc index e6595d2c61..2aca85bb98 100644 --- a/src/HttpHdrRange.cc +++ b/src/HttpHdrRange.cc @@ -1,6 +1,6 @@ /* - * $Id: HttpHdrRange.cc,v 1.8 1998/05/22 23:43:51 wessels Exp $ + * $Id: HttpHdrRange.cc,v 1.9 1998/06/02 21:38:04 rousskov Exp $ * * DEBUG: section 64 HTTP Range Header * AUTHOR: Alex Rousskov @@ -147,6 +147,8 @@ httpHdrRangeSpecPackInto(const HttpHdrRangeSpec * spec, Packer * p) static int httpHdrRangeSpecCanonize(HttpHdrRangeSpec * spec, size_t clen) { + debug(64, 5) ("httpHdrRangeSpecCanonize: have: [%d, %d) len: %d\n", + spec->offset, spec->offset+spec->length, spec->length); if (!known_spec(spec->offset)) /* suffix */ spec->offset = size_diff(clen, spec->length); else if (!known_spec(spec->length)) /* trailer */ @@ -156,6 +158,8 @@ httpHdrRangeSpecCanonize(HttpHdrRangeSpec * spec, size_t clen) assert(known_spec(spec->offset)); spec->length = size_min(size_diff(clen, spec->offset), spec->length); /* check range validity */ + debug(64, 5) ("httpHdrRangeSpecCanonize: done: [%d, %d) len: %d\n", + spec->offset, spec->offset+spec->length, spec->length); return spec->length > 0; } @@ -209,7 +213,7 @@ httpHdrRangeParseInit(HttpHdrRange * range, const String * str) stackPush(&range->specs, spec); count++; } - debug(68, 8) ("parsed range range count: %d\n", range->specs.count); + debug(64, 8) ("parsed range range count: %d\n", range->specs.count); return range->specs.count; } @@ -260,17 +264,24 @@ httpHdrRangePackInto(const HttpHdrRange * range, Packer * p) int httpHdrRangeCanonize(HttpHdrRange * range, size_t clen) { + int bad_count = 0; int i; assert(range); - for (i = 0; i < range->specs.count; i++) + assert(clen >= 0); + for (i = 0; i < range->specs.count; i++) { if (!httpHdrRangeSpecCanonize(range->specs.items[i], clen)) - return 0; - return range->specs.count; + bad_count++; + } + if (bad_count) + debug(64, 2) ("httpHdrRangeCanonize: found %d invalid specs out of %d\n", bad_count, range->specs.count); + else + debug(64, 3) ("httpHdrRangeCanonize: got %d valid specs\n", range->specs.count); + return !bad_count; } /* searches for next range, returns true if found */ int -httpHdrRangeGetSpec(const HttpHdrRange * range, HttpHdrRangeSpec * spec, int *pos) +httpHdrRangeGetSpec(const HttpHdrRange * range, HttpHdrRangeSpec * spec, HttpHdrRangePos *pos) { assert(range && spec); assert(pos && *pos >= -1 && *pos < range->specs.count); @@ -282,3 +293,35 @@ httpHdrRangeGetSpec(const HttpHdrRange * range, HttpHdrRangeSpec * spec, int *po spec->offset = spec->length = 0; return 0; } + +/* hack: returns true if range specs are too "complex" for Squid to handle */ +int +httpHdrRangeIsComplex(const HttpHdrRange * range) +{ + HttpHdrRangePos pos = HttpHdrRangeInitPos; + HttpHdrRangeSpec spec; + size_t offset = 0; + assert(range); + /* check that all rangers are in "strong" order */ + while (httpHdrRangeGetSpec(range, &spec, &pos)) { + if (spec.offset < offset) + return 1; + offset = spec.offset + spec.length; + } + return 0; +} + +/* generates a "unique" boundary string for multipart responses + * the caller is responsible for cleaning the string */ +String +httpHdrRangeBoundaryStr(clientHttpRequest * http) +{ + const char *key; + String b = StringNull; + assert(http); + stringAppend(&b, full_appname_string, strlen(full_appname_string)); + stringAppend(&b, ":", 1); + key = storeKeyText(http->entry->key); + stringAppend(&b, key, strlen(key)); + return b; +} diff --git a/src/HttpHeader.cc b/src/HttpHeader.cc index 739a15638a..f655d837a2 100644 --- a/src/HttpHeader.cc +++ b/src/HttpHeader.cc @@ -1,6 +1,6 @@ /* - * $Id: HttpHeader.cc,v 1.39 1998/06/02 04:28:09 rousskov Exp $ + * $Id: HttpHeader.cc,v 1.40 1998/06/02 21:38:05 rousskov Exp $ * * DEBUG: section 55 HTTP Header * AUTHOR: Alex Rousskov @@ -184,7 +184,7 @@ static http_hdr_type ReplyHeadersArr[] = HDR_MIME_VERSION, HDR_PUBLIC, HDR_RETRY_AFTER, HDR_SERVER, HDR_SET_COOKIE, HDR_VARY, HDR_WARNING, HDR_PROXY_CONNECTION, HDR_X_CACHE, - HDR_X_CACHE_LOOKUP, + HDR_X_CACHE_LOOKUP, HDR_X_REQUEST_URI, HDR_X_SQUID_ERROR }; @@ -336,7 +336,7 @@ httpHeaderAppend(HttpHeader * dest, const HttpHeader * src) /* use fresh entries to replace old ones */ void -httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask) +httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask *denied_mask) { const HttpHeaderEntry *e; HttpHeaderPos pos = HttpHeaderInitPos; @@ -509,7 +509,7 @@ httpHeaderDelById(HttpHeader * hdr, http_hdr_type id) debug(55, 8) ("%p del-by-id %d\n", hdr, id); assert(hdr); assert_eid(id); - assert_eid(id != HDR_OTHER); /* does not make sense */ + assert_eid(id != HDR_OTHER); /* does not make sense */ if (!CBIT_TEST(hdr->mask, id)) return 0; while ((e = httpHeaderGetEntry(hdr, &pos))) { @@ -654,6 +654,44 @@ httpHeaderPutCc(HttpHeader * hdr, const HttpHdrCc * cc) memBufClean(&mb); } +void +httpHeaderPutContRange(HttpHeader * hdr, const HttpHdrContRange *cr) +{ + MemBuf mb; + Packer p; + assert(hdr && cr); + /* remove old directives if any */ + httpHeaderDelById(hdr, HDR_CONTENT_RANGE); + /* pack into mb */ + memBufDefInit(&mb); + packerToMemInit(&p, &mb); + httpHdrContRangePackInto(cr, &p); + /* put */ + httpHeaderAddEntry(hdr, httpHeaderEntryCreate(HDR_CONTENT_RANGE, NULL, mb.buf)); + /* cleanup */ + packerClean(&p); + memBufClean(&mb); +} + +void +httpHeaderPutRange(HttpHeader * hdr, const HttpHdrRange *range) +{ + MemBuf mb; + Packer p; + assert(hdr && range); + /* remove old directives if any */ + httpHeaderDelById(hdr, HDR_CONTENT_RANGE); + /* pack into mb */ + memBufDefInit(&mb); + packerToMemInit(&p, &mb); + httpHdrRangePackInto(range, &p); + /* put */ + httpHeaderAddEntry(hdr, httpHeaderEntryCreate(HDR_RANGE, NULL, mb.buf)); + /* cleanup */ + packerClean(&p); + memBufClean(&mb); +} + /* add extension header (these fields are not parsed/analyzed/joined, etc.) */ void httpHeaderPutExt(HttpHeader * hdr, const char *name, const char *value) @@ -741,22 +779,20 @@ httpHeaderGetCc(const HttpHeader * hdr) HttpHdrRange * httpHeaderGetRange(const HttpHeader * hdr) { - HttpHdrRange *r; - String s; - if (!CBIT_TEST(hdr->mask, HDR_RANGE)) - return NULL; - s = httpHeaderGetList(hdr, HDR_RANGE); - r = httpHdrRangeParseCreate(&s); - httpHeaderNoteParsedEntry(HDR_RANGE, s, !r); - stringClean(&s); + HttpHdrRange *r = NULL; + HttpHeaderEntry *e; + if ((e = httpHeaderFindEntry(hdr, HDR_RANGE))) { + r = httpHdrRangeParseCreate(&e->value); + httpHeaderNoteParsedEntry(HDR_RANGE, e->value, !r); + } return r; } HttpHdrContRange * httpHeaderGetContRange(const HttpHeader * hdr) { - HttpHeaderEntry *e; HttpHdrContRange *cr = NULL; + HttpHeaderEntry *e; if ((e = httpHeaderFindEntry(hdr, HDR_CONTENT_RANGE))) { cr = httpHdrContRangeParseCreate(strBuf(e->value)); httpHeaderNoteParsedEntry(e->id, e->value, !cr); diff --git a/src/HttpHeaderTools.cc b/src/HttpHeaderTools.cc index 9547c9c43f..801e0b730e 100644 --- a/src/HttpHeaderTools.cc +++ b/src/HttpHeaderTools.cc @@ -1,5 +1,5 @@ /* - * $Id: HttpHeaderTools.cc,v 1.17 1998/06/02 04:18:13 wessels Exp $ + * $Id: HttpHeaderTools.cc,v 1.18 1998/06/02 21:38:06 rousskov Exp $ * * DEBUG: section 66 HTTP Header Tools * AUTHOR: Alex Rousskov @@ -143,6 +143,17 @@ httpHeaderPutStrvf(HttpHeader * hdr, http_hdr_type id, const char *fmt, va_list } +/* wrapper arrounf PutContRange */ +void +httpHeaderAddContRange(HttpHeader * hdr, HttpHdrRangeSpec spec, size_t ent_len) +{ + HttpHdrContRange *cr = httpHdrContRangeCreate(); + assert(hdr && ent_len >= 0); + httpHdrContRangeSet(cr, spec, ent_len); + httpHeaderPutContRange(hdr, cr); + httpHdrContRangeDestroy(cr); +} + /* * return true if a given directive is found in at least one of the "connection" header-fields * note: if HDR_PROXY_CONNECTION is present we ignore HDR_CONNECTION diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc index d61229ffc7..0f27e22010 100644 --- a/src/HttpRequest.cc +++ b/src/HttpRequest.cc @@ -1,6 +1,6 @@ /* - * $Id: HttpRequest.cc,v 1.6 1998/06/02 04:18:14 wessels Exp $ + * $Id: HttpRequest.cc,v 1.7 1998/06/02 21:38:06 rousskov Exp $ * * DEBUG: section 73 HTTP Request * AUTHOR: Duane Wessels @@ -57,6 +57,8 @@ requestDestroy(request_t * req) httpHeaderClean(&req->header); if (req->cache_control) httpHdrCcDestroy(req->cache_control); + if (req->range) + httpHdrRangeDestroy(req->range); memFree(MEM_REQUEST_T, req); } diff --git a/src/client_side.cc b/src/client_side.cc index ebd25684b4..707d484baf 100644 --- a/src/client_side.cc +++ b/src/client_side.cc @@ -1,6 +1,6 @@ /* - * $Id: client_side.cc,v 1.322 1998/06/02 04:18:18 wessels Exp $ + * $Id: client_side.cc,v 1.323 1998/06/02 21:38:07 rousskov Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -661,6 +661,7 @@ httpRequestFree(void *data) safe_free(http->log_uri); safe_free(http->al.headers.request); safe_free(http->al.headers.reply); + stringClean(&http->range_iter.boundary); if (entry) { http->entry = NULL; storeUnregister(entry, http); @@ -750,11 +751,17 @@ clientInterpretRequestHeaders(clientHttpRequest * http) EBIT_SET(request->flags, REQ_NOCACHE); stringClean(&s); } +#if OLD_CODE if (httpHeaderHas(req_hdr, HDR_RANGE)) { EBIT_SET(request->flags, REQ_NOCACHE); EBIT_SET(request->flags, REQ_RANGE); /* Request-Range: deleted, not in the specs. Does it exist? */ } +#else + request->range = httpHeaderGetRange(req_hdr); + if (request->range) + EBIT_SET(request->flags, REQ_RANGE); +#endif if (httpHeaderHas(req_hdr, HDR_AUTHORIZATION)) EBIT_SET(request->flags, REQ_AUTH); if (request->login[0] != '\0') @@ -1050,6 +1057,9 @@ clientBuildReplyHeader(clientHttpRequest * http, } #endif +/* filters out unwanted entries from original reply header + * adds extra entries if we have more info than origin server + * adds Squid specific entries */ static void clientBuildReplyHeader(clientHttpRequest * http, HttpReply * rep) { @@ -1057,7 +1067,7 @@ clientBuildReplyHeader(clientHttpRequest * http, HttpReply * rep) int is_hit = isTcpHit(http->log_type); #if DONT_FILTER_THESE /* but you might want to if you run Squid as an HTTP accelerator */ - httpHeaderDelById(hdr, HDR_ACCEPT_RANGES); + /* httpHeaderDelById(hdr, HDR_ACCEPT_RANGES); */ httpHeaderDelById(hdr, HDR_ETAG); #endif httpHeaderDelById(hdr, HDR_PROXY_CONNECTION); @@ -1072,6 +1082,8 @@ clientBuildReplyHeader(clientHttpRequest * http, HttpReply * rep) String strConnection = httpHeaderGetList(hdr, HDR_CONNECTION); const HttpHeaderEntry *e; HttpHeaderPos pos = HttpHeaderInitPos; + /* think: on-average-best nesting of the two loops (hdrEntry and strListItem) @?@ */ + /* maybe we should delete standard stuff ("keep-alive","close") from strConnection first? */ while ((e = httpHeaderGetEntry(hdr, &pos))) { if (strListIsMember(&strConnection, strBuf(e->name), ',')) httpHeaderDelAt(hdr, pos); @@ -1079,6 +1091,64 @@ clientBuildReplyHeader(clientHttpRequest * http, HttpReply * rep) httpHeaderDelById(hdr, HDR_CONNECTION); stringClean(&strConnection); } + /* Handle Ranges */ + /* move this "if" to a separate function! @?@ @?@ */ + if (http->request->range) { + const int spec_count = http->request->range->specs.count; + const char *range_err = NULL; + debug(33, 1) ("clientBuildReplyHeader: range spec count: %d clen: %d\n", + spec_count, rep->content_length); + /* check if we still want to do ranges */ + if (rep->sline.status != HTTP_OK) + range_err = "wrong status code"; + else + if (httpHeaderHas(hdr, HDR_CONTENT_RANGE)) + range_err = "origin server does ranges"; + else + if (rep->content_length < 0) + range_err = "unknown length"; + else + if (rep->content_length != http->entry->mem_obj->reply->content_length) + range_err = "INCONSISTENT length"; /* bug? */ + else + if (!httpHdrRangeCanonize(http->request->range, rep->content_length)) + range_err = "canonization failed"; + else + if (httpHdrRangeIsComplex(http->request->range)) + range_err = "too complex range header"; + /* get rid of our range specs on error */ + if (range_err) { + debug(33, 1) ("clientBuildReplyHeader: will not do ranges: %s.\n", range_err); + httpHdrRangeDestroy(http->request->range); + http->request->range = NULL; + } + /* if we still want to do ranges, append appropriate header(s) */ + if (http->request->range) { + assert(spec_count > 0); + if (spec_count == 1) { + HttpHdrRangePos pos = HttpHdrRangeInitPos; + HttpHdrRangeSpec spec; + assert(httpHdrRangeGetSpec(http->request->range, &spec, &pos)); + /* append Content-Range */ + httpHeaderAddContRange(hdr, spec, rep->content_length); + /* set new Content-Length to the actual number of OCTETs + * transmitted in the message-body */ + httpHeaderDelById(hdr, HDR_CONTENT_LENGTH); + httpHeaderPutInt(hdr, HDR_CONTENT_LENGTH, spec.length); + debug(33, 1) ("clientBuildReplyHeader: actual content length: %d\n", spec.length); + } else { + /* multipart! */ + /* generate boundary string */ + http->range_iter.boundary = httpHdrRangeBoundaryStr(http); + /* delete old Content-Type, add ours */ + httpHeaderDelById(hdr, HDR_CONTENT_TYPE); + httpHeaderPutStrf(hdr, HDR_CONTENT_TYPE, + "multipart/byteranges; boundary=\"%s\"", + strBuf(http->range_iter.boundary)); + /* no need for Content-Length in multipart responses */ + } + } + } /* Append X-Cache */ httpHeaderPutStrf(hdr, HDR_X_CACHE, "%s from %s", is_hit ? "HIT" : "MISS", getMyHostname()); @@ -1097,6 +1167,9 @@ clientBuildReplyHeader(clientHttpRequest * http, HttpReply * rep) httpHeaderPutStr(hdr, http->flags.accel ? HDR_CONNECTION : HDR_PROXY_CONNECTION, "keep-alive"); + /* Accept-Range header for cached objects if not there already */ + if (is_hit && !httpHeaderHas(hdr, HDR_ACCEPT_RANGES)) + httpHeaderPutStr(hdr, HDR_ACCEPT_RANGES, "bytes"); #if ADD_X_REQUEST_URI /* * Knowing the URI of the request is useful when debugging persistent @@ -1119,6 +1192,9 @@ clientBuildReply(clientHttpRequest * http, const char *buf, size_t size) rep->sline.version = 1.0; /* do header conversions */ clientBuildReplyHeader(http, rep); + /* if we do ranges, change status to "Partial Content" */ + if (http->request->range) + httpStatusLineSet(&rep->sline, rep->sline.version, HTTP_PARTIAL_CONTENT, NULL); } else { /* parsing failure, get rid of the invalid reply */ httpReplyDestroy(rep); @@ -1150,6 +1226,118 @@ clientCacheHit(void *data, char *buf, ssize_t size) } } + +/* extracts a "range" from *buf and appends them to mb, updating all offsets and such */ +static void +clientPackRange(clientHttpRequest *http, HttpHdrRangeIter *i, const char **buf, ssize_t *size, MemBuf *mb) +{ + const size_t copy_sz = i->debt_size <= *size ? i->debt_size : *size; + off_t body_off = http->out.offset - i->prefix_size; + assert(*size > 0); + /* intersection of "have" and "need" ranges must not be empty */ + assert(body_off < i->spec.offset + i->spec.length); + assert(body_off + *size > i->spec.offset); + /* put boundary and headers at the beginning of range in a multi-range */ + if (http->request->range->specs.count > 1 && i->debt_size == i->spec.length) { + HttpReply *rep = http->entry->mem_obj ? /* original reply */ + http->entry->mem_obj->reply : NULL; + HttpHeader hdr; + Packer p; + assert(rep); + /* put boundary */ + debug(33,1) ("clientPackRange: appending boundary: %s\n", strBuf(i->boundary)); + /* rfc2046 requires to _prepend_ boundary with ! */ + memBufPrintf(mb, "\r\n--%s\r\n", strBuf(i->boundary)); + httpHeaderInit(&hdr, hoReply); + if (httpHeaderHas(&rep->header, HDR_CONTENT_TYPE)) + httpHeaderPutStr(&hdr, HDR_CONTENT_TYPE, httpHeaderGetStr(&rep->header, HDR_CONTENT_TYPE)); + httpHeaderAddContRange(&hdr, i->spec, rep->content_length); + packerToMemInit(&p, mb); + httpHeaderPackInto(&hdr, &p); + packerClean(&p); + httpHeaderClean(&hdr); + /* append (we packed a header, not a reply */ + memBufPrintf(mb, "\r\n"); + } + /* append */ + debug(33,1) ("clientPackRange: appending %d bytes\n", copy_sz); + memBufAppend(mb, *buf, copy_sz); + /* update offsets */ + *size -= copy_sz; + i->debt_size -= copy_sz; + body_off += copy_sz; + *buf += copy_sz; + http->out.offset = body_off + i->prefix_size; /* sync */ + /* paranoid check */ + assert(*size >= 0 && i->debt_size >= 0); +} + + +/* extracts "ranges" from buf and appends them to mb, updating all offsets and such */ +/* returns true if we need more data */ +static int +clientPackMoreRanges(clientHttpRequest *http, const char *buf, ssize_t size, MemBuf *mb) +{ + HttpHdrRangeIter *i = &http->range_iter; + int need_more = i->debt_size > 0; + /* offset in range specs does not count the prefix of an http msg */ + off_t body_off = http->out.offset - i->prefix_size; + assert(size >= 0); + /* filter out data according to range specs */ + /* note: order of loop conditions is significant! */ + while ((i->debt_size || (need_more = httpHdrRangeGetSpec(http->request->range, &i->spec, &i->pos))) && + size > 0) { + off_t start; /* offset of still missing data */ + if (!i->debt_size) + i->debt_size = i->spec.length; + if (!i->debt_size) + continue; + start = i->spec.offset + i->spec.length - i->debt_size; + debug(33,1) ("clientPackMoreRanges: in: offset: %d size: %d\n", + (int)body_off, size); + debug(33,1) ("clientPackMoreRanges: out: start: %d spec[%d]: [%d, %d), len: %d debt: %d\n", + (int)start, (int)i->pos, i->spec.offset, (int)(i->spec.offset+i->spec.length), i->spec.length, i->debt_size); + assert(body_off <= start); /* we did not miss it */ + /* skip up to start */ + if (body_off + size > start) { + const size_t skip_size = start - body_off; + body_off = start; + size -= skip_size; + buf += skip_size; + } else { + /* has not reached start yet */ + body_off += size; + size = 0; + buf = NULL; + } + /* put next chunk if any */ + if (size) { + http->out.offset = body_off + i->prefix_size; /* sync */ + clientPackRange(http, i, &buf, &size, mb); + body_off = http->out.offset - i->prefix_size; /* sync */ + } + } + debug(33,1) ("clientPackMoreRanges: buf exhausted: in: offset: %d size: %d need_more: %d\n", + (int)body_off, size, need_more); + debug(33,1) ("clientPackMoreRanges: spec[%d]: [%d, %d), len: %d debt: %d\n", + (int)i->pos, i->spec.offset, (int)(i->spec.offset+i->spec.length), i->spec.length, i->debt_size); + /* skip the data we do not need */ + /* maybe, we have not seen that data yet! */ + if (need_more) { + if (i->debt_size == i->spec.length) /* at the start of the cur. spec */ + body_off = i->spec.offset; + else + assert(body_off == i->spec.offset + i->spec.length - i->debt_size); + } else + if (http->request->range->specs.count > 1) { + /* put terminating boundary for multiparts */ + memBufPrintf(mb, "\r\n--%s--\r\n", strBuf(i->boundary)); + } + + http->out.offset = body_off + i->prefix_size; /* sync */ + return need_more; +} + /* * accepts chunk of a http message in buf, parses prefix, filters headers and * such, writes processed message to the client's socket @@ -1241,6 +1429,7 @@ clientSendMoreData(void *data, char *buf, ssize_t size) body_size = size - rep->hdr_sz; assert(body_size >= 0); body_buf = buf + rep->hdr_sz; + http->range_iter.prefix_size = rep->hdr_sz; debug(33, 3) ("clientSendMoreData: Appending %d bytes after %d bytes of headers\n", body_size, rep->hdr_sz); #endif @@ -1261,6 +1450,8 @@ clientSendMoreData(void *data, char *buf, ssize_t size) return; } } + /* reset range iterator */ + http->range_iter.pos = HttpHdrRangeInitPos; } http->out.offset += size; /* @@ -1283,6 +1474,7 @@ clientSendMoreData(void *data, char *buf, ssize_t size) #if OLD_CODE comm_write(fd, buf, writelen, clientWriteComplete, http, freefunc); #else + /* write headers and/or body if any */ if (rep || (body_buf && body_size)) { MemBuf mb; /* init mb; put status line and headers if any */ @@ -1291,11 +1483,28 @@ clientSendMoreData(void *data, char *buf, ssize_t size) httpReplyDestroy(rep); rep = NULL; } else { - memBufInit(&mb, body_size, body_size); + /* leave space for growth incase we do ranges */ + memBufInit(&mb, SM_PAGE_SIZE, 2*SM_PAGE_SIZE); } /* append body if any */ if (body_buf && body_size) - memBufAppend(&mb, body_buf, body_size); + if (http->request->range && http->request->method != METHOD_HEAD) { + /* Returning out.offset its physical meaning; Argh! @?@ @?@ */ + http->out.offset -= size; + if (!http->out.offset) + http->out.offset += http->range_iter.prefix_size; + /* force the end of the transfer if we are done */ + if (!clientPackMoreRanges(http, body_buf, body_size, &mb)) { + /* ick ? */ + if (entry->store_status == STORE_PENDING) + /* @?@ @?@ @?@ Different from HEAD */ + http->out.offset = entry->mem_obj->reply->content_length + entry->mem_obj->reply->hdr_sz; + else + http->out.offset = objectLen(entry); + } + } else { + memBufAppend(&mb, body_buf, body_size); + } /* write */ comm_write_mbuf(fd, mb, clientWriteComplete, http); } @@ -1817,6 +2026,7 @@ parseHttpRequestAbort(ConnStateData * conn, const char *uri) http->req_sz = conn->in.offset; http->uri = xstrdup(uri); http->log_uri = xstrdup(uri); + http->range_iter.boundary = StringNull; return http; } @@ -1941,6 +2151,7 @@ parseHttpRequest(ConnStateData * conn, method_t * method_p, int *status, http->conn = conn; http->start = current_time; http->req_sz = prefix_sz; + http->range_iter.boundary = StringNull; *prefix_p = xmalloc(prefix_sz + 1); xmemcpy(*prefix_p, conn->in.buf, prefix_sz); *(*prefix_p + prefix_sz) = '\0'; diff --git a/src/globals.h b/src/globals.h index 5153c82577..348b6f0560 100644 --- a/src/globals.h +++ b/src/globals.h @@ -1,6 +1,6 @@ /* - * $Id: globals.h,v 1.58 1998/05/30 19:43:08 rousskov Exp $ + * $Id: globals.h,v 1.59 1998/06/02 21:38:08 rousskov Exp $ */ extern FILE *debug_log; /* NULL */ @@ -105,3 +105,4 @@ extern CacheDigest *store_digest; /* NULL */ extern const char *StoreDigestUrlPath; /* "store_digest" */ extern const char *StoreDigestMimeStr; /* "application/cache-digest" */ extern const Version CacheDigestVer; /* { 3, 3 } */ +extern const char *MultipartMsgBoundaryStr; /* "Unique-Squid-Separator" */ diff --git a/src/http.cc b/src/http.cc index 1ef9243303..eb24b1bbd9 100644 --- a/src/http.cc +++ b/src/http.cc @@ -1,6 +1,6 @@ /* - * $Id: http.cc,v 1.278 1998/06/02 04:18:23 wessels Exp $ + * $Id: http.cc,v 1.279 1998/06/02 21:38:09 rousskov Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -604,7 +604,7 @@ httpBuildRequestHeader(request_t * request, LOCAL_ARRAY(char, bbuf, BBUF_SZ); String strConnection = StringNull; const HttpHeader *hdr_in = &orig_request->header; - HttpHdrRange *range = NULL; + int filter_range; const HttpHeaderEntry *e; HttpHeaderPos pos = HttpHeaderInitPos; @@ -618,13 +618,11 @@ httpBuildRequestHeader(request_t * request, if (entry && entry->lastmod > -1 && request->method == METHOD_GET) httpHeaderPutTime(hdr_out, HDR_IF_MODIFIED_SINCE, entry->lastmod); -#if FUTURE_CODE /* decide if we want to filter out Range specs * no reason to filter out if the reply will not be cachable * or if we cannot parse the specs */ - if (EBIT_TEST(orig_request->flags, REQ_CACHABLE)) - range = httpHeaderGetRange(hdr_in); -#endif + filter_range = + orig_request->range && EBIT_TEST(orig_request->flags, REQ_CACHABLE); strConnection = httpHeaderGetList(hdr_in, HDR_CONNECTION); while ((e = httpHeaderGetEntry(hdr_in, &pos))) { @@ -657,12 +655,10 @@ httpBuildRequestHeader(request_t * request, httpHeaderPutInt(hdr_out, HDR_MAX_FORWARDS, hops - 1); } break; -#if FUTURE_CODE case HDR_RANGE: - if (!range) + if (!filter_range) httpHeaderAddEntry(hdr_out, httpHeaderEntryClone(e)); break; -#endif case HDR_PROXY_CONNECTION: case HDR_CONNECTION: case HDR_VIA: @@ -729,8 +725,6 @@ httpBuildRequestHeader(request_t * request, } } stringClean(&strConnection); - if (range) - httpHdrRangeDestroy(range); } /* build request prefix and append it to a given MemBuf; diff --git a/src/protos.h b/src/protos.h index 27ca830a48..c84569170a 100644 --- a/src/protos.h +++ b/src/protos.h @@ -60,7 +60,7 @@ extern void intlistDestroy(intlist **); extern int intlistFind(intlist * list, int i); extern void wordlistDestroy(wordlist **); extern void configFreeMemory(void); -extern void wordlistCat(const wordlist *, MemBuf * mb); +extern void wordlistCat(const wordlist *, MemBuf *mb); extern void cbdataInit(void); #if CBDATA_DEBUG @@ -289,15 +289,24 @@ extern void httpHdrRangeDestroy(HttpHdrRange * range); extern HttpHdrRange *httpHdrRangeDup(const HttpHdrRange * range); extern void httpHdrRangePackInto(const HttpHdrRange * range, Packer * p); /* iterate through specs */ -extern int httpHdrRangeGetSpec(const HttpHdrRange * range, HttpHdrRangeSpec * spec, int *pos); +extern int httpHdrRangeGetSpec(const HttpHdrRange * range, HttpHdrRangeSpec * spec, HttpHdrRangePos *pos); +/* adjust specs after the length is known */ +extern int httpHdrRangeCanonize(HttpHdrRange * range, size_t clen); +/* other */ +extern String httpHdrRangeBoundaryStr(clientHttpRequest * http); +extern int httpHdrRangeIsComplex(const HttpHdrRange * range); + /* Http Content Range Header Field */ +extern HttpHdrContRange *httpHdrContRangeCreate(); extern HttpHdrContRange *httpHdrContRangeParseCreate(const char *crange_spec); /* returns true if range is valid; inits HttpHdrContRange */ extern int httpHdrContRangeParseInit(HttpHdrContRange * crange, const char *crange_spec); extern void httpHdrContRangeDestroy(HttpHdrContRange * crange); extern HttpHdrContRange *httpHdrContRangeDup(const HttpHdrContRange * crange); extern void httpHdrContRangePackInto(const HttpHdrContRange * crange, Packer * p); +/* inits with given spec */ +extern void httpHdrContRangeSet(HttpHdrContRange *, HttpHdrRangeSpec, size_t ent_len); /* Http Header Tools */ extern HttpHeaderFieldInfo *httpHeaderBuildFieldsInfo(const HttpHeaderFieldAttrs * attrs, int count); @@ -306,6 +315,7 @@ extern int httpHeaderIdByName(const char *name, int name_len, const HttpHeaderFi extern void httpHeaderMaskInit(HttpHeaderMask * mask); extern void httpHeaderCalcMask(HttpHeaderMask * mask, const int *enums, int count); extern int httpHeaderHasConnDir(const HttpHeader * hdr, const char *directive); +extern void httpHeaderAddContRange(HttpHeader * hdr, HttpHdrRangeSpec spec, size_t ent_len); extern void strListAdd(String * str, const char *item, char del); extern int strListIsMember(const String * str, const char *item, char del); extern int strListIsSubstr(const String * list, const char *s, char del); @@ -314,12 +324,12 @@ extern const char *getStringPrefix(const char *str, const char *end); extern int httpHeaderParseInt(const char *start, int *val); extern int httpHeaderParseSize(const char *start, size_t * sz); #ifdef __STDC__ -extern void httpHeaderPutStrf(HttpHeader * hdr, http_hdr_type id, const char *fmt,...); + extern void httpHeaderPutStrf(HttpHeader * hdr, http_hdr_type id, const char *fmt, ...); #else -extern void -httpHeaderPutStrf() + extern void httpHeaderPutStrf() #endif + /* Http Header */ extern void httpHeaderInitModule(); extern void httpHeaderCleanModule(); @@ -328,7 +338,7 @@ httpHeaderPutStrf() extern void httpHeaderClean(HttpHeader * hdr); /* append/update */ extern void httpHeaderAppend(HttpHeader * dest, const HttpHeader * src); - extern void httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask); + extern void httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask *denied_mask); /* parse/pack */ extern int httpHeaderParse(HttpHeader * hdr, const char *header_start, const char *header_end); extern void httpHeaderPackInto(const HttpHeader * hdr, Packer * p); @@ -339,6 +349,8 @@ httpHeaderPutStrf() extern void httpHeaderPutStr(HttpHeader * hdr, http_hdr_type type, const char *str); extern void httpHeaderPutAuth(HttpHeader * hdr, const char *authScheme, const char *realm); extern void httpHeaderPutCc(HttpHeader * hdr, const HttpHdrCc * cc); + extern void httpHeaderPutContRange(HttpHeader * hdr, const HttpHdrContRange *cr); + extern void httpHeaderPutRange(HttpHeader * hdr, const HttpHdrRange *range); extern void httpHeaderPutExt(HttpHeader * hdr, const char *name, const char *value); extern int httpHeaderGetInt(const HttpHeader * hdr, http_hdr_type id); extern time_t httpHeaderGetTime(const HttpHeader * hdr, http_hdr_type id); @@ -404,8 +416,8 @@ httpHeaderPutStrf() extern request_t *requestLink(request_t *); extern void requestUnlink(request_t *); extern int httpRequestParseHeader(request_t * req, const char *parse_start); - extern void httpRequestSwapOut(const request_t * req, StoreEntry * e); - extern int httpRequestPrefixLen(const request_t * req); + extern void httpRequestSwapOut(const request_t *req, StoreEntry *e); + extern int httpRequestPrefixLen(const request_t *req); extern int httpRequestHdrAllowed(const HttpHeaderEntry * e, String * strConnection); extern void icmpOpen(void); diff --git a/src/structs.h b/src/structs.h index e701951002..b7fa4e714c 100644 --- a/src/structs.h +++ b/src/structs.h @@ -522,6 +522,14 @@ struct _HttpHdrContRange { size_t elength; /* entity length, not content length */ }; +/* data for iterating thru range specs */ +struct _HttpHdrRangeIter { + HttpHdrRangeSpec spec; + HttpHdrRangePos pos; + size_t debt_size; /* bytes left to send from the current spec */ + size_t prefix_size; /* the size of the incoming HTTP msg prefix */ + String boundary; /* boundary for multipart responses */ +}; /* per field statistics */ struct _HttpHeaderFieldStat { @@ -670,7 +678,8 @@ struct _clientHttpRequest { off_t offset; size_t size; } out; - size_t req_sz; /* raw request size on input, not current request size */ + HttpHdrRangeIter range_iter;/* data for iterating thru range specs */ + size_t req_sz; /* raw request size on input, not current request size */ StoreEntry *entry; StoreEntry *old_entry; log_type log_type; @@ -1099,18 +1108,14 @@ struct _request_t { int link_count; /* free when zero */ int flags; HttpHdrCc *cache_control; + HttpHdrRange *range; time_t max_age; float http_ver; time_t ims; int imslen; int max_forwards; struct in_addr client_addr; -#if OLD_CODE - char *headers; - size_t headers_sz; -#else HttpHeader header; -#endif char *body; size_t body_sz; HierarchyLogEntry hier; diff --git a/src/typedefs.h b/src/typedefs.h index 6ebfedaca8..b88b17fd54 100644 --- a/src/typedefs.h +++ b/src/typedefs.h @@ -58,6 +58,7 @@ typedef struct _HttpHeader HttpHeader; typedef struct _HttpHdrCc HttpHdrCc; typedef struct _HttpHdrRangeSpec HttpHdrRangeSpec; typedef struct _HttpHdrRange HttpHdrRange; +typedef struct _HttpHdrRangeIter HttpHdrRangeIter; typedef struct _HttpHdrContRange HttpHdrContRange; typedef struct _HttpHeaderEntry HttpHeaderEntry; typedef struct _HttpHeaderFieldStat HttpHeaderFieldStat;