]> git.ipfire.org Git - thirdparty/squid.git/blob - src/HttpRequest.cc
Docs: Copyright updates for 2018 (#114)
[thirdparty/squid.git] / src / HttpRequest.cc
1 /*
2 * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9 /* DEBUG: section 73 HTTP Request */
10
11 #include "squid.h"
12 #include "AccessLogEntry.h"
13 #include "acl/AclSizeLimit.h"
14 #include "acl/FilledChecklist.h"
15 #include "client_side.h"
16 #include "client_side_request.h"
17 #include "dns/LookupDetails.h"
18 #include "Downloader.h"
19 #include "err_detail_type.h"
20 #include "globals.h"
21 #include "gopher.h"
22 #include "http.h"
23 #include "http/one/RequestParser.h"
24 #include "http/Stream.h"
25 #include "HttpHdrCc.h"
26 #include "HttpHeaderRange.h"
27 #include "HttpRequest.h"
28 #include "log/Config.h"
29 #include "MemBuf.h"
30 #include "sbuf/StringConvert.h"
31 #include "SquidConfig.h"
32 #include "Store.h"
33 #include "URL.h"
34
35 #if USE_AUTH
36 #include "auth/UserRequest.h"
37 #endif
38 #if ICAP_CLIENT
39 #include "adaptation/icap/icap_log.h"
40 #endif
41
42 HttpRequest::HttpRequest(const MasterXaction::Pointer &mx) :
43 Http::Message(hoRequest),
44 masterXaction(mx)
45 {
46 assert(mx);
47 init();
48 }
49
50 HttpRequest::HttpRequest(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aSchemeImg, const char *aUrlpath, const MasterXaction::Pointer &mx) :
51 Http::Message(hoRequest),
52 masterXaction(mx)
53 {
54 assert(mx);
55 static unsigned int id = 1;
56 debugs(93,7, HERE << "constructed, this=" << this << " id=" << ++id);
57 init();
58 initHTTP(aMethod, aProtocol, aSchemeImg, aUrlpath);
59 }
60
61 HttpRequest::~HttpRequest()
62 {
63 clean();
64 debugs(93,7, HERE << "destructed, this=" << this);
65 }
66
67 void
68 HttpRequest::initHTTP(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aSchemeImg, const char *aUrlpath)
69 {
70 method = aMethod;
71 url.setScheme(aProtocol, aSchemeImg);
72 url.path(aUrlpath);
73 }
74
75 void
76 HttpRequest::init()
77 {
78 method = Http::METHOD_NONE;
79 url.clear();
80 #if USE_AUTH
81 auth_user_request = NULL;
82 #endif
83 memset(&flags, '\0', sizeof(flags));
84 range = NULL;
85 ims = -1;
86 imslen = 0;
87 lastmod = -1;
88 client_addr.setEmpty();
89 my_addr.setEmpty();
90 body_pipe = NULL;
91 // hier
92 dnsWait = -1;
93 errType = ERR_NONE;
94 errDetail = ERR_DETAIL_NONE;
95 peer_login = NULL; // not allocated/deallocated by this class
96 peer_domain = NULL; // not allocated/deallocated by this class
97 peer_host = NULL;
98 vary_headers = SBuf();
99 myportname = null_string;
100 tag = null_string;
101 #if USE_AUTH
102 extacl_user = null_string;
103 extacl_passwd = null_string;
104 #endif
105 extacl_log = null_string;
106 extacl_message = null_string;
107 pstate = Http::Message::psReadyToParseStartLine;
108 #if FOLLOW_X_FORWARDED_FOR
109 indirect_client_addr.setEmpty();
110 #endif /* FOLLOW_X_FORWARDED_FOR */
111 #if USE_ADAPTATION
112 adaptHistory_ = NULL;
113 #endif
114 #if ICAP_CLIENT
115 icapHistory_ = NULL;
116 #endif
117 rangeOffsetLimit = -2; //a value of -2 means not checked yet
118 forcedBodyContinuation = false;
119 }
120
121 void
122 HttpRequest::clean()
123 {
124 // we used to assert that the pipe is NULL, but now the request only
125 // points to a pipe that is owned and initiated by another object.
126 body_pipe = NULL;
127 #if USE_AUTH
128 auth_user_request = NULL;
129 #endif
130 vary_headers.clear();
131 url.clear();
132
133 header.clean();
134
135 if (cache_control) {
136 delete cache_control;
137 cache_control = NULL;
138 }
139
140 if (range) {
141 delete range;
142 range = NULL;
143 }
144
145 myportname.clean();
146
147 theNotes = nullptr;
148
149 tag.clean();
150 #if USE_AUTH
151 extacl_user.clean();
152 extacl_passwd.clean();
153 #endif
154 extacl_log.clean();
155
156 extacl_message.clean();
157
158 etag.clean();
159
160 #if USE_ADAPTATION
161 adaptHistory_ = NULL;
162 #endif
163 #if ICAP_CLIENT
164 icapHistory_ = NULL;
165 #endif
166 }
167
168 void
169 HttpRequest::reset()
170 {
171 clean();
172 init();
173 }
174
175 HttpRequest *
176 HttpRequest::clone() const
177 {
178 HttpRequest *copy = new HttpRequest(masterXaction);
179 copy->method = method;
180 // TODO: move common cloning clone to Msg::copyTo() or copy ctor
181 copy->header.append(&header);
182 copy->hdrCacheInit();
183 copy->hdr_sz = hdr_sz;
184 copy->http_ver = http_ver;
185 copy->pstate = pstate; // TODO: should we assert a specific state here?
186 copy->body_pipe = body_pipe;
187
188 copy->url = url;
189
190 // range handled in hdrCacheInit()
191 copy->ims = ims;
192 copy->imslen = imslen;
193 copy->hier = hier; // Is it safe to copy? Should we?
194
195 copy->errType = errType;
196
197 // XXX: what to do with copy->peer_login?
198
199 copy->lastmod = lastmod;
200 copy->etag = etag;
201 copy->vary_headers = vary_headers;
202 // XXX: what to do with copy->peer_domain?
203
204 copy->tag = tag;
205 copy->extacl_log = extacl_log;
206 copy->extacl_message = extacl_message;
207
208 const bool inheritWorked = copy->inheritProperties(this);
209 assert(inheritWorked);
210
211 return copy;
212 }
213
214 bool
215 HttpRequest::inheritProperties(const Http::Message *aMsg)
216 {
217 const HttpRequest* aReq = dynamic_cast<const HttpRequest*>(aMsg);
218 if (!aReq)
219 return false;
220
221 client_addr = aReq->client_addr;
222 #if FOLLOW_X_FORWARDED_FOR
223 indirect_client_addr = aReq->indirect_client_addr;
224 #endif
225 my_addr = aReq->my_addr;
226
227 dnsWait = aReq->dnsWait;
228
229 #if USE_ADAPTATION
230 adaptHistory_ = aReq->adaptHistory();
231 #endif
232 #if ICAP_CLIENT
233 icapHistory_ = aReq->icapHistory();
234 #endif
235
236 // This may be too conservative for the 204 No Content case
237 // may eventually need cloneNullAdaptationImmune() for that.
238 flags = aReq->flags.cloneAdaptationImmune();
239
240 errType = aReq->errType;
241 errDetail = aReq->errDetail;
242 #if USE_AUTH
243 auth_user_request = aReq->auth_user_request;
244 extacl_user = aReq->extacl_user;
245 extacl_passwd = aReq->extacl_passwd;
246 #endif
247
248 myportname = aReq->myportname;
249
250 forcedBodyContinuation = aReq->forcedBodyContinuation;
251
252 // main property is which connection the request was received on (if any)
253 clientConnectionManager = aReq->clientConnectionManager;
254
255 downloader = aReq->downloader;
256
257 theNotes = aReq->theNotes;
258
259 sources = aReq->sources;
260 return true;
261 }
262
263 /**
264 * Checks the first line of an HTTP request is valid
265 * currently just checks the request method is present.
266 *
267 * NP: Other errors are left for detection later in the parse.
268 */
269 bool
270 HttpRequest::sanityCheckStartLine(const char *buf, const size_t hdr_len, Http::StatusCode *error)
271 {
272 // content is long enough to possibly hold a reply
273 // 2 being magic size of a 1-byte request method plus space delimiter
274 if (hdr_len < 2) {
275 // this is ony a real error if the headers apparently complete.
276 if (hdr_len > 0) {
277 debugs(58, 3, HERE << "Too large request header (" << hdr_len << " bytes)");
278 *error = Http::scInvalidHeader;
279 }
280 return false;
281 }
282
283 /* See if the request buffer starts with a non-whitespace HTTP request 'method'. */
284 HttpRequestMethod m;
285 m.HttpRequestMethodXXX(buf);
286 if (m == Http::METHOD_NONE) {
287 debugs(73, 3, "HttpRequest::sanityCheckStartLine: did not find HTTP request method");
288 *error = Http::scInvalidHeader;
289 return false;
290 }
291
292 return true;
293 }
294
295 bool
296 HttpRequest::parseFirstLine(const char *start, const char *end)
297 {
298 method.HttpRequestMethodXXX(start);
299
300 if (method == Http::METHOD_NONE)
301 return false;
302
303 // XXX: performance regression, strcspn() over the method bytes a second time.
304 // cheaper than allocate+copy+deallocate cycle to SBuf convert a piece of start.
305 const char *t = start + strcspn(start, w_space);
306
307 start = t + strspn(t, w_space); // skip w_space after method
308
309 const char *ver = findTrailingHTTPVersion(start, end);
310
311 if (ver) {
312 end = ver - 1;
313
314 while (xisspace(*end)) // find prev non-space
315 --end;
316
317 ++end; // back to space
318
319 if (2 != sscanf(ver + 5, "%d.%d", &http_ver.major, &http_ver.minor)) {
320 debugs(73, DBG_IMPORTANT, "parseRequestLine: Invalid HTTP identifier.");
321 return false;
322 }
323 } else {
324 http_ver.major = 0;
325 http_ver.minor = 9;
326 }
327
328 if (end < start) // missing URI
329 return false;
330
331 char save = *end;
332
333 * (char *) end = '\0'; // temp terminate URI, XXX dangerous?
334
335 const bool ret = url.parse(method, start);
336
337 * (char *) end = save;
338
339 return ret;
340 }
341
342 /* swaps out request using httpRequestPack */
343 void
344 HttpRequest::swapOut(StoreEntry * e)
345 {
346 assert(e);
347 e->buffer();
348 pack(e);
349 e->flush();
350 }
351
352 /* packs request-line and headers, appends <crlf> terminator */
353 void
354 HttpRequest::pack(Packable * p) const
355 {
356 assert(p);
357 /* pack request-line */
358 p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
359 SQUIDSBUFPRINT(method.image()), SQUIDSBUFPRINT(url.path()),
360 http_ver.major, http_ver.minor);
361 /* headers */
362 header.packInto(p);
363 /* trailer */
364 p->append("\r\n", 2);
365 }
366
367 /*
368 * A wrapper for debugObj()
369 */
370 void
371 httpRequestPack(void *obj, Packable *p)
372 {
373 HttpRequest *request = static_cast<HttpRequest*>(obj);
374 request->pack(p);
375 }
376
377 /* returns the length of request line + headers + crlf */
378 int
379 HttpRequest::prefixLen() const
380 {
381 return method.image().length() + 1 +
382 url.path().length() + 1 +
383 4 + 1 + 3 + 2 +
384 header.len + 2;
385 }
386
387 /* sync this routine when you update HttpRequest struct */
388 void
389 HttpRequest::hdrCacheInit()
390 {
391 Http::Message::hdrCacheInit();
392
393 assert(!range);
394 range = header.getRange();
395 }
396
397 #if ICAP_CLIENT
398 Adaptation::Icap::History::Pointer
399 HttpRequest::icapHistory() const
400 {
401 if (!icapHistory_) {
402 if (Log::TheConfig.hasIcapToken || IcapLogfileStatus == LOG_ENABLE) {
403 icapHistory_ = new Adaptation::Icap::History();
404 debugs(93,4, HERE << "made " << icapHistory_ << " for " << this);
405 }
406 }
407
408 return icapHistory_;
409 }
410 #endif
411
412 #if USE_ADAPTATION
413 Adaptation::History::Pointer
414 HttpRequest::adaptHistory(bool createIfNone) const
415 {
416 if (!adaptHistory_ && createIfNone) {
417 adaptHistory_ = new Adaptation::History();
418 debugs(93,4, HERE << "made " << adaptHistory_ << " for " << this);
419 }
420
421 return adaptHistory_;
422 }
423
424 Adaptation::History::Pointer
425 HttpRequest::adaptLogHistory() const
426 {
427 return HttpRequest::adaptHistory(Log::TheConfig.hasAdaptToken);
428 }
429
430 void
431 HttpRequest::adaptHistoryImport(const HttpRequest &them)
432 {
433 if (!adaptHistory_) {
434 adaptHistory_ = them.adaptHistory_; // may be nil
435 } else {
436 // check that histories did not diverge
437 Must(!them.adaptHistory_ || them.adaptHistory_ == adaptHistory_);
438 }
439 }
440
441 #endif
442
443 bool
444 HttpRequest::multipartRangeRequest() const
445 {
446 return (range && range->specs.size() > 1);
447 }
448
449 bool
450 HttpRequest::bodyNibbled() const
451 {
452 return body_pipe != NULL && body_pipe->consumedSize() > 0;
453 }
454
455 void
456 HttpRequest::detailError(err_type aType, int aDetail)
457 {
458 if (errType || errDetail)
459 debugs(11, 5, HERE << "old error details: " << errType << '/' << errDetail);
460 debugs(11, 5, HERE << "current error details: " << aType << '/' << aDetail);
461 // checking type and detail separately may cause inconsistency, but
462 // may result in more details available if they only become available later
463 if (!errType)
464 errType = aType;
465 if (!errDetail)
466 errDetail = aDetail;
467 }
468
469 void
470 HttpRequest::clearError()
471 {
472 debugs(11, 7, HERE << "old error details: " << errType << '/' << errDetail);
473 errType = ERR_NONE;
474 errDetail = ERR_DETAIL_NONE;
475 }
476
477 void
478 HttpRequest::packFirstLineInto(Packable * p, bool full_uri) const
479 {
480 const SBuf tmp(full_uri ? effectiveRequestUri() : url.path());
481
482 // form HTTP request-line
483 p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
484 SQUIDSBUFPRINT(method.image()),
485 SQUIDSBUFPRINT(tmp),
486 http_ver.major, http_ver.minor);
487 }
488
489 /*
490 * Indicate whether or not we would expect an entity-body
491 * along with this request
492 */
493 bool
494 HttpRequest::expectingBody(const HttpRequestMethod &, int64_t &theSize) const
495 {
496 bool expectBody = false;
497
498 /*
499 * Note: Checks for message validity is in clientIsContentLengthValid().
500 * this just checks if a entity-body is expected based on HTTP message syntax
501 */
502 if (header.chunked()) {
503 expectBody = true;
504 theSize = -1;
505 } else if (content_length >= 0) {
506 expectBody = true;
507 theSize = content_length;
508 } else {
509 expectBody = false;
510 // theSize undefined
511 }
512
513 return expectBody;
514 }
515
516 /*
517 * Create a Request from a URL and METHOD.
518 *
519 * If the METHOD is CONNECT, then a host:port pair is looked for instead of a URL.
520 * If the request cannot be created cleanly, NULL is returned
521 */
522 HttpRequest *
523 HttpRequest::FromUrl(const char * url, const MasterXaction::Pointer &mx, const HttpRequestMethod& method)
524 {
525 std::unique_ptr<HttpRequest> req(new HttpRequest(mx));
526 if (req->url.parse(method, url)) {
527 req->method = method;
528 return req.release();
529 }
530 return nullptr;
531 }
532
533 /**
534 * Are responses to this request possible cacheable ?
535 * If false then no matter what the response must not be cached.
536 */
537 bool
538 HttpRequest::maybeCacheable()
539 {
540 // Intercepted request with Host: header which cannot be trusted.
541 // Because it failed verification, or someone bypassed the security tests
542 // we cannot cache the reponse for sharing between clients.
543 // TODO: update cache to store for particular clients only (going to same Host: and destination IP)
544 if (!flags.hostVerified && (flags.intercepted || flags.interceptTproxy))
545 return false;
546
547 switch (url.getScheme()) {
548 case AnyP::PROTO_HTTP:
549 case AnyP::PROTO_HTTPS:
550 if (!method.respMaybeCacheable())
551 return false;
552
553 // RFC 7234 section 5.2.1.5:
554 // "cache MUST NOT store any part of either this request or any response to it"
555 //
556 // NP: refresh_pattern ignore-no-store only applies to response messages
557 // this test is handling request message CC header.
558 if (!flags.ignoreCc && cache_control && cache_control->hasNoStore())
559 return false;
560 break;
561
562 case AnyP::PROTO_GOPHER:
563 if (!gopherCachable(this))
564 return false;
565 break;
566
567 case AnyP::PROTO_CACHE_OBJECT:
568 return false;
569
570 //case AnyP::PROTO_FTP:
571 default:
572 break;
573 }
574
575 return true;
576 }
577
578 bool
579 HttpRequest::conditional() const
580 {
581 return flags.ims ||
582 header.has(Http::HdrType::IF_MATCH) ||
583 header.has(Http::HdrType::IF_NONE_MATCH);
584 }
585
586 void
587 HttpRequest::recordLookup(const Dns::LookupDetails &dns)
588 {
589 if (dns.wait >= 0) { // known delay
590 if (dnsWait >= 0) { // have recorded DNS wait before
591 debugs(78, 7, this << " " << dnsWait << " += " << dns);
592 dnsWait += dns.wait;
593 } else {
594 debugs(78, 7, this << " " << dns);
595 dnsWait = dns.wait;
596 }
597 }
598 }
599
600 int64_t
601 HttpRequest::getRangeOffsetLimit()
602 {
603 /* -2 is the starting value of rangeOffsetLimit.
604 * If it is -2, that means we haven't checked it yet.
605 * Otherwise, return the current value */
606 if (rangeOffsetLimit != -2)
607 return rangeOffsetLimit;
608
609 rangeOffsetLimit = 0; // default value for rangeOffsetLimit
610
611 ACLFilledChecklist ch(NULL, this, NULL);
612 ch.src_addr = client_addr;
613 ch.my_addr = my_addr;
614
615 for (AclSizeLimit *l = Config.rangeOffsetLimit; l; l = l -> next) {
616 /* if there is no ACL list or if the ACLs listed match use this limit value */
617 if (!l->aclList || ch.fastCheck(l->aclList).allowed()) {
618 debugs(58, 4, HERE << "rangeOffsetLimit=" << rangeOffsetLimit);
619 rangeOffsetLimit = l->size; // may be -1
620 break;
621 }
622 }
623
624 return rangeOffsetLimit;
625 }
626
627 void
628 HttpRequest::ignoreRange(const char *reason)
629 {
630 if (range) {
631 debugs(73, 3, static_cast<void*>(range) << " for " << reason);
632 delete range;
633 range = NULL;
634 }
635 // Some callers also reset isRanged but it may not be safe for all callers:
636 // isRanged is used to determine whether a weak ETag comparison is allowed,
637 // and that check should not ignore the Range header if it was present.
638 // TODO: Some callers also delete HDR_RANGE, HDR_REQUEST_RANGE. Should we?
639 }
640
641 bool
642 HttpRequest::canHandle1xx() const
643 {
644 // old clients do not support 1xx unless they sent Expect: 100-continue
645 // (we reject all other Http::HdrType::EXPECT values so just check for Http::HdrType::EXPECT)
646 if (http_ver <= Http::ProtocolVersion(1,0) && !header.has(Http::HdrType::EXPECT))
647 return false;
648
649 // others must support 1xx control messages
650 return true;
651 }
652
653 ConnStateData *
654 HttpRequest::pinnedConnection()
655 {
656 if (clientConnectionManager.valid() && clientConnectionManager->pinning.pinned)
657 return clientConnectionManager.get();
658 return NULL;
659 }
660
661 const SBuf
662 HttpRequest::storeId()
663 {
664 if (store_id.size() != 0) {
665 debugs(73, 3, "sent back store_id: " << store_id);
666 return StringToSBuf(store_id);
667 }
668 debugs(73, 3, "sent back effectiveRequestUrl: " << effectiveRequestUri());
669 return effectiveRequestUri();
670 }
671
672 const SBuf &
673 HttpRequest::effectiveRequestUri() const
674 {
675 if (method.id() == Http::METHOD_CONNECT || url.getScheme() == AnyP::PROTO_AUTHORITY_FORM)
676 return url.authority(true); // host:port
677 return url.absolute();
678 }
679
680 NotePairs::Pointer
681 HttpRequest::notes()
682 {
683 if (!theNotes)
684 theNotes = new NotePairs;
685 return theNotes;
686 }
687
688 void
689 UpdateRequestNotes(ConnStateData *csd, HttpRequest &request, NotePairs const &helperNotes)
690 {
691 // Tag client connection if the helper responded with clt_conn_tag=tag.
692 const char *cltTag = "clt_conn_tag";
693 if (const char *connTag = helperNotes.findFirst(cltTag)) {
694 if (csd) {
695 csd->notes()->remove(cltTag);
696 csd->notes()->add(cltTag, connTag);
697 }
698 }
699 request.notes()->replaceOrAdd(&helperNotes);
700 }
701
702 void
703 HttpRequest::manager(const CbcPointer<ConnStateData> &aMgr, const AccessLogEntryPointer &al)
704 {
705 clientConnectionManager = aMgr;
706
707 if (!clientConnectionManager.valid())
708 return;
709
710 AnyP::PortCfgPointer port = clientConnectionManager->port;
711 if (port) {
712 myportname = port->name;
713 flags.ignoreCc = port->ignore_cc;
714 }
715
716 if (auto clientConnection = clientConnectionManager->clientConnection) {
717 client_addr = clientConnection->remote; // XXX: remove request->client_addr member.
718 #if FOLLOW_X_FORWARDED_FOR
719 // indirect client gets stored here because it is an HTTP header result (from X-Forwarded-For:)
720 // not details about the TCP connection itself
721 indirect_client_addr = clientConnection->remote;
722 #endif /* FOLLOW_X_FORWARDED_FOR */
723 my_addr = clientConnection->local;
724
725 flags.intercepted = ((clientConnection->flags & COMM_INTERCEPTION) != 0);
726 flags.interceptTproxy = ((clientConnection->flags & COMM_TRANSPARENT) != 0 ) ;
727 const bool proxyProtocolPort = port ? port->flags.proxySurrogate : false;
728 if (flags.interceptTproxy && !proxyProtocolPort) {
729 if (Config.accessList.spoof_client_ip) {
730 ACLFilledChecklist *checklist = new ACLFilledChecklist(Config.accessList.spoof_client_ip, this, clientConnection->rfc931);
731 checklist->al = al;
732 flags.spoofClientIp = checklist->fastCheck().allowed();
733 delete checklist;
734 } else
735 flags.spoofClientIp = true;
736 } else
737 flags.spoofClientIp = false;
738 }
739 }
740