]>
git.ipfire.org Git - thirdparty/squid.git/blob - src/url.cc
aab3abc1815e07d44f377ccbd8369d397e1d3a42
2 * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 23 URL Parsing */
13 #include "HttpRequest.h"
15 #include "SquidConfig.h"
16 #include "SquidString.h"
19 static HttpRequest
*urlParseFinish(const HttpRequestMethod
& method
,
20 const AnyP::ProtocolType protocol
,
21 const char *const urlpath
,
22 const char *const host
,
25 HttpRequest
*request
);
26 static HttpRequest
*urnParse(const HttpRequestMethod
& method
, char *urn
, HttpRequest
*request
);
27 static const char valid_hostname_chars_u
[] =
28 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
29 "abcdefghijklmnopqrstuvwxyz"
33 static const char valid_hostname_chars
[] =
34 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
35 "abcdefghijklmnopqrstuvwxyz"
43 static SBuf
star("*");
50 static SBuf
slash("/");
55 URL::host(const char *src
)
59 if (hostAddr_
.isAnyAddr()) {
60 xstrncpy(host_
, src
, sizeof(host_
));
61 hostIsNumeric_
= false;
63 hostAddr_
.toHostStr(host_
, sizeof(host_
));
64 debugs(23, 3, "given IP: " << hostAddr_
);
73 // RFC 3986 section 3.3 says path can be empty (path-abempty).
74 // RFC 7230 sections 2.7.3, 5.3.1, 5.7.2 - says path cannot be empty, default to "/"
75 // at least when sending and using. We must still accept path-abempty as input.
76 if (path_
.isEmpty() && (scheme_
== AnyP::PROTO_HTTP
|| scheme_
== AnyP::PROTO_HTTPS
))
85 debugs(23, 5, "urlInitialize: Initializing...");
86 /* this ensures that the number of protocol strings is the same as
87 * the enum slots allocated because the last enum is always 'MAX'.
89 assert(strcmp(AnyP::ProtocolType_str
[AnyP::PROTO_MAX
], "MAX") == 0);
91 * These test that our matchDomainName() function works the
92 * way we expect it to.
94 assert(0 == matchDomainName("foo.com", "foo.com"));
95 assert(0 == matchDomainName(".foo.com", "foo.com"));
96 assert(0 == matchDomainName("foo.com", ".foo.com"));
97 assert(0 == matchDomainName(".foo.com", ".foo.com"));
98 assert(0 == matchDomainName("x.foo.com", ".foo.com"));
99 assert(0 != matchDomainName("x.foo.com", "foo.com"));
100 assert(0 != matchDomainName("foo.com", "x.foo.com"));
101 assert(0 != matchDomainName("bar.com", "foo.com"));
102 assert(0 != matchDomainName(".bar.com", "foo.com"));
103 assert(0 != matchDomainName(".bar.com", ".foo.com"));
104 assert(0 != matchDomainName("bar.com", ".foo.com"));
105 assert(0 < matchDomainName("zzz.com", "foo.com"));
106 assert(0 > matchDomainName("aaa.com", "foo.com"));
107 assert(0 == matchDomainName("FOO.com", "foo.COM"));
108 assert(0 < matchDomainName("bfoo.com", "afoo.com"));
109 assert(0 > matchDomainName("afoo.com", "bfoo.com"));
110 assert(0 < matchDomainName("x-foo.com", ".foo.com"));
115 * Parse the scheme name from string b, into protocol type.
116 * The string must be 0-terminated.
119 urlParseProtocol(const char *b
)
121 // make e point to the ':' character
122 const char *e
= b
+ strcspn(b
, ":");
125 /* test common stuff first */
127 if (strncasecmp(b
, "http", len
) == 0)
128 return AnyP::PROTO_HTTP
;
130 if (strncasecmp(b
, "ftp", len
) == 0)
131 return AnyP::PROTO_FTP
;
133 if (strncasecmp(b
, "https", len
) == 0)
134 return AnyP::PROTO_HTTPS
;
136 if (strncasecmp(b
, "file", len
) == 0)
137 return AnyP::PROTO_FTP
;
139 if (strncasecmp(b
, "coap", len
) == 0)
140 return AnyP::PROTO_COAP
;
142 if (strncasecmp(b
, "coaps", len
) == 0)
143 return AnyP::PROTO_COAPS
;
145 if (strncasecmp(b
, "gopher", len
) == 0)
146 return AnyP::PROTO_GOPHER
;
148 if (strncasecmp(b
, "wais", len
) == 0)
149 return AnyP::PROTO_WAIS
;
151 if (strncasecmp(b
, "cache_object", len
) == 0)
152 return AnyP::PROTO_CACHE_OBJECT
;
154 if (strncasecmp(b
, "urn", len
) == 0)
155 return AnyP::PROTO_URN
;
157 if (strncasecmp(b
, "whois", len
) == 0)
158 return AnyP::PROTO_WHOIS
;
160 return AnyP::PROTO_NONE
;
166 * If the 'request' arg is non-NULL, put parsed values there instead
167 * of allocating a new HttpRequest.
169 * This abuses HttpRequest as a way of representing the parsed url
170 * and its components.
171 * method is used to switch parsers and to init the HttpRequest.
172 * If method is Http::METHOD_CONNECT, then rather than a URL a hostname:port is
174 * The url is non const so that if its too long we can NULL-terminate it in place.
178 * This routine parses a URL. Its assumed that the URL is complete -
179 * ie, the end of the string is the end of the URL. Don't pass a partial
180 * URL here as this routine doesn't have any way of knowing whether
181 * its partial or not (ie, it handles the case of no trailing slash as
182 * being "end of host with implied path of /".
185 urlParse(const HttpRequestMethod
& method
, char *url
, HttpRequest
*request
)
187 LOCAL_ARRAY(char, proto
, MAX_URL
);
188 LOCAL_ARRAY(char, login
, MAX_URL
);
189 LOCAL_ARRAY(char, host
, MAX_URL
);
190 LOCAL_ARRAY(char, urlpath
, MAX_URL
);
194 AnyP::ProtocolType protocol
= AnyP::PROTO_NONE
;
199 proto
[0] = host
[0] = urlpath
[0] = login
[0] = '\0';
201 if ((l
= strlen(url
)) + Config
.appendDomainLen
> (MAX_URL
- 1)) {
202 /* terminate so it doesn't overflow other buffers */
203 *(url
+ (MAX_URL
>> 1)) = '\0';
204 debugs(23, DBG_IMPORTANT
, "urlParse: URL too large (" << l
<< " bytes)");
207 if (method
== Http::METHOD_CONNECT
) {
210 if (sscanf(url
, "[%[^]]]:%d", host
, &port
) < 1)
211 if (sscanf(url
, "%[^:]:%d", host
, &port
) < 1)
214 } else if ((method
== Http::METHOD_OPTIONS
|| method
== Http::METHOD_TRACE
) &&
215 URL::Asterisk().cmp(url
) == 0) {
216 protocol
= AnyP::PROTO_HTTP
;
217 port
= AnyP::UriScheme(protocol
).defaultPort();
218 return urlParseFinish(method
, protocol
, url
, host
, SBuf(), port
, request
);
219 } else if (!strncmp(url
, "urn:", 4)) {
220 return urnParse(method
, url
, request
);
225 /* Find first : - everything before is protocol */
226 for (i
= 0, dst
= proto
; i
< l
&& *src
!= ':'; ++i
, ++src
, ++dst
) {
234 if ((i
+3) > l
|| *src
!= ':' || *(src
+ 1) != '/' || *(src
+ 2) != '/')
239 /* Then everything until first /; thats host (and port; which we'll look for here later) */
240 // bug 1881: If we don't get a "/" then we imply it was there
241 // bug 3074: We could just be given a "?" or "#". These also imply "/"
242 // bug 3233: whitespace is also a hostname delimiter.
243 for (dst
= host
; i
< l
&& *src
!= '/' && *src
!= '?' && *src
!= '#' && *src
!= '\0' && !xisspace(*src
); ++i
, ++src
, ++dst
) {
248 * We can't check for "i >= l" here because we could be at the end of the line
249 * and have a perfectly valid URL w/ no trailing '/'. In this case we assume we've
250 * been -given- a valid URL and the path is just '/'.
256 // bug 3074: received 'path' starting with '?', '#', or '\0' implies '/'
257 if (*src
== '?' || *src
== '#' || *src
== '\0') {
263 /* Then everything from / (inclusive) until \r\n or \0 - thats urlpath */
264 for (; i
< l
&& *src
!= '\r' && *src
!= '\n' && *src
!= '\0'; ++i
, ++src
, ++dst
) {
268 /* We -could- be at the end of the buffer here */
271 /* If the URL path is empty we set it to be "/" */
272 if (dst
== urlpath
) {
278 protocol
= urlParseProtocol(proto
);
279 port
= AnyP::UriScheme(protocol
).defaultPort();
281 /* Is there any login information? (we should eventually parse it above) */
282 t
= strrchr(host
, '@');
284 strncpy((char *) login
, (char *) host
, sizeof(login
)-1);
285 login
[sizeof(login
)-1] = '\0';
286 t
= strrchr(login
, '@');
288 strncpy((char *) host
, t
+ 1, sizeof(host
)-1);
289 host
[sizeof(host
)-1] = '\0';
292 /* Is there any host information? (we should eventually parse it above) */
294 /* strip any IPA brackets. valid under IPv6. */
296 /* only for IPv6 sadly, pre-IPv6/URL code can't handle the clean result properly anyway. */
301 for (; i
< l
&& *src
!= ']' && *src
!= '\0'; ++i
, ++src
, ++dst
) {
305 /* we moved in-place, so truncate the actual hostname found */
309 /* skip ahead to either start of port, or original EOS */
310 while (*dst
!= '\0' && *dst
!= ':')
314 t
= strrchr(host
, ':');
316 if (t
!= strchr(host
,':') ) {
317 /* RFC 2732 states IPv6 "SHOULD" be bracketed. allowing for times when its not. */
318 /* RFC 3986 'update' simply modifies this to an "is" with no emphasis at all! */
319 /* therefore we MUST accept the case where they are not bracketed at all. */
324 // Bug 3183 sanity check: If scheme is present, host must be too.
325 if (protocol
!= AnyP::PROTO_NONE
&& host
[0] == '\0') {
326 debugs(23, DBG_IMPORTANT
, "SECURITY ALERT: Missing hostname in URL '" << url
<< "'. see access.log for details.");
330 if (t
&& *t
== ':') {
337 for (t
= host
; *t
; ++t
)
340 if (stringHasWhitespace(host
)) {
341 if (URI_WHITESPACE_STRIP
== Config
.uri_whitespace
) {
354 debugs(23, 3, "urlParse: Split URL '" << url
<< "' into proto='" << proto
<< "', host='" << host
<< "', port='" << port
<< "', path='" << urlpath
<< "'");
356 if (Config
.onoff
.check_hostnames
&& strspn(host
, Config
.onoff
.allow_underscore
? valid_hostname_chars_u
: valid_hostname_chars
) != strlen(host
)) {
357 debugs(23, DBG_IMPORTANT
, "urlParse: Illegal character in hostname '" << host
<< "'");
361 /* For IPV6 addresses also check for a colon */
362 if (Config
.appendDomain
&& !strchr(host
, '.') && !strchr(host
, ':'))
363 strncat(host
, Config
.appendDomain
, SQUIDHOSTNAMELEN
- strlen(host
) - 1);
365 /* remove trailing dots from hostnames */
366 while ((l
= strlen(host
)) > 0 && host
[--l
] == '.')
369 /* reject duplicate or leading dots */
370 if (strstr(host
, "..") || *host
== '.') {
371 debugs(23, DBG_IMPORTANT
, "urlParse: Illegal hostname '" << host
<< "'");
375 if (port
< 1 || port
> 65535) {
376 debugs(23, 3, "urlParse: Invalid port '" << port
<< "'");
380 #if HARDCODE_DENY_PORTS
381 /* These ports are filtered in the default squid.conf, but
382 * maybe someone wants them hardcoded... */
383 if (port
== 7 || port
== 9 || port
== 19) {
384 debugs(23, DBG_CRITICAL
, "urlParse: Deny access to port " << port
);
389 if (stringHasWhitespace(urlpath
)) {
390 debugs(23, 2, "urlParse: URI has whitespace: {" << url
<< "}");
392 switch (Config
.uri_whitespace
) {
394 case URI_WHITESPACE_DENY
:
397 case URI_WHITESPACE_ALLOW
:
400 case URI_WHITESPACE_ENCODE
:
401 t
= rfc1738_escape_unescaped(urlpath
);
402 xstrncpy(urlpath
, t
, MAX_URL
);
405 case URI_WHITESPACE_CHOP
:
406 *(urlpath
+ strcspn(urlpath
, w_space
)) = '\0';
409 case URI_WHITESPACE_STRIP
:
423 return urlParseFinish(method
, protocol
, urlpath
, host
, SBuf(login
), port
, request
);
427 * Update request with parsed URI data. If the request arg is
428 * non-NULL, put parsed values there instead of allocating a new
432 urlParseFinish(const HttpRequestMethod
& method
,
433 const AnyP::ProtocolType protocol
,
434 const char *const urlpath
,
435 const char *const host
,
438 HttpRequest
*request
)
441 request
= new HttpRequest(method
, protocol
, urlpath
);
443 request
->initHTTP(method
, protocol
, urlpath
);
446 request
->url
.host(host
);
447 request
->url
.userInfo(login
);
448 request
->url
.port(port
);
453 urnParse(const HttpRequestMethod
& method
, char *urn
, HttpRequest
*request
)
455 debugs(50, 5, "urnParse: " << urn
);
457 request
->initHTTP(method
, AnyP::PROTO_URN
, urn
+ 4);
461 return new HttpRequest(method
, AnyP::PROTO_URN
, urn
+ 4);
468 authorityHttp_
.clear();
469 authorityWithPort_
.clear();
473 URL::authority(bool requirePort
) const
475 if (authorityHttp_
.isEmpty()) {
477 // both formats contain Host/IP
478 authorityWithPort_
.append(host());
479 authorityHttp_
= authorityWithPort_
;
481 // authorityForm_ only has :port if it is non-default
482 authorityWithPort_
.appendf(":%u",port());
483 if (port() != getScheme().defaultPort())
484 authorityHttp_
= authorityWithPort_
;
487 return requirePort
? authorityWithPort_
: authorityHttp_
;
491 URL::absolute() const
493 if (absolute_
.isEmpty()) {
494 // TODO: most URL will be much shorter, avoid allocating this much
495 absolute_
.reserveCapacity(MAX_URL
);
497 absolute_
.appendf("%s:", getScheme().c_str());
498 if (getScheme() != AnyP::PROTO_URN
) {
499 absolute_
.append("//", 2);
500 const bool omitUserInfo
= getScheme() == AnyP::PROTO_HTTP
||
501 getScheme() != AnyP::PROTO_HTTPS
||
502 userInfo().isEmpty();
504 absolute_
.append(userInfo());
505 absolute_
.append("@", 1);
507 absolute_
.append(authority());
509 absolute_
.append(path());
515 /** \todo AYJ: Performance: This is an *almost* duplicate of HttpRequest::effectiveRequestUri(). But elides the query-string.
516 * After copying it on in the first place! Would be less code to merge the two with a flag parameter.
517 * and never copy the query-string part in the first place
520 urlCanonicalClean(const HttpRequest
* request
)
522 LOCAL_ARRAY(char, buf
, MAX_URL
);
524 snprintf(buf
, sizeof(buf
), SQUIDSBUFPH
, SQUIDSBUFPRINT(request
->effectiveRequestUri()));
525 buf
[sizeof(buf
)-1] = '\0';
527 // URN, CONNECT method, and non-stripped URIs can go straight out
528 if (Config
.onoff
.strip_query_terms
&& !(request
->method
== Http::METHOD_CONNECT
|| request
->url
.getScheme() == AnyP::PROTO_URN
)) {
529 // strip anything AFTER a question-mark
530 // leaving the '?' in place
531 if (auto t
= strchr(buf
, '?')) {
536 if (stringHasCntl(buf
))
537 xstrncpy(buf
, rfc1738_escape_unescaped(buf
), MAX_URL
);
543 * Yet another alternative to urlCanonical.
544 * This one adds the https:// parts to Http::METHOD_CONNECT URL
545 * for use in error page outputs.
546 * Luckily we can leverage the others instead of duplicating.
549 urlCanonicalFakeHttps(const HttpRequest
* request
)
551 LOCAL_ARRAY(char, buf
, MAX_URL
);
553 // method CONNECT and port HTTPS
554 if (request
->method
== Http::METHOD_CONNECT
&& request
->url
.port() == 443) {
555 snprintf(buf
, MAX_URL
, "https://%s/*", request
->url
.host());
559 // else do the normal complete canonical thing.
560 return urlCanonicalClean(request
);
564 * Test if a URL is relative.
566 * RFC 2396, Section 5 (Page 17) implies that in a relative URL, a '/' will
567 * appear before a ':'.
570 urlIsRelative(const char *url
)
581 for (p
= url
; *p
!= '\0' && *p
!= ':' && *p
!= '/'; ++p
);
590 * Convert a relative URL to an absolute URL using the context of a given
593 * It is assumed that you have already ensured that the URL is relative.
595 * If NULL is returned it is an indication that the method in use in the
596 * request does not distinguish between relative and absolute and you should
597 * use the url unchanged.
599 * If non-NULL is returned, it is up to the caller to free the resulting
600 * memory using safe_free().
603 urlMakeAbsolute(const HttpRequest
* req
, const char *relUrl
)
606 if (req
->method
.id() == Http::METHOD_CONNECT
) {
610 char *urlbuf
= (char *)xmalloc(MAX_URL
* sizeof(char));
612 if (req
->url
.getScheme() == AnyP::PROTO_URN
) {
613 // XXX: this is what the original code did, but it seems to break the
614 // intended behaviour of this function. It returns the stored URN path,
615 // not converting the given one into a URN...
616 snprintf(urlbuf
, MAX_URL
, SQUIDSBUFPH
, SQUIDSBUFPRINT(req
->url
.absolute()));
620 SBuf authorityForm
= req
->url
.authority(); // host[:port]
621 size_t urllen
= snprintf(urlbuf
, MAX_URL
, "%s://" SQUIDSBUFPH
"%s" SQUIDSBUFPH
,
622 req
->url
.getScheme().c_str(),
623 SQUIDSBUFPRINT(req
->url
.userInfo()),
624 !req
->url
.userInfo().isEmpty() ? "@" : "",
625 SQUIDSBUFPRINT(authorityForm
));
627 // if the first char is '/' assume its a relative path
628 // XXX: this breaks on scheme-relative URLs,
629 // but we should not see those outside ESI, and rarely there.
630 // XXX: also breaks on any URL containing a '/' in the query-string portion
631 if (relUrl
[0] == '/') {
632 xstrncpy(&urlbuf
[urllen
], relUrl
, MAX_URL
- urllen
- 1);
634 SBuf path
= req
->url
.path();
635 SBuf::size_type lastSlashPos
= path
.rfind('/');
637 if (lastSlashPos
== SBuf::npos
) {
638 // replace the whole path with the given bit(s)
639 urlbuf
[urllen
] = '/';
641 xstrncpy(&urlbuf
[urllen
], relUrl
, MAX_URL
- urllen
- 1);
643 // replace only the last (file?) segment with the given bit(s)
645 if (lastSlashPos
> MAX_URL
- urllen
- 1) {
646 // XXX: crops bits in the middle of the combined URL.
647 lastSlashPos
= MAX_URL
- urllen
- 1;
649 SBufToCstring(&urlbuf
[urllen
], path
.substr(0,lastSlashPos
));
650 urllen
+= lastSlashPos
;
651 if (urllen
+ 1 < MAX_URL
) {
652 xstrncpy(&urlbuf
[urllen
], relUrl
, MAX_URL
- urllen
- 1);
661 matchDomainName(const char *h
, const char *d
, bool honorWildcards
)
674 * Start at the ends of the two strings and work towards the
677 while (xtolower(h
[--hl
]) == xtolower(d
[--dl
])) {
678 if (hl
== 0 && dl
== 0) {
680 * We made it all the way to the beginning of both
681 * strings without finding any difference.
688 * The host string is shorter than the domain string.
689 * There is only one case when this can be a match.
690 * If the domain is just one character longer, and if
691 * that character is a leading '.' then we call it a
695 if (1 == dl
&& '.' == d
[0])
703 * The domain string is shorter than the host string.
704 * This is a match only if the first domain character
716 * We found different characters in the same position (from the end).
719 // If the h has a form of "*.foo.com" and d has a form of "x.foo.com"
720 // then the h[hl] points to '*', h[hl+1] to '.' and d[dl] to 'x'
721 // The following checks are safe, the "h[hl + 1]" in the worst case is '\0'.
722 if (honorWildcards
&& h
[hl
] == '*' && h
[hl
+ 1] == '.')
726 * If one of those character is '.' then its special. In order
727 * for splay tree sorting to work properly, "x-foo.com" must
728 * be greater than ".foo.com" even though '-' is less than '.'.
736 return (xtolower(h
[hl
]) - xtolower(d
[dl
]));
740 * return true if we can serve requests for this method.
743 urlCheckRequest(const HttpRequest
* r
)
746 /* protocol "independent" methods
748 * actually these methods are specific to HTTP:
749 * they are methods we recieve on our HTTP port,
750 * and if we had a FTP listener would not be relevant
753 * So, we should delegate them to HTTP. The problem is that we
754 * do not have a default protocol from the client side of HTTP.
757 if (r
->method
== Http::METHOD_CONNECT
)
760 // we support OPTIONS and TRACE directed at us (with a 501 reply, for now)
761 // we also support forwarding OPTIONS and TRACE, except for the *-URI ones
762 if (r
->method
== Http::METHOD_OPTIONS
|| r
->method
== Http::METHOD_TRACE
)
763 return (r
->header
.getInt64(Http::HdrType::MAX_FORWARDS
) == 0 || r
->url
.path() != URL::Asterisk());
765 if (r
->method
== Http::METHOD_PURGE
)
768 /* does method match the protocol? */
769 switch (r
->url
.getScheme()) {
771 case AnyP::PROTO_URN
:
773 case AnyP::PROTO_HTTP
:
775 case AnyP::PROTO_CACHE_OBJECT
:
779 case AnyP::PROTO_FTP
:
781 if (r
->method
== Http::METHOD_PUT
)
784 case AnyP::PROTO_GOPHER
:
786 case AnyP::PROTO_WAIS
:
788 case AnyP::PROTO_WHOIS
:
789 if (r
->method
== Http::METHOD_GET
)
791 else if (r
->method
== Http::METHOD_HEAD
)
796 case AnyP::PROTO_HTTPS
:
805 * Squid can't originate an SSL connection, so it should
806 * never receive an "https:" URL. It should always be
821 * Quick-n-dirty host extraction from a URL. Steps:
823 * Skip any '/' after the colon
824 * Copy the next SQUID_MAXHOSTNAMELEN bytes to host[]
825 * Look for an ending '/' or ':' and terminate
826 * Look for login info preceeded by '@'
833 char * extract(char const *url
);
836 static char Host
[SQUIDHOSTNAMELEN
];
837 void init(char const *);
838 void findHostStart();
839 void trimTrailingChars();
841 char const *hostStart
;
846 urlHostname(const char *url
)
848 return URLHostName().extract(url
);
851 char URLHostName::Host
[SQUIDHOSTNAMELEN
];
854 URLHostName::init(char const *aUrl
)
861 URLHostName::findHostStart()
863 if (NULL
== (hostStart
= strchr(url
, ':')))
868 while (*hostStart
!= '\0' && *hostStart
== '/')
871 if (*hostStart
== ']')
876 URLHostName::trimTrailingChars()
880 if ((t
= strchr(Host
, '/')))
883 if ((t
= strrchr(Host
, ':')))
886 if ((t
= strchr(Host
, ']')))
891 URLHostName::trimAuth()
895 if ((t
= strrchr(Host
, '@'))) {
897 memmove(Host
, t
, strlen(t
) + 1);
902 URLHostName::extract(char const *aUrl
)
907 if (hostStart
== NULL
)
910 xstrncpy(Host
, hostStart
, SQUIDHOSTNAMELEN
);
919 URL::URL(AnyP::UriScheme
const &aScheme
) :
921 hostIsNumeric_(false),