]>
git.ipfire.org Git - thirdparty/squid.git/blob - src/url.cc
2 * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 23 URL Parsing */
13 #include "HttpRequest.h"
15 #include "SquidConfig.h"
16 #include "SquidString.h"
19 static HttpRequest
*urlParseFinish(const HttpRequestMethod
& method
,
20 const AnyP::ProtocolType protocol
,
21 const char *const urlpath
,
22 const char *const host
,
25 HttpRequest
*request
);
26 static HttpRequest
*urnParse(const HttpRequestMethod
& method
, char *urn
, HttpRequest
*request
);
27 static const char valid_hostname_chars_u
[] =
28 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
29 "abcdefghijklmnopqrstuvwxyz"
33 static const char valid_hostname_chars
[] =
34 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
35 "abcdefghijklmnopqrstuvwxyz"
43 static SBuf
star("*");
50 static SBuf
slash("/");
55 URL::host(const char *src
)
59 if (hostAddr_
.isAnyAddr()) {
60 xstrncpy(host_
, src
, sizeof(host_
));
61 hostIsNumeric_
= false;
63 hostAddr_
.toHostStr(host_
, sizeof(host_
));
64 debugs(23, 3, "given IP: " << hostAddr_
);
73 // RFC 3986 section 3.3 says path can be empty (path-abempty).
74 // RFC 7230 sections 2.7.3, 5.3.1, 5.7.2 - says path cannot be empty, default to "/"
75 // at least when sending and using. We must still accept path-abempty as input.
76 if (path_
.isEmpty() && (scheme_
== AnyP::PROTO_HTTP
|| scheme_
== AnyP::PROTO_HTTPS
))
85 debugs(23, 5, "urlInitialize: Initializing...");
86 /* this ensures that the number of protocol strings is the same as
87 * the enum slots allocated because the last enum is always 'MAX'.
89 assert(strcmp(AnyP::ProtocolType_str
[AnyP::PROTO_MAX
], "MAX") == 0);
91 * These test that our matchDomainName() function works the
92 * way we expect it to.
94 assert(0 == matchDomainName("foo.com", "foo.com"));
95 assert(0 == matchDomainName(".foo.com", "foo.com"));
96 assert(0 == matchDomainName("foo.com", ".foo.com"));
97 assert(0 == matchDomainName(".foo.com", ".foo.com"));
98 assert(0 == matchDomainName("x.foo.com", ".foo.com"));
99 assert(0 != matchDomainName("x.foo.com", "foo.com"));
100 assert(0 != matchDomainName("foo.com", "x.foo.com"));
101 assert(0 != matchDomainName("bar.com", "foo.com"));
102 assert(0 != matchDomainName(".bar.com", "foo.com"));
103 assert(0 != matchDomainName(".bar.com", ".foo.com"));
104 assert(0 != matchDomainName("bar.com", ".foo.com"));
105 assert(0 < matchDomainName("zzz.com", "foo.com"));
106 assert(0 > matchDomainName("aaa.com", "foo.com"));
107 assert(0 == matchDomainName("FOO.com", "foo.COM"));
108 assert(0 < matchDomainName("bfoo.com", "afoo.com"));
109 assert(0 > matchDomainName("afoo.com", "bfoo.com"));
110 assert(0 < matchDomainName("x-foo.com", ".foo.com"));
115 * urlParseProtocol() takes begin (b) and end (e) pointers, but for
116 * backwards compatibility, e defaults to NULL, in which case we
117 * assume b is NULL-terminated.
120 urlParseProtocol(const char *b
, const char *e
)
123 * if e is NULL, b must be NULL terminated and we
124 * make e point to the first whitespace character
129 e
= b
+ strcspn(b
, ":");
133 /* test common stuff first */
135 if (strncasecmp(b
, "http", len
) == 0)
136 return AnyP::PROTO_HTTP
;
138 if (strncasecmp(b
, "ftp", len
) == 0)
139 return AnyP::PROTO_FTP
;
141 if (strncasecmp(b
, "https", len
) == 0)
142 return AnyP::PROTO_HTTPS
;
144 if (strncasecmp(b
, "file", len
) == 0)
145 return AnyP::PROTO_FTP
;
147 if (strncasecmp(b
, "coap", len
) == 0)
148 return AnyP::PROTO_COAP
;
150 if (strncasecmp(b
, "coaps", len
) == 0)
151 return AnyP::PROTO_COAPS
;
153 if (strncasecmp(b
, "gopher", len
) == 0)
154 return AnyP::PROTO_GOPHER
;
156 if (strncasecmp(b
, "wais", len
) == 0)
157 return AnyP::PROTO_WAIS
;
159 if (strncasecmp(b
, "cache_object", len
) == 0)
160 return AnyP::PROTO_CACHE_OBJECT
;
162 if (strncasecmp(b
, "urn", len
) == 0)
163 return AnyP::PROTO_URN
;
165 if (strncasecmp(b
, "whois", len
) == 0)
166 return AnyP::PROTO_WHOIS
;
168 return AnyP::PROTO_NONE
;
174 * If the 'request' arg is non-NULL, put parsed values there instead
175 * of allocating a new HttpRequest.
177 * This abuses HttpRequest as a way of representing the parsed url
178 * and its components.
179 * method is used to switch parsers and to init the HttpRequest.
180 * If method is Http::METHOD_CONNECT, then rather than a URL a hostname:port is
182 * The url is non const so that if its too long we can NULL-terminate it in place.
186 * This routine parses a URL. Its assumed that the URL is complete -
187 * ie, the end of the string is the end of the URL. Don't pass a partial
188 * URL here as this routine doesn't have any way of knowing whether
189 * its partial or not (ie, it handles the case of no trailing slash as
190 * being "end of host with implied path of /".
193 urlParse(const HttpRequestMethod
& method
, char *url
, HttpRequest
*request
)
195 LOCAL_ARRAY(char, proto
, MAX_URL
);
196 LOCAL_ARRAY(char, login
, MAX_URL
);
197 LOCAL_ARRAY(char, host
, MAX_URL
);
198 LOCAL_ARRAY(char, urlpath
, MAX_URL
);
202 AnyP::ProtocolType protocol
= AnyP::PROTO_NONE
;
207 proto
[0] = host
[0] = urlpath
[0] = login
[0] = '\0';
209 if ((l
= strlen(url
)) + Config
.appendDomainLen
> (MAX_URL
- 1)) {
210 /* terminate so it doesn't overflow other buffers */
211 *(url
+ (MAX_URL
>> 1)) = '\0';
212 debugs(23, DBG_IMPORTANT
, "urlParse: URL too large (" << l
<< " bytes)");
215 if (method
== Http::METHOD_CONNECT
) {
218 if (sscanf(url
, "[%[^]]]:%d", host
, &port
) < 1)
219 if (sscanf(url
, "%[^:]:%d", host
, &port
) < 1)
222 } else if ((method
== Http::METHOD_OPTIONS
|| method
== Http::METHOD_TRACE
) &&
223 URL::Asterisk().cmp(url
) == 0) {
224 protocol
= AnyP::PROTO_HTTP
;
225 port
= AnyP::UriScheme(protocol
).defaultPort();
226 return urlParseFinish(method
, protocol
, url
, host
, SBuf(), port
, request
);
227 } else if (!strncmp(url
, "urn:", 4)) {
228 return urnParse(method
, url
, request
);
233 /* Find first : - everything before is protocol */
234 for (i
= 0, dst
= proto
; i
< l
&& *src
!= ':'; ++i
, ++src
, ++dst
) {
242 if ((i
+3) > l
|| *src
!= ':' || *(src
+ 1) != '/' || *(src
+ 2) != '/')
247 /* Then everything until first /; thats host (and port; which we'll look for here later) */
248 // bug 1881: If we don't get a "/" then we imply it was there
249 // bug 3074: We could just be given a "?" or "#". These also imply "/"
250 // bug 3233: whitespace is also a hostname delimiter.
251 for (dst
= host
; i
< l
&& *src
!= '/' && *src
!= '?' && *src
!= '#' && *src
!= '\0' && !xisspace(*src
); ++i
, ++src
, ++dst
) {
256 * We can't check for "i >= l" here because we could be at the end of the line
257 * and have a perfectly valid URL w/ no trailing '/'. In this case we assume we've
258 * been -given- a valid URL and the path is just '/'.
264 // bug 3074: received 'path' starting with '?', '#', or '\0' implies '/'
265 if (*src
== '?' || *src
== '#' || *src
== '\0') {
271 /* Then everything from / (inclusive) until \r\n or \0 - thats urlpath */
272 for (; i
< l
&& *src
!= '\r' && *src
!= '\n' && *src
!= '\0'; ++i
, ++src
, ++dst
) {
276 /* We -could- be at the end of the buffer here */
279 /* If the URL path is empty we set it to be "/" */
280 if (dst
== urlpath
) {
286 protocol
= urlParseProtocol(proto
);
287 port
= AnyP::UriScheme(protocol
).defaultPort();
289 /* Is there any login information? (we should eventually parse it above) */
290 t
= strrchr(host
, '@');
292 strncpy((char *) login
, (char *) host
, sizeof(login
)-1);
293 login
[sizeof(login
)-1] = '\0';
294 t
= strrchr(login
, '@');
296 strncpy((char *) host
, t
+ 1, sizeof(host
)-1);
297 host
[sizeof(host
)-1] = '\0';
300 /* Is there any host information? (we should eventually parse it above) */
302 /* strip any IPA brackets. valid under IPv6. */
304 /* only for IPv6 sadly, pre-IPv6/URL code can't handle the clean result properly anyway. */
309 for (; i
< l
&& *src
!= ']' && *src
!= '\0'; ++i
, ++src
, ++dst
) {
313 /* we moved in-place, so truncate the actual hostname found */
317 /* skip ahead to either start of port, or original EOS */
318 while (*dst
!= '\0' && *dst
!= ':')
322 t
= strrchr(host
, ':');
324 if (t
!= strchr(host
,':') ) {
325 /* RFC 2732 states IPv6 "SHOULD" be bracketed. allowing for times when its not. */
326 /* RFC 3986 'update' simply modifies this to an "is" with no emphasis at all! */
327 /* therefore we MUST accept the case where they are not bracketed at all. */
332 // Bug 3183 sanity check: If scheme is present, host must be too.
333 if (protocol
!= AnyP::PROTO_NONE
&& host
[0] == '\0') {
334 debugs(23, DBG_IMPORTANT
, "SECURITY ALERT: Missing hostname in URL '" << url
<< "'. see access.log for details.");
338 if (t
&& *t
== ':') {
345 for (t
= host
; *t
; ++t
)
348 if (stringHasWhitespace(host
)) {
349 if (URI_WHITESPACE_STRIP
== Config
.uri_whitespace
) {
362 debugs(23, 3, "urlParse: Split URL '" << url
<< "' into proto='" << proto
<< "', host='" << host
<< "', port='" << port
<< "', path='" << urlpath
<< "'");
364 if (Config
.onoff
.check_hostnames
&& strspn(host
, Config
.onoff
.allow_underscore
? valid_hostname_chars_u
: valid_hostname_chars
) != strlen(host
)) {
365 debugs(23, DBG_IMPORTANT
, "urlParse: Illegal character in hostname '" << host
<< "'");
369 /* For IPV6 addresses also check for a colon */
370 if (Config
.appendDomain
&& !strchr(host
, '.') && !strchr(host
, ':'))
371 strncat(host
, Config
.appendDomain
, SQUIDHOSTNAMELEN
- strlen(host
) - 1);
373 /* remove trailing dots from hostnames */
374 while ((l
= strlen(host
)) > 0 && host
[--l
] == '.')
377 /* reject duplicate or leading dots */
378 if (strstr(host
, "..") || *host
== '.') {
379 debugs(23, DBG_IMPORTANT
, "urlParse: Illegal hostname '" << host
<< "'");
383 if (port
< 1 || port
> 65535) {
384 debugs(23, 3, "urlParse: Invalid port '" << port
<< "'");
388 #if HARDCODE_DENY_PORTS
389 /* These ports are filtered in the default squid.conf, but
390 * maybe someone wants them hardcoded... */
391 if (port
== 7 || port
== 9 || port
== 19) {
392 debugs(23, DBG_CRITICAL
, "urlParse: Deny access to port " << port
);
397 if (stringHasWhitespace(urlpath
)) {
398 debugs(23, 2, "urlParse: URI has whitespace: {" << url
<< "}");
400 switch (Config
.uri_whitespace
) {
402 case URI_WHITESPACE_DENY
:
405 case URI_WHITESPACE_ALLOW
:
408 case URI_WHITESPACE_ENCODE
:
409 t
= rfc1738_escape_unescaped(urlpath
);
410 xstrncpy(urlpath
, t
, MAX_URL
);
413 case URI_WHITESPACE_CHOP
:
414 *(urlpath
+ strcspn(urlpath
, w_space
)) = '\0';
417 case URI_WHITESPACE_STRIP
:
431 return urlParseFinish(method
, protocol
, urlpath
, host
, SBuf(login
), port
, request
);
435 * Update request with parsed URI data. If the request arg is
436 * non-NULL, put parsed values there instead of allocating a new
440 urlParseFinish(const HttpRequestMethod
& method
,
441 const AnyP::ProtocolType protocol
,
442 const char *const urlpath
,
443 const char *const host
,
446 HttpRequest
*request
)
449 request
= new HttpRequest(method
, protocol
, urlpath
);
451 request
->initHTTP(method
, protocol
, urlpath
);
454 request
->url
.host(host
);
455 request
->url
.userInfo(login
);
456 request
->url
.port(port
);
461 urnParse(const HttpRequestMethod
& method
, char *urn
, HttpRequest
*request
)
463 debugs(50, 5, "urnParse: " << urn
);
465 request
->initHTTP(method
, AnyP::PROTO_URN
, urn
+ 4);
469 return new HttpRequest(method
, AnyP::PROTO_URN
, urn
+ 4);
476 authorityHttp_
.clear();
477 authorityWithPort_
.clear();
481 URL::authority(bool requirePort
) const
483 if (authorityHttp_
.isEmpty()) {
485 // both formats contain Host/IP
486 authorityWithPort_
.append(host());
487 authorityHttp_
= authorityWithPort_
;
489 // authorityForm_ only has :port if it is non-default
490 authorityWithPort_
.appendf(":%u",port());
491 if (port() != getScheme().defaultPort())
492 authorityHttp_
= authorityWithPort_
;
495 return requirePort
? authorityWithPort_
: authorityHttp_
;
499 URL::absolute() const
501 if (absolute_
.isEmpty()) {
502 // TODO: most URL will be much shorter, avoid allocating this much
503 absolute_
.reserveCapacity(MAX_URL
);
505 absolute_
.appendf("%s:", getScheme().c_str());
506 if (getScheme() != AnyP::PROTO_URN
) {
507 absolute_
.append("//", 2);
508 const bool omitUserInfo
= getScheme() == AnyP::PROTO_HTTP
||
509 getScheme() != AnyP::PROTO_HTTPS
||
510 userInfo().isEmpty();
512 absolute_
.append(userInfo());
513 absolute_
.append("@", 1);
515 absolute_
.append(authority());
517 absolute_
.append(path());
523 /** \todo AYJ: Performance: This is an *almost* duplicate of HttpRequest::effectiveRequestUri(). But elides the query-string.
524 * After copying it on in the first place! Would be less code to merge the two with a flag parameter.
525 * and never copy the query-string part in the first place
528 urlCanonicalClean(const HttpRequest
* request
)
530 LOCAL_ARRAY(char, buf
, MAX_URL
);
532 snprintf(buf
, sizeof(buf
), SQUIDSBUFPH
, SQUIDSBUFPRINT(request
->effectiveRequestUri()));
533 buf
[sizeof(buf
)-1] = '\0';
535 // URN, CONNECT method, and non-stripped URIs can go straight out
536 if (Config
.onoff
.strip_query_terms
&& !(request
->method
== Http::METHOD_CONNECT
|| request
->url
.getScheme() == AnyP::PROTO_URN
)) {
537 // strip anything AFTER a question-mark
538 // leaving the '?' in place
539 if (auto t
= strchr(buf
, '?')) {
544 if (stringHasCntl(buf
))
545 xstrncpy(buf
, rfc1738_escape_unescaped(buf
), MAX_URL
);
551 * Yet another alternative to urlCanonical.
552 * This one adds the https:// parts to Http::METHOD_CONNECT URL
553 * for use in error page outputs.
554 * Luckily we can leverage the others instead of duplicating.
557 urlCanonicalFakeHttps(const HttpRequest
* request
)
559 LOCAL_ARRAY(char, buf
, MAX_URL
);
561 // method CONNECT and port HTTPS
562 if (request
->method
== Http::METHOD_CONNECT
&& request
->url
.port() == 443) {
563 snprintf(buf
, MAX_URL
, "https://%s/*", request
->url
.host());
567 // else do the normal complete canonical thing.
568 return urlCanonicalClean(request
);
572 * Test if a URL is relative.
574 * RFC 2396, Section 5 (Page 17) implies that in a relative URL, a '/' will
575 * appear before a ':'.
578 urlIsRelative(const char *url
)
589 for (p
= url
; *p
!= '\0' && *p
!= ':' && *p
!= '/'; ++p
);
598 * Convert a relative URL to an absolute URL using the context of a given
601 * It is assumed that you have already ensured that the URL is relative.
603 * If NULL is returned it is an indication that the method in use in the
604 * request does not distinguish between relative and absolute and you should
605 * use the url unchanged.
607 * If non-NULL is returned, it is up to the caller to free the resulting
608 * memory using safe_free().
611 urlMakeAbsolute(const HttpRequest
* req
, const char *relUrl
)
614 if (req
->method
.id() == Http::METHOD_CONNECT
) {
618 char *urlbuf
= (char *)xmalloc(MAX_URL
* sizeof(char));
620 if (req
->url
.getScheme() == AnyP::PROTO_URN
) {
621 // XXX: this is what the original code did, but it seems to break the
622 // intended behaviour of this function. It returns the stored URN path,
623 // not converting the given one into a URN...
624 snprintf(urlbuf
, MAX_URL
, SQUIDSBUFPH
, SQUIDSBUFPRINT(req
->url
.absolute()));
628 SBuf authorityForm
= req
->url
.authority(); // host[:port]
629 size_t urllen
= snprintf(urlbuf
, MAX_URL
, "%s://" SQUIDSBUFPH
"%s" SQUIDSBUFPH
,
630 req
->url
.getScheme().c_str(),
631 SQUIDSBUFPRINT(req
->url
.userInfo()),
632 !req
->url
.userInfo().isEmpty() ? "@" : "",
633 SQUIDSBUFPRINT(authorityForm
));
635 // if the first char is '/' assume its a relative path
636 // XXX: this breaks on scheme-relative URLs,
637 // but we should not see those outside ESI, and rarely there.
638 // XXX: also breaks on any URL containing a '/' in the query-string portion
639 if (relUrl
[0] == '/') {
640 xstrncpy(&urlbuf
[urllen
], relUrl
, MAX_URL
- urllen
- 1);
642 SBuf path
= req
->url
.path();
643 SBuf::size_type lastSlashPos
= path
.rfind('/');
645 if (lastSlashPos
== SBuf::npos
) {
646 // replace the whole path with the given bit(s)
647 urlbuf
[urllen
] = '/';
649 xstrncpy(&urlbuf
[urllen
], relUrl
, MAX_URL
- urllen
- 1);
651 // replace only the last (file?) segment with the given bit(s)
653 if (lastSlashPos
> MAX_URL
- urllen
- 1) {
654 // XXX: crops bits in the middle of the combined URL.
655 lastSlashPos
= MAX_URL
- urllen
- 1;
657 xstrncpy(&urlbuf
[urllen
], path
.rawContent(), lastSlashPos
);
658 urllen
+= lastSlashPos
;
659 if (urllen
+ 1 < MAX_URL
) {
660 xstrncpy(&urlbuf
[urllen
], relUrl
, MAX_URL
- urllen
- 1);
669 matchDomainName(const char *h
, const char *d
, bool honorWildcards
)
682 * Start at the ends of the two strings and work towards the
685 while (xtolower(h
[--hl
]) == xtolower(d
[--dl
])) {
686 if (hl
== 0 && dl
== 0) {
688 * We made it all the way to the beginning of both
689 * strings without finding any difference.
696 * The host string is shorter than the domain string.
697 * There is only one case when this can be a match.
698 * If the domain is just one character longer, and if
699 * that character is a leading '.' then we call it a
703 if (1 == dl
&& '.' == d
[0])
711 * The domain string is shorter than the host string.
712 * This is a match only if the first domain character
724 * We found different characters in the same position (from the end).
727 // If the h has a form of "*.foo.com" and d has a form of "x.foo.com"
728 // then the h[hl] points to '*', h[hl+1] to '.' and d[dl] to 'x'
729 // The following checks are safe, the "h[hl + 1]" in the worst case is '\0'.
730 if (honorWildcards
&& h
[hl
] == '*' && h
[hl
+ 1] == '.')
734 * If one of those character is '.' then its special. In order
735 * for splay tree sorting to work properly, "x-foo.com" must
736 * be greater than ".foo.com" even though '-' is less than '.'.
744 return (xtolower(h
[hl
]) - xtolower(d
[dl
]));
748 * return true if we can serve requests for this method.
751 urlCheckRequest(const HttpRequest
* r
)
754 /* protocol "independent" methods
756 * actually these methods are specific to HTTP:
757 * they are methods we recieve on our HTTP port,
758 * and if we had a FTP listener would not be relevant
761 * So, we should delegate them to HTTP. The problem is that we
762 * do not have a default protocol from the client side of HTTP.
765 if (r
->method
== Http::METHOD_CONNECT
)
768 // we support OPTIONS and TRACE directed at us (with a 501 reply, for now)
769 // we also support forwarding OPTIONS and TRACE, except for the *-URI ones
770 if (r
->method
== Http::METHOD_OPTIONS
|| r
->method
== Http::METHOD_TRACE
)
771 return (r
->header
.getInt64(HDR_MAX_FORWARDS
) == 0 || r
->url
.path() != URL::Asterisk());
773 if (r
->method
== Http::METHOD_PURGE
)
776 /* does method match the protocol? */
777 switch (r
->url
.getScheme()) {
779 case AnyP::PROTO_URN
:
781 case AnyP::PROTO_HTTP
:
783 case AnyP::PROTO_CACHE_OBJECT
:
787 case AnyP::PROTO_FTP
:
789 if (r
->method
== Http::METHOD_PUT
)
792 case AnyP::PROTO_GOPHER
:
794 case AnyP::PROTO_WAIS
:
796 case AnyP::PROTO_WHOIS
:
797 if (r
->method
== Http::METHOD_GET
)
799 else if (r
->method
== Http::METHOD_HEAD
)
804 case AnyP::PROTO_HTTPS
:
813 * Squid can't originate an SSL connection, so it should
814 * never receive an "https:" URL. It should always be
829 * Quick-n-dirty host extraction from a URL. Steps:
831 * Skip any '/' after the colon
832 * Copy the next SQUID_MAXHOSTNAMELEN bytes to host[]
833 * Look for an ending '/' or ':' and terminate
834 * Look for login info preceeded by '@'
841 char * extract(char const *url
);
844 static char Host
[SQUIDHOSTNAMELEN
];
845 void init(char const *);
846 void findHostStart();
847 void trimTrailingChars();
849 char const *hostStart
;
854 urlHostname(const char *url
)
856 return URLHostName().extract(url
);
859 char URLHostName::Host
[SQUIDHOSTNAMELEN
];
862 URLHostName::init(char const *aUrl
)
869 URLHostName::findHostStart()
871 if (NULL
== (hostStart
= strchr(url
, ':')))
876 while (*hostStart
!= '\0' && *hostStart
== '/')
879 if (*hostStart
== ']')
884 URLHostName::trimTrailingChars()
888 if ((t
= strchr(Host
, '/')))
891 if ((t
= strrchr(Host
, ':')))
894 if ((t
= strchr(Host
, ']')))
899 URLHostName::trimAuth()
903 if ((t
= strrchr(Host
, '@'))) {
905 memmove(Host
, t
, strlen(t
) + 1);
910 URLHostName::extract(char const *aUrl
)
915 if (hostStart
== NULL
)
918 xstrncpy(Host
, hostStart
, SQUIDHOSTNAMELEN
);