From: Gregor Jasny Date: Wed, 14 Jul 2021 07:26:13 +0000 (+0200) Subject: HTTP storage: Implement authentication support (#895) X-Git-Tag: v4.4~129 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=6b69c698ca90a0c6635987e7a7b0c3b1ff6adea3;p=thirdparty%2Fccache.git HTTP storage: Implement authentication support (#895) Closes #887. --- diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index 6b66f9aa6..6cd295908 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -945,12 +945,11 @@ Note that ccache will not perform any cleanup of the HTTP storage. Examples: * `http://localhost:8080/` -* `http://example.com/cache` +* `http://someusername:p4ssw0rd@example.com/cache` Known issues and limitations: * There are no HTTP timeouts implemented or configured. -* Authentication is not yet supported. * HTTPS is not yet supported. * URLs containing IPv6 addresses like `http://[::1]/` are not supported. diff --git a/src/storage/secondary/HttpStorage.cpp b/src/storage/secondary/HttpStorage.cpp index 7ec99c348..3d16e42b4 100644 --- a/src/storage/secondary/HttpStorage.cpp +++ b/src/storage/secondary/HttpStorage.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -71,18 +72,6 @@ to_string(const httplib::Error error) return "Unknown"; } -int -get_url_port(const Url& url) -{ - if (!url.port().empty()) { - return Util::parse_unsigned(url.port(), 1, 65535, "port"); - } else if (url.scheme() == "http") { - return 80; - } else { - throw Error("Unknown scheme: {}", url.scheme()); - } -} - std::string get_url_path(const Url& url) { @@ -93,12 +82,36 @@ get_url_path(const Url& url) return path; } +std::unique_ptr +make_client(const Url& url) +{ + std::string scheme_host_port; + + if (url.port().empty()) { + scheme_host_port = FMT("{}://{}", url.scheme(), url.host()); + } else { + scheme_host_port = FMT("{}://{}:{}", url.scheme(), url.host(), url.port()); + } + + auto client = std::make_unique(scheme_host_port.c_str()); + if (!url.user_info().empty()) { + const auto pair = util::split_once(url.user_info(), ':'); + if (!pair.second) { + throw Error("Expected username:password in URL but got: '{}'", + url.user_info()); + } + client->set_basic_auth(nonstd::sv_lite::to_string(pair.first).c_str(), + nonstd::sv_lite::to_string(*pair.second).c_str()); + } + + return client; +} + } // namespace HttpStorage::HttpStorage(const Url& url, const AttributeMap&) : m_url_path(get_url_path(url)), - m_http_client( - std::make_unique(url.host(), get_url_port(url))) + m_http_client(make_client(url)) { m_http_client->set_default_headers( {{"User-Agent", FMT("{}/{}", CCACHE_NAME, CCACHE_VERSION)}}); diff --git a/test/http-client b/test/http-client index 1b9eba50b..54739fd4d 100755 --- a/test/http-client +++ b/test/http-client @@ -7,9 +7,13 @@ import sys import time import urllib.request -def run(url, timeout): +def run(url, timeout, basic_auth): deadline = time.time() + timeout req = urllib.request.Request(url, method="HEAD") + if basic_auth: + import base64 + encoded_credentials = base64.b64encode(basic_auth.encode("ascii")).decode("ascii") + req.add_header("Authorization", f"Basic {encoded_credentials}") while True: try: response = urllib.request.urlopen(req) @@ -26,6 +30,8 @@ if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() + parser.add_argument('--basic-auth', '-B', + help='Basic auth tuple like user:pass') parser.add_argument('--timeout', '-t', metavar='TIMEOUT', default=10, type=int, help='Maximum seconds to wait for successful connection attempt ' @@ -38,4 +44,5 @@ if __name__ == "__main__": run( url=args.url, timeout=args.timeout, + basic_auth=args.basic_auth, ) diff --git a/test/http-server b/test/http-server index d061fee2c..390e7bdef 100755 --- a/test/http-server +++ b/test/http-server @@ -14,29 +14,70 @@ import os import socket import sys +class AuthenticationError(Exception): + pass + class PUTEnabledHTTPRequestHandler(SimpleHTTPRequestHandler): + def __init__(self, *args, basic_auth=None, **kwargs): + self.basic_auth = None + if basic_auth: + import base64 + self.basic_auth = base64.b64encode(basic_auth.encode("ascii")).decode("ascii") + super().__init__(*args, **kwargs) + + def do_GET(self): + try: + self._handle_auth() + super().do_GET() + except AuthenticationError: + self.send_error(HTTPStatus.UNAUTHORIZED, "Need Authentication") + + def do_HEAD(self): + try: + self._handle_auth() + super().do_HEAD() + except AuthenticationError: + self.send_error(HTTPStatus.UNAUTHORIZED, "Need Authentication") + def do_PUT(self): path = self.translate_path(self.path) try: + self._handle_auth() file_length = int(self.headers['Content-Length']) with open(path, 'wb') as output_file: output_file.write(self.rfile.read(file_length)) self.send_response(HTTPStatus.CREATED) self.send_header("Content-Length", "0") self.end_headers() + except AuthenticationError: + self.send_error(HTTPStatus.UNAUTHORIZED, "Need Authentication") except OSError: self.send_error(HTTPStatus.INTERNAL_SERVER_ERROR, "Cannot open file for writing") def do_DELETE(self): path = self.translate_path(self.path) try: + self._handle_auth() os.remove(path) self.send_response(HTTPStatus.OK) self.send_header("Content-Length", "0") self.end_headers() + except AuthenticationError: + self.send_error(HTTPStatus.UNAUTHORIZED, "Need Authentication") except OSError: self.send_error(HTTPStatus.INTERNAL_SERVER_ERROR, "Cannot delete file") + def _handle_auth(self): + if not self.basic_auth: + return + authorization = self.headers.get("authorization") + if authorization: + authorization = authorization.split() + if len(authorization) == 2: + if authorization[0] == "Basic" and authorization[1] == self.basic_auth: + return + raise AuthenticationError("Authentication required") + def _get_best_family(*address): infos = socket.getaddrinfo( *address, @@ -50,7 +91,7 @@ def run(HandlerClass, ServerClass, port, bind): HandlerClass.protocol_version = "HTTP/1.1" ServerClass.address_family, addr = _get_best_family(bind, port) - with ServerClass(addr, handler_class) as httpd: + with ServerClass(addr, HandlerClass) as httpd: host, port = httpd.socket.getsockname()[:2] url_host = f'[{host}]' if ':' in host else host print( @@ -67,6 +108,8 @@ if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() + parser.add_argument('--basic-auth', '-B', + help='Basic auth tuple like user:pass') parser.add_argument('--bind', '-b', metavar='ADDRESS', help='Specify alternate bind address ' '[default: all interfaces]') @@ -79,12 +122,12 @@ if __name__ == "__main__": help='Specify alternate port [default: 8080]') args = parser.parse_args() - handler_class = partial(PUTEnabledHTTPRequestHandler) + handler_class = partial(PUTEnabledHTTPRequestHandler, basic_auth=args.basic_auth) os.chdir(args.directory) run( - HandlerClass=PUTEnabledHTTPRequestHandler, + HandlerClass=handler_class, ServerClass=HTTPServer, port=args.port, bind=args.bind, diff --git a/test/suites/secondary_http.bash b/test/suites/secondary_http.bash index 893b4d52d..fa4a8b994 100644 --- a/test/suites/secondary_http.bash +++ b/test/suites/secondary_http.bash @@ -1,11 +1,14 @@ start_http_server() { local port="$1" local cache_dir="$2" + local credentials="$3" # optional parameter mkdir -p "${cache_dir}" "${HTTP_SERVER}" --bind localhost --directory "${cache_dir}" "${port}" \ + ${credentials:+--basic-auth ${credentials}} \ &>http-server.log & "${HTTP_CLIENT}" "http://localhost:${port}" &>http-client.log \ + ${credentials:+--basic-auth ${credentials}} \ || test_failed_internal "Cannot connect to server" } @@ -50,4 +53,43 @@ SUITE_secondary_http() { expect_stat 'files in cache' 0 expect_stat 'files in cache' 0 expect_file_count 2 '*' secondary # result + manifest + + # ------------------------------------------------------------------------- + TEST "Basic auth" + + start_http_server 12780 secondary "somebody:secret" + export CCACHE_SECONDARY_STORAGE="http://somebody:secret@localhost:12780" + + $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 0 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 2 + expect_file_count 2 '*' secondary # result + manifest + + # ------------------------------------------------------------------------- + TEST "Basic auth required" + + start_http_server 12780 secondary "somebody:secret" + # no authentication configured on client + export CCACHE_SECONDARY_STORAGE="http://localhost:12780" + + CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 0 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 2 + expect_file_count 0 '*' secondary # result + manifest + expect_contains test.o.ccache-log "status code: 401" + + # ------------------------------------------------------------------------- + TEST "Basic auth failed" + + start_http_server 12780 secondary "somebody:secret" + export CCACHE_SECONDARY_STORAGE="http://somebody:wrong@localhost:12780" + + CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c + expect_stat 'cache hit (direct)' 0 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 2 + expect_file_count 0 '*' secondary # result + manifest + expect_contains test.o.ccache-log "status code: 401" }