From: Stefan Eissing Date: Thu, 11 Dec 2025 15:02:41 +0000 (+0100) Subject: pytest: add tests using sshd X-Git-Tag: rc-8_18_0-2~27 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=eb39fee40be6a8e68be2551e36b6fcb94170aaed;p=thirdparty%2Fcurl.git pytest: add tests using sshd With either /usr/sbin/sshd found or configured via --with-test-sshd=path add tests for SCP down- and uploads, insecure, with known hosts or not, with authorized user key or unauthorized one. Working now with libssh and libssh2, using a hashed known_hosts file. Closes #19934 --- diff --git a/configure.ac b/configure.ac index e465ee6a8a..f0e439fa23 100644 --- a/configure.ac +++ b/configure.ac @@ -409,6 +409,51 @@ if test "$DANTED_ENABLED" = "no"; then fi AC_SUBST(DANTED) +dnl we would like a sshd as test server +dnl +SSHD_ENABLED="maybe" +AC_ARG_WITH(test-sshd, [AS_HELP_STRING([--with-test-sshd=PATH], + [where to find sshd for testing])], + [request_sshd=$withval], [request_sshd=check]) +if test "x$request_sshd" = "xcheck" || test "x$request_sshd" = "xyes"; then + if test -x "/usr/sbin/sshd"; then + # common location on distros (debian/ubuntu) + SSHD="/usr/sbin/sshd" + else + AC_PATH_PROG([SSHD], [sshd]) + if test -z "$SSHD"; then + AC_PATH_PROG([SSHD], [sshd]) + fi + fi +elif test "x$request_sshd" != "xno"; then + SSHD="${request_sshd}" + if test ! -x "${SSHD}"; then + AC_MSG_NOTICE([sshd not found as ${SSHD}, sshd tests disabled]) + SSHD_ENABLED="no" + else + AC_MSG_NOTICE([using SSHD=$SSHD for tests]) + fi +fi +if test "$SSHD_ENABLED" = "no"; then + SSHD="" + SFTPD="" +else + if test -x "/usr/libexec/sftp-server"; then + # common location on macOS) + SFTPD="/usr/libexec/sftp-server" + elif test -x "/usr/lib/openssh/sftp-server"; then + # common location on debian + SFTPD="/usr/lib/openssh/sftp-server" + else + AC_PATH_PROG([SFTPD], [sftp-server]) + if test -z "$SFTPD"; then + AC_PATH_PROG([SFTPD], [sftp-server]) + fi + fi +fi +AC_SUBST(SSHD) +AC_SUBST(SFTPD) + dnl the nghttpx we might use in httpd testing if test -n "$TEST_NGHTTPX" && test "x$TEST_NGHTTPX" != "xnghttpx"; then HTTPD_NGHTTPX="$TEST_NGHTTPX" diff --git a/docs/INSTALL-CMAKE.md b/docs/INSTALL-CMAKE.md index 026db07628..b00cc01030 100644 --- a/docs/INSTALL-CMAKE.md +++ b/docs/INSTALL-CMAKE.md @@ -483,6 +483,8 @@ Examples: - `DANTED`: Default: `danted` - `TEST_NGHTTPX`: Default: `nghttpx` - `VSFTPD`: Default: `vsftps` +- `SSHD`: Default: `sshd` +- `SFTPD`: Default: `sftp-server` ## Feature detection variables diff --git a/tests/http/CMakeLists.txt b/tests/http/CMakeLists.txt index c53a8a32dc..62deb173a1 100644 --- a/tests/http/CMakeLists.txt +++ b/tests/http/CMakeLists.txt @@ -58,5 +58,17 @@ if(NOT DANTED) endif() mark_as_advanced(DANTED) -# Consumed variables: APXS, CADDY, HTTPD, HTTPD_NGHTTPX, DANTED, VSFTPD +find_program(SSHD "sshd" PATHS "/usr/sbin" "/usr/bin") +if(NOT SSHD) + set(SSHD "") +endif() +mark_as_advanced(SSHD) + +find_program(SFTPD "sftp-server" PATHS "/usr/libexec" "/usr/lib/openssh") +if(NOT SFTPD) + set(SFTPD "") +endif() +mark_as_advanced(SFTPD) + +# Consumed variables: APXS, CADDY, HTTPD, HTTPD_NGHTTPX, DANTED, VSFTPD, SSHD configure_file("config.ini.in" "${CMAKE_CURRENT_BINARY_DIR}/config.ini" @ONLY) diff --git a/tests/http/Makefile.am b/tests/http/Makefile.am index fc5bad0bd0..a9990be9c3 100644 --- a/tests/http/Makefile.am +++ b/tests/http/Makefile.am @@ -34,6 +34,7 @@ testenv/httpd.py \ testenv/mod_curltest/mod_curltest.c \ testenv/nghttpx.py \ testenv/ports.py \ +testenv/sshd.py \ testenv/vsftpd.py \ testenv/ws_echo_server.py @@ -65,7 +66,9 @@ test_20_websockets.py \ test_30_vsftpd.py \ test_31_vsftpds.py \ test_32_ftps_vsftpd.py \ -test_40_socks.py \ +test_40_socks.py \ +test_50_scp.py \ +test_51_sftp.py \ $(TESTENV) clean-local: diff --git a/tests/http/config.ini.in b/tests/http/config.ini.in index 5fe51a6465..78808e966d 100644 --- a/tests/http/config.ini.in +++ b/tests/http/config.ini.in @@ -40,3 +40,7 @@ vsftpd = @VSFTPD@ [danted] danted = @DANTED@ + +[sshd] +sshd = @SSHD@ +sftpd = @SFTPD@ diff --git a/tests/http/conftest.py b/tests/http/conftest.py index 418ad10ec8..79ed3eacca 100644 --- a/tests/http/conftest.py +++ b/tests/http/conftest.py @@ -34,7 +34,7 @@ from testenv.env import EnvConfig sys.path.append(os.path.join(os.path.dirname(__file__), '.')) -from testenv import Env, Nghttpx, Httpd, NghttpxQuic, NghttpxFwd +from testenv import Env, Nghttpx, Httpd, NghttpxQuic, NghttpxFwd, Sshd log = logging.getLogger(__name__) @@ -134,6 +134,17 @@ def nghttpx_fwd(env, httpd) -> Generator[Union[Nghttpx,bool], None, None]: yield False +@pytest.fixture(scope='session') +def sshd(env: Env) -> Generator[Union[Sshd,bool], None, None]: + if env.has_sshd(): + sshd = Sshd(env=env) + assert sshd.initial_start(), f'{sshd.dump_log()}' + yield sshd + sshd.stop() + else: + yield False + + @pytest.fixture(scope='session') def configures_httpd(env, httpd) -> Generator[bool, None, None]: # include this fixture as test parameter if the test configures httpd itself diff --git a/tests/http/test_50_scp.py b/tests/http/test_50_scp.py new file mode 100644 index 0000000000..56f1d61281 --- /dev/null +++ b/tests/http/test_50_scp.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) Daniel Stenberg, , et al. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +# SPDX-License-Identifier: curl +# +########################################################################### +# +import difflib +import filecmp +import logging +import os +import pytest + +from testenv import Env, CurlClient, Sshd + + +log = logging.getLogger(__name__) + + +@pytest.mark.skipif(condition=not Env.curl_has_protocol('scp'), reason="curl built without scp:") +@pytest.mark.skipif(condition=not Env.has_sshd(), reason="missing sshd") +class TestScp: + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env, sshd): + env.make_data_file(indir=sshd.home_dir, fname="data-10k", fsize=10*1024) + env.make_data_file(indir=sshd.home_dir, fname="data-10m", fsize=10*1024*1024) + env.make_data_file(indir=env.gen_dir, fname="data-10k", fsize=10*1024) + env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10*1024*1024) + + def test_50_01_insecure(self, env: Env, sshd: Sshd): + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data') + url = f'scp://{env.domain1}:{sshd.port}/{doc_file}' + r = curl.ssh_download(urls=[url], extra_args=[ + '--insecure', + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + + def test_50_02_unknown_hosts(self, env: Env, sshd: Sshd): + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data') + url = f'scp://{env.domain1}:{sshd.port}/{doc_file}' + r = curl.ssh_download(urls=[url], extra_args=[ + '--knownhosts', sshd.unknown_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(60) # CURLE_PEER_FAILED_VERIFICATION + + def test_50_03_known_hosts(self, env: Env, sshd: Sshd): + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data') + url = f'scp://{env.domain1}:{sshd.port}/{doc_file}' + r = curl.ssh_download(urls=[url], extra_args=[ + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + + # use key not in authorized_keys file + def test_50_04_unauth_user(self, env: Env, sshd: Sshd): + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data') + url = f'scp://{env.domain1}:{sshd.port}/{doc_file}' + r = curl.ssh_download(urls=[url], extra_args=[ + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user2_pubkey_file, + '--key', sshd.user2_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(67) # CURLE_LOGIN_DENIED + + def test_50_10_dl_single(self, env: Env, sshd: Sshd): + count = 1 + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data-10k') + url = f'scp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]' + r = curl.ssh_download(urls=[url], extra_args=[ + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + self.check_downloads(curl, doc_file, count) + + def test_50_11_dl_serial(self, env: Env, sshd: Sshd): + count = 5 + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data-10k') + url = f'scp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]' + r = curl.ssh_download(urls=[url], extra_args=[ + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + self.check_downloads(curl, doc_file, count) + + def test_50_12_dl_parallel(self, env: Env, sshd: Sshd): + count = 5 + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data-10k') + url = f'scp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]' + r = curl.http_download(urls=[url], extra_args=[ + '--parallel', + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + self.check_downloads(curl, doc_file, count) + + def test_50_20_ul_single(self, env: Env, sshd: Sshd): + srcfile = os.path.join(env.gen_dir, 'data-10k') + destfile = os.path.join(sshd.home_dir, 'upload_20.data') + curl = CurlClient(env=env) + url = f'scp://{env.domain1}:{sshd.port}/{destfile}' + r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[ + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + self.check_upload(sshd, srcfile, destfile) + + def test_50_21_ul_serial(self, env: Env, sshd: Sshd): + count = 5 + srcfile = os.path.join(env.gen_dir, 'data-10k') + destfile = os.path.join(sshd.home_dir, 'upload_21.data') + curl = CurlClient(env=env) + url = f'scp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]' + r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[ + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + for i in range(count): + self.check_upload(sshd, srcfile, f'{destfile}.{i}') + + def test_50_22_ul_parallel(self, env: Env, sshd: Sshd): + count = 5 + srcfile = os.path.join(env.gen_dir, 'data-10k') + destfile = os.path.join(sshd.home_dir, 'upload_22.data') + curl = CurlClient(env=env) + url = f'scp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]' + r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[ + '--parallel', + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + for i in range(count): + self.check_upload(sshd, srcfile, f'{destfile}.{i}') + + def check_downloads(self, client, srcfile: str, count: int, + complete: bool = True): + for i in range(count): + dfile = client.download_file(i) + assert os.path.exists(dfile) + if complete and not filecmp.cmp(srcfile, dfile, shallow=False): + diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(), + b=open(dfile).readlines(), + fromfile=srcfile, + tofile=dfile, + n=1)) + assert False, f'download {dfile} differs:\n{diff}' + + def check_upload(self, sshd: Sshd, srcfile, destfile, binary=True): + assert os.path.exists(srcfile) + assert os.path.exists(destfile) + if not filecmp.cmp(srcfile, destfile, shallow=False): + diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(), + b=open(destfile).readlines(), + fromfile=srcfile, + tofile=destfile, + n=1)) + assert not binary and len(diff) == 0, f'upload {destfile} differs:\n{diff}' diff --git a/tests/http/test_51_sftp.py b/tests/http/test_51_sftp.py new file mode 100644 index 0000000000..ec4de35d53 --- /dev/null +++ b/tests/http/test_51_sftp.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) Daniel Stenberg, , et al. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +# SPDX-License-Identifier: curl +# +########################################################################### +# +import difflib +import filecmp +import logging +import os +import pytest + +from testenv import Env, CurlClient, Sshd + + +log = logging.getLogger(__name__) + + +@pytest.mark.skipif(condition=not Env.curl_has_protocol('sftp'), reason="curl built without sfto:") +@pytest.mark.skipif(condition=not Env.has_sftpd(), reason="missing sftp server") +class TestSftp: + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env, sshd): + env.make_data_file(indir=sshd.home_dir, fname="data-10k", fsize=10*1024) + env.make_data_file(indir=sshd.home_dir, fname="data-10m", fsize=10*1024*1024) + env.make_data_file(indir=env.gen_dir, fname="data-10k", fsize=10*1024) + env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10*1024*1024) + + def test_51_01_insecure(self, env: Env, sshd: Sshd): + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data') + url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}' + r = curl.ssh_download(urls=[url], extra_args=[ + '--insecure', + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + + def test_51_02_unknown_hosts(self, env: Env, sshd: Sshd): + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data') + url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}' + r = curl.ssh_download(urls=[url], extra_args=[ + '--knownhosts', sshd.unknown_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(60) # CURLE_PEER_FAILED_VERIFICATION + + def test_51_03_known_hosts(self, env: Env, sshd: Sshd): + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data') + url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}' + r = curl.ssh_download(urls=[url], extra_args=[ + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + + # use key not in authorized_keys file + def test_51_04_unauth_user(self, env: Env, sshd: Sshd): + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data') + url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}' + r = curl.ssh_download(urls=[url], extra_args=[ + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user2_pubkey_file, + '--key', sshd.user2_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(67) # CURLE_LOGIN_DENIED + + def test_51_10_dl_single(self, env: Env, sshd: Sshd): + count = 1 + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data-10k') + url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]' + r = curl.ssh_download(urls=[url], extra_args=[ + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + self.check_downloads(curl, doc_file, count) + + def test_51_11_dl_serial(self, env: Env, sshd: Sshd): + count = 5 + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data-10k') + url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]' + r = curl.ssh_download(urls=[url], extra_args=[ + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + self.check_downloads(curl, doc_file, count) + + def test_51_12_dl_parallel(self, env: Env, sshd: Sshd): + count = 5 + curl = CurlClient(env=env) + doc_file = os.path.join(sshd.home_dir, 'data-10k') + url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]' + r = curl.http_download(urls=[url], extra_args=[ + '--parallel', + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + self.check_downloads(curl, doc_file, count) + + def test_51_20_ul_single(self, env: Env, sshd: Sshd): + srcfile = os.path.join(env.gen_dir, 'data-10k') + destfile = os.path.join(sshd.home_dir, 'upload_20.data') + curl = CurlClient(env=env) + url = f'sftp://{env.domain1}:{sshd.port}/{destfile}' + r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[ + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + self.check_upload(sshd, srcfile, destfile) + + def test_51_21_ul_serial(self, env: Env, sshd: Sshd): + count = 5 + srcfile = os.path.join(env.gen_dir, 'data-10k') + destfile = os.path.join(sshd.home_dir, 'upload_21.data') + curl = CurlClient(env=env) + url = f'sftp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]' + r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[ + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + for i in range(count): + self.check_upload(sshd, srcfile, f'{destfile}.{i}') + + def test_51_22_ul_parallel(self, env: Env, sshd: Sshd): + count = 5 + srcfile = os.path.join(env.gen_dir, 'data-10k') + destfile = os.path.join(sshd.home_dir, 'upload_22.data') + curl = CurlClient(env=env) + url = f'sftp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]' + r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[ + '--parallel', + '--knownhosts', sshd.known_hosts, + '--pubkey', sshd.user1_pubkey_file, + '--key', sshd.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + r.check_exit_code(0) + for i in range(count): + self.check_upload(sshd, srcfile, f'{destfile}.{i}') + + def check_downloads(self, client, srcfile: str, count: int, + complete: bool = True): + for i in range(count): + dfile = client.download_file(i) + assert os.path.exists(dfile) + if complete and not filecmp.cmp(srcfile, dfile, shallow=False): + diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(), + b=open(dfile).readlines(), + fromfile=srcfile, + tofile=dfile, + n=1)) + assert False, f'download {dfile} differs:\n{diff}' + + def check_upload(self, sshd: Sshd, srcfile, destfile, binary=True): + assert os.path.exists(srcfile) + assert os.path.exists(destfile) + if not filecmp.cmp(srcfile, destfile, shallow=False): + diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(), + b=open(destfile).readlines(), + fromfile=srcfile, + tofile=destfile, + n=1)) + assert not binary and len(diff) == 0, f'upload {destfile} differs:\n{diff}' diff --git a/tests/http/testenv/__init__.py b/tests/http/testenv/__init__.py index fb2f9e53fe..3c26780d01 100644 --- a/tests/http/testenv/__init__.py +++ b/tests/http/testenv/__init__.py @@ -37,3 +37,4 @@ from .client import LocalClient from .nghttpx import Nghttpx, NghttpxQuic, NghttpxFwd from .vsftpd import VsFTPD from .dante import Dante +from .sshd import Sshd diff --git a/tests/http/testenv/curl.py b/tests/http/testenv/curl.py index d9fe0706ac..c78e262164 100644 --- a/tests/http/testenv/curl.py +++ b/tests/http/testenv/curl.py @@ -916,6 +916,65 @@ class CurlClient: with_tcpdump=with_tcpdump, extra_args=extra_args) + def ssh_download(self, urls: List[str], + with_stats: bool = True, + with_profile: bool = False, + with_tcpdump: bool = False, + no_save: bool = False, + extra_args: Optional[List[str]] = None): + if extra_args is None: + extra_args = [] + if no_save: + extra_args.extend([ + '--out-null', + ]) + else: + extra_args.extend([ + '-o', 'download_#1.data', + ]) + # remove any existing ones + for i in range(100): + self._rmf(self.download_file(i)) + if with_stats: + extra_args.extend([ + '-w', '%{json}\\n' + ]) + return self._raw(urls, options=extra_args, + with_stats=with_stats, + with_headers=False, + with_profile=with_profile, + with_tcpdump=with_tcpdump) + + def ssh_upload(self, urls: List[str], + fupload: Optional[Any] = None, + updata: Optional[str] = None, + with_stats: bool = True, + with_profile: bool = False, + with_tcpdump: bool = False, + extra_args: Optional[List[str]] = None): + if extra_args is None: + extra_args = [] + if fupload is not None: + extra_args.extend([ + '--upload-file', fupload + ]) + elif updata is not None: + extra_args.extend([ + '--upload-file', '-' + ]) + else: + raise Exception('need either file or data to upload') + if with_stats: + extra_args.extend([ + '-w', '%{json}\\n' + ]) + return self._raw(urls, options=extra_args, + intext=updata, + with_stats=with_stats, + with_headers=False, + with_profile=with_profile, + with_tcpdump=with_tcpdump) + def response_file(self, idx: int): return os.path.join(self._run_dir, f'download_{idx}.data') diff --git a/tests/http/testenv/env.py b/tests/http/testenv/env.py index 5e6bcfc687..8fbc3bd2f3 100644 --- a/tests/http/testenv/env.py +++ b/tests/http/testenv/env.py @@ -298,6 +298,35 @@ class EnvConfig: except Exception: self.danted = None + self.sshd = self.config['sshd']['sshd'] + if self.sshd == '': + self.sshd = None + self._sshd_version = None + if self.sshd is not None: + try: + p = subprocess.run(args=[self.sshd, '-V'], + capture_output=True, text=True) + assert p.returncode == 0 + if p.returncode != 0: + self.sshd = None + else: + m = re.match(r'^OpenSSH_(\d+\.\d+.*),.*', p.stderr) + assert m, f'version: {p.stderr}' + if m: + self._sshd_version = m.group(1) + else: + self.sshd = None + raise Exception(f'Unable to determine sshd version from: {p.stderr}') + except Exception: + self.sshd = None + + if self.sshd: + self.sftpd = self.config['sshd']['sftpd'] + if self.sftpd == '': + self.sftpd = None + else: + self.sftpd = None + self._tcpdump = shutil.which('tcpdump') @property @@ -576,6 +605,14 @@ class Env: def has_danted() -> bool: return Env.CONFIG.danted is not None + @staticmethod + def has_sshd() -> bool: + return Env.CONFIG.sshd is not None + + @staticmethod + def has_sftpd() -> bool: + return Env.has_sshd() and Env.CONFIG.sftpd is not None + @staticmethod def tcpdump() -> Optional[str]: return Env.CONFIG.tcpdmp diff --git a/tests/http/testenv/sshd.py b/tests/http/testenv/sshd.py new file mode 100644 index 0000000000..18e5e2cfd7 --- /dev/null +++ b/tests/http/testenv/sshd.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) Daniel Stenberg, , et al. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +# SPDX-License-Identifier: curl +# +########################################################################### +# +import logging +import os +import socket +import stat +import subprocess +import time +from datetime import timedelta, datetime + +from typing import Dict + +from . import CurlClient +from .env import Env +from .ports import alloc_ports_and_do + +log = logging.getLogger(__name__) + + +class Sshd: + + def __init__(self, env: Env): + self.env = env + self._cmd = Env.CONFIG.sshd + self._sftpd = Env.CONFIG.sftpd + self._keygen = 'ssh-keygen' + self._port = 0 + self.name = 'sshd' + self._port_skey = 'sshd' + self._port_specs = { + 'sshd': socket.SOCK_STREAM, + } + self._sshd_dir = os.path.join(env.gen_dir, self.name) + self._home_dir = os.path.join(self._sshd_dir, 'home') + self._run_dir = os.path.join(self._sshd_dir, 'run') + self._tmp_dir = os.path.join(self._sshd_dir, 'tmp') + self._conf_file = os.path.join(self._sshd_dir, 'test.conf') + self._auth_keys = os.path.join(self._sshd_dir, 'authorized_keys') + self._known_hosts = os.path.join(self._sshd_dir, 'known_hosts') + self._unknown_hosts = os.path.join(self._sshd_dir, 'unknown_hosts') + self._sshd_log = os.path.join(self._sshd_dir, 'sshd.log') + self._pid_file = os.path.join(self._sshd_dir, 'sshd.pid') + self._key_algs = [ + 'rsa', 'ecdsa', 'ed25519', + ] + self._host_key_files = [] + self._host_pub_files = [] + self._users = [ + 'user1', + 'user2', + ] + self._user_key_files = [] + self._user_pub_files = [] + self._process = None + + self.clear_logs() + self._mkpath(self._home_dir) + env.make_data_file(indir=self._home_dir, fname="data", fsize=1024) + self._mkpath(self._tmp_dir) + + @property + def port(self) -> int: + return self._port + + @property + def home_dir(self): + return self._home_dir + + @property + def known_hosts(self): + return self._known_hosts + + @property + def unknown_hosts(self): + return self._unknown_hosts + + @property + def user1_pubkey_file(self): + return self._user_pub_files[0] + + @property + def user1_privkey_file(self): + return self._user_key_files[0] + + @property + def user2_pubkey_file(self): + return self._user_pub_files[1] + + @property + def user2_privkey_file(self): + return self._user_key_files[1] + + def mk_host_keys(self): + self._host_key_files = [] + self._host_pub_files = [] + # a known_host file that knows all our test host pubkeys + with open(self._unknown_hosts, 'w') as fd_unknown, \ + open(self._known_hosts, 'w') as fd_known: + os.chmod(self._unknown_hosts, stat.S_IRUSR | stat.S_IWUSR) + os.chmod(self._known_hosts, stat.S_IRUSR | stat.S_IWUSR) + for alg in self._key_algs: + key_file = os.path.join(self._sshd_dir, f'ssh_host_{alg}_key') + if not os.path.exists(key_file): + p = subprocess.run(args=[ + self._keygen, '-q', '-N', '', '-t', alg, '-f', key_file + ], capture_output=True, text=True) + if p.returncode != 0: + raise RuntimeError(f'error generating host key {key_file}: {p.returncode}') + self._host_key_files.append(key_file) + pub_file = f'{key_file}.pub' + self._host_pub_files.append(pub_file) + pubkey = open(pub_file).read() + # fd_known.write(f'[127.0.0.1]:{self.port} {pubkey}') + fd_known.write(f'[{self.env.domain1.lower()}]:{self.port} {pubkey}') + fd_unknown.write(f'dummy.invalid {pubkey}') + # hash the known_hosts file, libssh requires it + p = subprocess.run(args=[ + self._keygen, '-H', '-f', self._known_hosts + ], capture_output=True, text=True) + if p.returncode != 0: + raise RuntimeError(f'error hashing {self._known_hosts}: {p.returncode}') + + def mk_user_keys(self): + self._user_key_files = [] + self._user_pub_files = [] + alg = 'rsa' + for user in self._users: + key_file = os.path.join(self._sshd_dir, f'id_{user}_user_{alg}_key') + if not os.path.exists(key_file): + p = subprocess.run(args=[ + self._keygen, '-q', '-N', '', '-t', alg, '-f', key_file + ], capture_output=True, text=True) + if p.returncode != 0: + raise RuntimeError(f'error generating user key {key_file}: {p.returncode}') + self._user_key_files.append(key_file) + self._user_pub_files.append(f'{key_file}.pub') + with open(self._auth_keys, 'w') as fd: + os.chmod(self._auth_keys, stat.S_IRUSR | stat.S_IWUSR) + pubkey = open(self._user_pub_files[0]).read() + fd.write(pubkey) + + def clear_logs(self): + self._rmf(self._sshd_log) + + def dump_log(self): + lines = ['>>--sshd log ----------------------------------------------\n'] + lines.extend(open(self._sshd_log)) + lines.extend(['>>--curl log ----------------------------------------------\n']) + lines.extend(open(os.path.join(self._tmp_dir, 'curl.stderr'))) + lines.append('<<-------------------------------------------------------\n') + return ''.join(lines) + + def exists(self): + return os.path.exists(self._cmd) + + def is_running(self): + if self._process: + self._process.poll() + return self._process.returncode is None + return False + + def start_if_needed(self): + if not self.is_running(): + return self.start() + return True + + def stop(self, wait_dead=True): + self._mkpath(self._tmp_dir) + if self._process: + self._process.terminate() + self._process.wait(timeout=2) + self._process = None + return not wait_dead or True + return True + + def restart(self): + self.stop() + return self.start() + + def initial_start(self): + + def startup(ports: Dict[str, int]) -> bool: + self._port = ports[self._port_skey] + self.mk_host_keys() + self.mk_user_keys() + if self.start(): + self.env.update_ports(ports) + return True + self.stop() + self._port = 0 + return False + + return alloc_ports_and_do(self._port_specs, startup, + self.env.gen_root, max_tries=3) + + def start(self, wait_live=True): + assert self._port > 0 + self._mkpath(self._tmp_dir) + if self._process: + self.stop() + self._write_config() + args = [ + self._cmd, + '-D', + '-f', f'{self._conf_file}', + '-E', f'{self._sshd_log}', + ] + run_env = os.environ.copy() + # does not have any effect, sadly + # run_env['HOME'] = f'{self._home_dir}' + procerr = open(self._sshd_log, 'a') + self._process = subprocess.Popen(args=args, stderr=procerr, env=run_env) + if self._process.returncode is not None: + return False + return self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT)) + + def wait_live(self, timeout: timedelta): + curl = CurlClient(env=self.env, run_dir=self._tmp_dir, + timeout=timeout.total_seconds()) + try_until = datetime.now() + timeout + while datetime.now() < try_until: + r = curl.http_get(url=f'scp://{self.env.domain1}:{self._port}/{self.home_dir}/data', + extra_args=[ + '--insecure', + '--pubkey', self.user1_pubkey_file, + '--key', self.user1_privkey_file, + '--user', f'{os.environ["USER"]}:', + ]) + if r.exit_code == 0: + return True + time.sleep(.1) + log.error(f"sshd still not responding after {timeout}") + return False + + def _rmf(self, path): + if os.path.exists(path): + return os.remove(path) + + def _mkpath(self, path): + if not os.path.exists(path): + return os.makedirs(path) + + def _write_config(self): + conf = [ + f'ListenAddress 127.0.0.1:{self._port}', + 'AllowTcpForwarding yes', + 'AuthenticationMethods publickey', + 'PasswordAuthentication no', + # in CI, we might run as root, allow this + 'PermitRootLogin yes', + 'LogLevel VERBOSE', + f'AuthorizedKeysFile {self._auth_keys}', + f'PidFile {self._pid_file}', + ] + conf.extend([f'HostKey {key_file}' for key_file in self._host_key_files]) + if self._sftpd: + conf.append(f'Subsystem sftp {self._sftpd}') + conf.append('\n') + with open(self._conf_file, 'w') as fd: + fd.write("\n".join(conf))