fi
AC_SUBST(DANTED)
+dnl we would like a sshd as test server
+dnl
+SSHD_ENABLED="maybe"
+AC_ARG_WITH(test-sshd, [AS_HELP_STRING([--with-test-sshd=PATH],
+ [where to find sshd for testing])],
+ [request_sshd=$withval], [request_sshd=check])
+if test "x$request_sshd" = "xcheck" || test "x$request_sshd" = "xyes"; then
+ if test -x "/usr/sbin/sshd"; then
+ # common location on distros (debian/ubuntu)
+ SSHD="/usr/sbin/sshd"
+ else
+ AC_PATH_PROG([SSHD], [sshd])
+ if test -z "$SSHD"; then
+ AC_PATH_PROG([SSHD], [sshd])
+ fi
+ fi
+elif test "x$request_sshd" != "xno"; then
+ SSHD="${request_sshd}"
+ if test ! -x "${SSHD}"; then
+ AC_MSG_NOTICE([sshd not found as ${SSHD}, sshd tests disabled])
+ SSHD_ENABLED="no"
+ else
+ AC_MSG_NOTICE([using SSHD=$SSHD for tests])
+ fi
+fi
+if test "$SSHD_ENABLED" = "no"; then
+ SSHD=""
+ SFTPD=""
+else
+ if test -x "/usr/libexec/sftp-server"; then
+ # common location on macOS)
+ SFTPD="/usr/libexec/sftp-server"
+ elif test -x "/usr/lib/openssh/sftp-server"; then
+ # common location on debian
+ SFTPD="/usr/lib/openssh/sftp-server"
+ else
+ AC_PATH_PROG([SFTPD], [sftp-server])
+ if test -z "$SFTPD"; then
+ AC_PATH_PROG([SFTPD], [sftp-server])
+ fi
+ fi
+fi
+AC_SUBST(SSHD)
+AC_SUBST(SFTPD)
+
dnl the nghttpx we might use in httpd testing
if test -n "$TEST_NGHTTPX" && test "x$TEST_NGHTTPX" != "xnghttpx"; then
HTTPD_NGHTTPX="$TEST_NGHTTPX"
- `DANTED`: Default: `danted`
- `TEST_NGHTTPX`: Default: `nghttpx`
- `VSFTPD`: Default: `vsftps`
+- `SSHD`: Default: `sshd`
+- `SFTPD`: Default: `sftp-server`
## Feature detection variables
endif()
mark_as_advanced(DANTED)
-# Consumed variables: APXS, CADDY, HTTPD, HTTPD_NGHTTPX, DANTED, VSFTPD
+find_program(SSHD "sshd" PATHS "/usr/sbin" "/usr/bin")
+if(NOT SSHD)
+ set(SSHD "")
+endif()
+mark_as_advanced(SSHD)
+
+find_program(SFTPD "sftp-server" PATHS "/usr/libexec" "/usr/lib/openssh")
+if(NOT SFTPD)
+ set(SFTPD "")
+endif()
+mark_as_advanced(SFTPD)
+
+# Consumed variables: APXS, CADDY, HTTPD, HTTPD_NGHTTPX, DANTED, VSFTPD, SSHD
configure_file("config.ini.in" "${CMAKE_CURRENT_BINARY_DIR}/config.ini" @ONLY)
testenv/mod_curltest/mod_curltest.c \
testenv/nghttpx.py \
testenv/ports.py \
+testenv/sshd.py \
testenv/vsftpd.py \
testenv/ws_echo_server.py
test_30_vsftpd.py \
test_31_vsftpds.py \
test_32_ftps_vsftpd.py \
-test_40_socks.py \
+test_40_socks.py \
+test_50_scp.py \
+test_51_sftp.py \
$(TESTENV)
clean-local:
[danted]
danted = @DANTED@
+
+[sshd]
+sshd = @SSHD@
+sftpd = @SFTPD@
sys.path.append(os.path.join(os.path.dirname(__file__), '.'))
-from testenv import Env, Nghttpx, Httpd, NghttpxQuic, NghttpxFwd
+from testenv import Env, Nghttpx, Httpd, NghttpxQuic, NghttpxFwd, Sshd
log = logging.getLogger(__name__)
yield False
+@pytest.fixture(scope='session')
+def sshd(env: Env) -> Generator[Union[Sshd,bool], None, None]:
+ if env.has_sshd():
+ sshd = Sshd(env=env)
+ assert sshd.initial_start(), f'{sshd.dump_log()}'
+ yield sshd
+ sshd.stop()
+ else:
+ yield False
+
+
@pytest.fixture(scope='session')
def configures_httpd(env, httpd) -> Generator[bool, None, None]:
# include this fixture as test parameter if the test configures httpd itself
--- /dev/null
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#***************************************************************************
+# _ _ ____ _
+# Project ___| | | | _ \| |
+# / __| | | | |_) | |
+# | (__| |_| | _ <| |___
+# \___|\___/|_| \_\_____|
+#
+# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at https://curl.se/docs/copyright.html.
+#
+# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+# copies of the Software, and permit persons to whom the Software is
+# furnished to do so, under the terms of the COPYING file.
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+# KIND, either express or implied.
+#
+# SPDX-License-Identifier: curl
+#
+###########################################################################
+#
+import difflib
+import filecmp
+import logging
+import os
+import pytest
+
+from testenv import Env, CurlClient, Sshd
+
+
+log = logging.getLogger(__name__)
+
+
+@pytest.mark.skipif(condition=not Env.curl_has_protocol('scp'), reason="curl built without scp:")
+@pytest.mark.skipif(condition=not Env.has_sshd(), reason="missing sshd")
+class TestScp:
+
+ @pytest.fixture(autouse=True, scope='class')
+ def _class_scope(self, env, sshd):
+ env.make_data_file(indir=sshd.home_dir, fname="data-10k", fsize=10*1024)
+ env.make_data_file(indir=sshd.home_dir, fname="data-10m", fsize=10*1024*1024)
+ env.make_data_file(indir=env.gen_dir, fname="data-10k", fsize=10*1024)
+ env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10*1024*1024)
+
+ def test_50_01_insecure(self, env: Env, sshd: Sshd):
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data')
+ url = f'scp://{env.domain1}:{sshd.port}/{doc_file}'
+ r = curl.ssh_download(urls=[url], extra_args=[
+ '--insecure',
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+
+ def test_50_02_unknown_hosts(self, env: Env, sshd: Sshd):
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data')
+ url = f'scp://{env.domain1}:{sshd.port}/{doc_file}'
+ r = curl.ssh_download(urls=[url], extra_args=[
+ '--knownhosts', sshd.unknown_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(60) # CURLE_PEER_FAILED_VERIFICATION
+
+ def test_50_03_known_hosts(self, env: Env, sshd: Sshd):
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data')
+ url = f'scp://{env.domain1}:{sshd.port}/{doc_file}'
+ r = curl.ssh_download(urls=[url], extra_args=[
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+
+ # use key not in authorized_keys file
+ def test_50_04_unauth_user(self, env: Env, sshd: Sshd):
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data')
+ url = f'scp://{env.domain1}:{sshd.port}/{doc_file}'
+ r = curl.ssh_download(urls=[url], extra_args=[
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user2_pubkey_file,
+ '--key', sshd.user2_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(67) # CURLE_LOGIN_DENIED
+
+ def test_50_10_dl_single(self, env: Env, sshd: Sshd):
+ count = 1
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data-10k')
+ url = f'scp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
+ r = curl.ssh_download(urls=[url], extra_args=[
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+ self.check_downloads(curl, doc_file, count)
+
+ def test_50_11_dl_serial(self, env: Env, sshd: Sshd):
+ count = 5
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data-10k')
+ url = f'scp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
+ r = curl.ssh_download(urls=[url], extra_args=[
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+ self.check_downloads(curl, doc_file, count)
+
+ def test_50_12_dl_parallel(self, env: Env, sshd: Sshd):
+ count = 5
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data-10k')
+ url = f'scp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
+ r = curl.http_download(urls=[url], extra_args=[
+ '--parallel',
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+ self.check_downloads(curl, doc_file, count)
+
+ def test_50_20_ul_single(self, env: Env, sshd: Sshd):
+ srcfile = os.path.join(env.gen_dir, 'data-10k')
+ destfile = os.path.join(sshd.home_dir, 'upload_20.data')
+ curl = CurlClient(env=env)
+ url = f'scp://{env.domain1}:{sshd.port}/{destfile}'
+ r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+ self.check_upload(sshd, srcfile, destfile)
+
+ def test_50_21_ul_serial(self, env: Env, sshd: Sshd):
+ count = 5
+ srcfile = os.path.join(env.gen_dir, 'data-10k')
+ destfile = os.path.join(sshd.home_dir, 'upload_21.data')
+ curl = CurlClient(env=env)
+ url = f'scp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]'
+ r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+ for i in range(count):
+ self.check_upload(sshd, srcfile, f'{destfile}.{i}')
+
+ def test_50_22_ul_parallel(self, env: Env, sshd: Sshd):
+ count = 5
+ srcfile = os.path.join(env.gen_dir, 'data-10k')
+ destfile = os.path.join(sshd.home_dir, 'upload_22.data')
+ curl = CurlClient(env=env)
+ url = f'scp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]'
+ r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
+ '--parallel',
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+ for i in range(count):
+ self.check_upload(sshd, srcfile, f'{destfile}.{i}')
+
+ def check_downloads(self, client, srcfile: str, count: int,
+ complete: bool = True):
+ for i in range(count):
+ dfile = client.download_file(i)
+ assert os.path.exists(dfile)
+ if complete and not filecmp.cmp(srcfile, dfile, shallow=False):
+ diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(),
+ b=open(dfile).readlines(),
+ fromfile=srcfile,
+ tofile=dfile,
+ n=1))
+ assert False, f'download {dfile} differs:\n{diff}'
+
+ def check_upload(self, sshd: Sshd, srcfile, destfile, binary=True):
+ assert os.path.exists(srcfile)
+ assert os.path.exists(destfile)
+ if not filecmp.cmp(srcfile, destfile, shallow=False):
+ diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(),
+ b=open(destfile).readlines(),
+ fromfile=srcfile,
+ tofile=destfile,
+ n=1))
+ assert not binary and len(diff) == 0, f'upload {destfile} differs:\n{diff}'
--- /dev/null
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#***************************************************************************
+# _ _ ____ _
+# Project ___| | | | _ \| |
+# / __| | | | |_) | |
+# | (__| |_| | _ <| |___
+# \___|\___/|_| \_\_____|
+#
+# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at https://curl.se/docs/copyright.html.
+#
+# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+# copies of the Software, and permit persons to whom the Software is
+# furnished to do so, under the terms of the COPYING file.
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+# KIND, either express or implied.
+#
+# SPDX-License-Identifier: curl
+#
+###########################################################################
+#
+import difflib
+import filecmp
+import logging
+import os
+import pytest
+
+from testenv import Env, CurlClient, Sshd
+
+
+log = logging.getLogger(__name__)
+
+
+@pytest.mark.skipif(condition=not Env.curl_has_protocol('sftp'), reason="curl built without sfto:")
+@pytest.mark.skipif(condition=not Env.has_sftpd(), reason="missing sftp server")
+class TestSftp:
+
+ @pytest.fixture(autouse=True, scope='class')
+ def _class_scope(self, env, sshd):
+ env.make_data_file(indir=sshd.home_dir, fname="data-10k", fsize=10*1024)
+ env.make_data_file(indir=sshd.home_dir, fname="data-10m", fsize=10*1024*1024)
+ env.make_data_file(indir=env.gen_dir, fname="data-10k", fsize=10*1024)
+ env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10*1024*1024)
+
+ def test_51_01_insecure(self, env: Env, sshd: Sshd):
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data')
+ url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}'
+ r = curl.ssh_download(urls=[url], extra_args=[
+ '--insecure',
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+
+ def test_51_02_unknown_hosts(self, env: Env, sshd: Sshd):
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data')
+ url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}'
+ r = curl.ssh_download(urls=[url], extra_args=[
+ '--knownhosts', sshd.unknown_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(60) # CURLE_PEER_FAILED_VERIFICATION
+
+ def test_51_03_known_hosts(self, env: Env, sshd: Sshd):
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data')
+ url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}'
+ r = curl.ssh_download(urls=[url], extra_args=[
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+
+ # use key not in authorized_keys file
+ def test_51_04_unauth_user(self, env: Env, sshd: Sshd):
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data')
+ url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}'
+ r = curl.ssh_download(urls=[url], extra_args=[
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user2_pubkey_file,
+ '--key', sshd.user2_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(67) # CURLE_LOGIN_DENIED
+
+ def test_51_10_dl_single(self, env: Env, sshd: Sshd):
+ count = 1
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data-10k')
+ url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
+ r = curl.ssh_download(urls=[url], extra_args=[
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+ self.check_downloads(curl, doc_file, count)
+
+ def test_51_11_dl_serial(self, env: Env, sshd: Sshd):
+ count = 5
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data-10k')
+ url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
+ r = curl.ssh_download(urls=[url], extra_args=[
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+ self.check_downloads(curl, doc_file, count)
+
+ def test_51_12_dl_parallel(self, env: Env, sshd: Sshd):
+ count = 5
+ curl = CurlClient(env=env)
+ doc_file = os.path.join(sshd.home_dir, 'data-10k')
+ url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
+ r = curl.http_download(urls=[url], extra_args=[
+ '--parallel',
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+ self.check_downloads(curl, doc_file, count)
+
+ def test_51_20_ul_single(self, env: Env, sshd: Sshd):
+ srcfile = os.path.join(env.gen_dir, 'data-10k')
+ destfile = os.path.join(sshd.home_dir, 'upload_20.data')
+ curl = CurlClient(env=env)
+ url = f'sftp://{env.domain1}:{sshd.port}/{destfile}'
+ r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+ self.check_upload(sshd, srcfile, destfile)
+
+ def test_51_21_ul_serial(self, env: Env, sshd: Sshd):
+ count = 5
+ srcfile = os.path.join(env.gen_dir, 'data-10k')
+ destfile = os.path.join(sshd.home_dir, 'upload_21.data')
+ curl = CurlClient(env=env)
+ url = f'sftp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]'
+ r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+ for i in range(count):
+ self.check_upload(sshd, srcfile, f'{destfile}.{i}')
+
+ def test_51_22_ul_parallel(self, env: Env, sshd: Sshd):
+ count = 5
+ srcfile = os.path.join(env.gen_dir, 'data-10k')
+ destfile = os.path.join(sshd.home_dir, 'upload_22.data')
+ curl = CurlClient(env=env)
+ url = f'sftp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]'
+ r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
+ '--parallel',
+ '--knownhosts', sshd.known_hosts,
+ '--pubkey', sshd.user1_pubkey_file,
+ '--key', sshd.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ r.check_exit_code(0)
+ for i in range(count):
+ self.check_upload(sshd, srcfile, f'{destfile}.{i}')
+
+ def check_downloads(self, client, srcfile: str, count: int,
+ complete: bool = True):
+ for i in range(count):
+ dfile = client.download_file(i)
+ assert os.path.exists(dfile)
+ if complete and not filecmp.cmp(srcfile, dfile, shallow=False):
+ diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(),
+ b=open(dfile).readlines(),
+ fromfile=srcfile,
+ tofile=dfile,
+ n=1))
+ assert False, f'download {dfile} differs:\n{diff}'
+
+ def check_upload(self, sshd: Sshd, srcfile, destfile, binary=True):
+ assert os.path.exists(srcfile)
+ assert os.path.exists(destfile)
+ if not filecmp.cmp(srcfile, destfile, shallow=False):
+ diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(),
+ b=open(destfile).readlines(),
+ fromfile=srcfile,
+ tofile=destfile,
+ n=1))
+ assert not binary and len(diff) == 0, f'upload {destfile} differs:\n{diff}'
from .nghttpx import Nghttpx, NghttpxQuic, NghttpxFwd
from .vsftpd import VsFTPD
from .dante import Dante
+from .sshd import Sshd
with_tcpdump=with_tcpdump,
extra_args=extra_args)
+ def ssh_download(self, urls: List[str],
+ with_stats: bool = True,
+ with_profile: bool = False,
+ with_tcpdump: bool = False,
+ no_save: bool = False,
+ extra_args: Optional[List[str]] = None):
+ if extra_args is None:
+ extra_args = []
+ if no_save:
+ extra_args.extend([
+ '--out-null',
+ ])
+ else:
+ extra_args.extend([
+ '-o', 'download_#1.data',
+ ])
+ # remove any existing ones
+ for i in range(100):
+ self._rmf(self.download_file(i))
+ if with_stats:
+ extra_args.extend([
+ '-w', '%{json}\\n'
+ ])
+ return self._raw(urls, options=extra_args,
+ with_stats=with_stats,
+ with_headers=False,
+ with_profile=with_profile,
+ with_tcpdump=with_tcpdump)
+
+ def ssh_upload(self, urls: List[str],
+ fupload: Optional[Any] = None,
+ updata: Optional[str] = None,
+ with_stats: bool = True,
+ with_profile: bool = False,
+ with_tcpdump: bool = False,
+ extra_args: Optional[List[str]] = None):
+ if extra_args is None:
+ extra_args = []
+ if fupload is not None:
+ extra_args.extend([
+ '--upload-file', fupload
+ ])
+ elif updata is not None:
+ extra_args.extend([
+ '--upload-file', '-'
+ ])
+ else:
+ raise Exception('need either file or data to upload')
+ if with_stats:
+ extra_args.extend([
+ '-w', '%{json}\\n'
+ ])
+ return self._raw(urls, options=extra_args,
+ intext=updata,
+ with_stats=with_stats,
+ with_headers=False,
+ with_profile=with_profile,
+ with_tcpdump=with_tcpdump)
+
def response_file(self, idx: int):
return os.path.join(self._run_dir, f'download_{idx}.data')
except Exception:
self.danted = None
+ self.sshd = self.config['sshd']['sshd']
+ if self.sshd == '':
+ self.sshd = None
+ self._sshd_version = None
+ if self.sshd is not None:
+ try:
+ p = subprocess.run(args=[self.sshd, '-V'],
+ capture_output=True, text=True)
+ assert p.returncode == 0
+ if p.returncode != 0:
+ self.sshd = None
+ else:
+ m = re.match(r'^OpenSSH_(\d+\.\d+.*),.*', p.stderr)
+ assert m, f'version: {p.stderr}'
+ if m:
+ self._sshd_version = m.group(1)
+ else:
+ self.sshd = None
+ raise Exception(f'Unable to determine sshd version from: {p.stderr}')
+ except Exception:
+ self.sshd = None
+
+ if self.sshd:
+ self.sftpd = self.config['sshd']['sftpd']
+ if self.sftpd == '':
+ self.sftpd = None
+ else:
+ self.sftpd = None
+
self._tcpdump = shutil.which('tcpdump')
@property
def has_danted() -> bool:
return Env.CONFIG.danted is not None
+ @staticmethod
+ def has_sshd() -> bool:
+ return Env.CONFIG.sshd is not None
+
+ @staticmethod
+ def has_sftpd() -> bool:
+ return Env.has_sshd() and Env.CONFIG.sftpd is not None
+
@staticmethod
def tcpdump() -> Optional[str]:
return Env.CONFIG.tcpdmp
--- /dev/null
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#***************************************************************************
+# _ _ ____ _
+# Project ___| | | | _ \| |
+# / __| | | | |_) | |
+# | (__| |_| | _ <| |___
+# \___|\___/|_| \_\_____|
+#
+# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at https://curl.se/docs/copyright.html.
+#
+# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+# copies of the Software, and permit persons to whom the Software is
+# furnished to do so, under the terms of the COPYING file.
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+# KIND, either express or implied.
+#
+# SPDX-License-Identifier: curl
+#
+###########################################################################
+#
+import logging
+import os
+import socket
+import stat
+import subprocess
+import time
+from datetime import timedelta, datetime
+
+from typing import Dict
+
+from . import CurlClient
+from .env import Env
+from .ports import alloc_ports_and_do
+
+log = logging.getLogger(__name__)
+
+
+class Sshd:
+
+ def __init__(self, env: Env):
+ self.env = env
+ self._cmd = Env.CONFIG.sshd
+ self._sftpd = Env.CONFIG.sftpd
+ self._keygen = 'ssh-keygen'
+ self._port = 0
+ self.name = 'sshd'
+ self._port_skey = 'sshd'
+ self._port_specs = {
+ 'sshd': socket.SOCK_STREAM,
+ }
+ self._sshd_dir = os.path.join(env.gen_dir, self.name)
+ self._home_dir = os.path.join(self._sshd_dir, 'home')
+ self._run_dir = os.path.join(self._sshd_dir, 'run')
+ self._tmp_dir = os.path.join(self._sshd_dir, 'tmp')
+ self._conf_file = os.path.join(self._sshd_dir, 'test.conf')
+ self._auth_keys = os.path.join(self._sshd_dir, 'authorized_keys')
+ self._known_hosts = os.path.join(self._sshd_dir, 'known_hosts')
+ self._unknown_hosts = os.path.join(self._sshd_dir, 'unknown_hosts')
+ self._sshd_log = os.path.join(self._sshd_dir, 'sshd.log')
+ self._pid_file = os.path.join(self._sshd_dir, 'sshd.pid')
+ self._key_algs = [
+ 'rsa', 'ecdsa', 'ed25519',
+ ]
+ self._host_key_files = []
+ self._host_pub_files = []
+ self._users = [
+ 'user1',
+ 'user2',
+ ]
+ self._user_key_files = []
+ self._user_pub_files = []
+ self._process = None
+
+ self.clear_logs()
+ self._mkpath(self._home_dir)
+ env.make_data_file(indir=self._home_dir, fname="data", fsize=1024)
+ self._mkpath(self._tmp_dir)
+
+ @property
+ def port(self) -> int:
+ return self._port
+
+ @property
+ def home_dir(self):
+ return self._home_dir
+
+ @property
+ def known_hosts(self):
+ return self._known_hosts
+
+ @property
+ def unknown_hosts(self):
+ return self._unknown_hosts
+
+ @property
+ def user1_pubkey_file(self):
+ return self._user_pub_files[0]
+
+ @property
+ def user1_privkey_file(self):
+ return self._user_key_files[0]
+
+ @property
+ def user2_pubkey_file(self):
+ return self._user_pub_files[1]
+
+ @property
+ def user2_privkey_file(self):
+ return self._user_key_files[1]
+
+ def mk_host_keys(self):
+ self._host_key_files = []
+ self._host_pub_files = []
+ # a known_host file that knows all our test host pubkeys
+ with open(self._unknown_hosts, 'w') as fd_unknown, \
+ open(self._known_hosts, 'w') as fd_known:
+ os.chmod(self._unknown_hosts, stat.S_IRUSR | stat.S_IWUSR)
+ os.chmod(self._known_hosts, stat.S_IRUSR | stat.S_IWUSR)
+ for alg in self._key_algs:
+ key_file = os.path.join(self._sshd_dir, f'ssh_host_{alg}_key')
+ if not os.path.exists(key_file):
+ p = subprocess.run(args=[
+ self._keygen, '-q', '-N', '', '-t', alg, '-f', key_file
+ ], capture_output=True, text=True)
+ if p.returncode != 0:
+ raise RuntimeError(f'error generating host key {key_file}: {p.returncode}')
+ self._host_key_files.append(key_file)
+ pub_file = f'{key_file}.pub'
+ self._host_pub_files.append(pub_file)
+ pubkey = open(pub_file).read()
+ # fd_known.write(f'[127.0.0.1]:{self.port} {pubkey}')
+ fd_known.write(f'[{self.env.domain1.lower()}]:{self.port} {pubkey}')
+ fd_unknown.write(f'dummy.invalid {pubkey}')
+ # hash the known_hosts file, libssh requires it
+ p = subprocess.run(args=[
+ self._keygen, '-H', '-f', self._known_hosts
+ ], capture_output=True, text=True)
+ if p.returncode != 0:
+ raise RuntimeError(f'error hashing {self._known_hosts}: {p.returncode}')
+
+ def mk_user_keys(self):
+ self._user_key_files = []
+ self._user_pub_files = []
+ alg = 'rsa'
+ for user in self._users:
+ key_file = os.path.join(self._sshd_dir, f'id_{user}_user_{alg}_key')
+ if not os.path.exists(key_file):
+ p = subprocess.run(args=[
+ self._keygen, '-q', '-N', '', '-t', alg, '-f', key_file
+ ], capture_output=True, text=True)
+ if p.returncode != 0:
+ raise RuntimeError(f'error generating user key {key_file}: {p.returncode}')
+ self._user_key_files.append(key_file)
+ self._user_pub_files.append(f'{key_file}.pub')
+ with open(self._auth_keys, 'w') as fd:
+ os.chmod(self._auth_keys, stat.S_IRUSR | stat.S_IWUSR)
+ pubkey = open(self._user_pub_files[0]).read()
+ fd.write(pubkey)
+
+ def clear_logs(self):
+ self._rmf(self._sshd_log)
+
+ def dump_log(self):
+ lines = ['>>--sshd log ----------------------------------------------\n']
+ lines.extend(open(self._sshd_log))
+ lines.extend(['>>--curl log ----------------------------------------------\n'])
+ lines.extend(open(os.path.join(self._tmp_dir, 'curl.stderr')))
+ lines.append('<<-------------------------------------------------------\n')
+ return ''.join(lines)
+
+ def exists(self):
+ return os.path.exists(self._cmd)
+
+ def is_running(self):
+ if self._process:
+ self._process.poll()
+ return self._process.returncode is None
+ return False
+
+ def start_if_needed(self):
+ if not self.is_running():
+ return self.start()
+ return True
+
+ def stop(self, wait_dead=True):
+ self._mkpath(self._tmp_dir)
+ if self._process:
+ self._process.terminate()
+ self._process.wait(timeout=2)
+ self._process = None
+ return not wait_dead or True
+ return True
+
+ def restart(self):
+ self.stop()
+ return self.start()
+
+ def initial_start(self):
+
+ def startup(ports: Dict[str, int]) -> bool:
+ self._port = ports[self._port_skey]
+ self.mk_host_keys()
+ self.mk_user_keys()
+ if self.start():
+ self.env.update_ports(ports)
+ return True
+ self.stop()
+ self._port = 0
+ return False
+
+ return alloc_ports_and_do(self._port_specs, startup,
+ self.env.gen_root, max_tries=3)
+
+ def start(self, wait_live=True):
+ assert self._port > 0
+ self._mkpath(self._tmp_dir)
+ if self._process:
+ self.stop()
+ self._write_config()
+ args = [
+ self._cmd,
+ '-D',
+ '-f', f'{self._conf_file}',
+ '-E', f'{self._sshd_log}',
+ ]
+ run_env = os.environ.copy()
+ # does not have any effect, sadly
+ # run_env['HOME'] = f'{self._home_dir}'
+ procerr = open(self._sshd_log, 'a')
+ self._process = subprocess.Popen(args=args, stderr=procerr, env=run_env)
+ if self._process.returncode is not None:
+ return False
+ return self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
+
+ def wait_live(self, timeout: timedelta):
+ curl = CurlClient(env=self.env, run_dir=self._tmp_dir,
+ timeout=timeout.total_seconds())
+ try_until = datetime.now() + timeout
+ while datetime.now() < try_until:
+ r = curl.http_get(url=f'scp://{self.env.domain1}:{self._port}/{self.home_dir}/data',
+ extra_args=[
+ '--insecure',
+ '--pubkey', self.user1_pubkey_file,
+ '--key', self.user1_privkey_file,
+ '--user', f'{os.environ["USER"]}:',
+ ])
+ if r.exit_code == 0:
+ return True
+ time.sleep(.1)
+ log.error(f"sshd still not responding after {timeout}")
+ return False
+
+ def _rmf(self, path):
+ if os.path.exists(path):
+ return os.remove(path)
+
+ def _mkpath(self, path):
+ if not os.path.exists(path):
+ return os.makedirs(path)
+
+ def _write_config(self):
+ conf = [
+ f'ListenAddress 127.0.0.1:{self._port}',
+ 'AllowTcpForwarding yes',
+ 'AuthenticationMethods publickey',
+ 'PasswordAuthentication no',
+ # in CI, we might run as root, allow this
+ 'PermitRootLogin yes',
+ 'LogLevel VERBOSE',
+ f'AuthorizedKeysFile {self._auth_keys}',
+ f'PidFile {self._pid_file}',
+ ]
+ conf.extend([f'HostKey {key_file}' for key_file in self._host_key_files])
+ if self._sftpd:
+ conf.append(f'Subsystem sftp {self._sftpd}')
+ conf.append('\n')
+ with open(self._conf_file, 'w') as fd:
+ fd.write("\n".join(conf))
projects / thirdparty / curl.git / commitdiff
