From: Kees Monshouwer Date: Mon, 1 Feb 2021 12:57:04 +0000 (+0100) Subject: auth: add gmysql, gpgsql and lmdb (partial) backend to the api tests X-Git-Tag: dnsdist-1.6.0-alpha2~23^2~11 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=0bbd0a64dc496720155c575c5ff438a92b05e76c;p=thirdparty%2Fpdns.git auth: add gmysql, gpgsql and lmdb (partial) backend to the api tests --- diff --git a/.circleci/config.yml b/.circleci/config.yml index 8f275fb8c3..8c6bc2ef3f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -208,7 +208,7 @@ commands: libkrb5-3 \ libldap-2.4-2 \ liblmdb0 \ - libluajit-5.1-2 \ + libluajit-5.1 \ libmaxminddb0 \ libp11-kit0 \ libpq5 \ @@ -791,7 +791,7 @@ jobs: - run: name: Set up mysql client lib to force TCP command: echo 'export GMYSQLHOST=127.0.0.1' > ./vars - workdir: ~/project/regression-tests + workdir: ~/project/regression-tests - auth-regress: context: bind-hybrid-nsec3 @@ -1062,7 +1062,7 @@ jobs: name: Test all algorithms command: /opt/pdns-auth/bin/pdnsutil test-algorithms - test-auth-api: + test-auth-api-gmysql: resource_class: small docker: @@ -1073,18 +1073,85 @@ jobs: environment: UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' ASAN_OPTIONS: detect_leaks=0 + - image: circleci/mysql:5 + auth: + username: powerdnsreadonly + password: $DOCKERHUB_PASSWORD steps: - auth-regress-setup - - run: apt-get -y -qq install python3-venv + - run: apt-get -y -qq install python3-venv python3-dev default-libmysqlclient-dev libpq-dev default-mysql-client - run: - name: Run API tests + name: Set up mysql client lib to force TCP + command: | + cat >> ~/.my.cnf \<<- __EOF__ + [client] + protocol=TCP + default-character-set=latin1 + __EOF__ + - run: + name: Run API tests gmysql workdir: ~/project/regression-tests.api command: | + MYSQL_HOST="127.0.0.1" \ PDNSSERVER="/opt/pdns-auth/sbin/pdns_server" \ PDNSUTIL="/opt/pdns-auth/bin/pdnsutil" \ SDIG="/opt/pdns-auth/bin/sdig" \ - ZONE2SQL="/opt/pdns-auth/bin/zone2sql" \ - ./runtests authoritative + ./runtests authoritative gmysql + + test-auth-api-gpgsql: + resource_class: small + + docker: + - image: debian:buster + auth: + username: powerdnsreadonly + password: $DOCKERHUB_PASSWORD + environment: + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' + ASAN_OPTIONS: detect_leaks=0 + - image: circleci/postgres:9 + auth: + username: powerdnsreadonly + password: $DOCKERHUB_PASSWORD + environment: + POSTGRES_USER: root + POSTGRES_HOST_AUTH_METHOD: trust + steps: + - auth-regress-setup + - run: apt-get -y -qq install python3-venv python3-dev default-libmysqlclient-dev libpq-dev postgresql-client + - run: + name: Run API tests gpgsql + workdir: ~/project/regression-tests.api + command: | + PGHOST="127.0.0.1" \ + PGPORT="5432" \ + PDNSSERVER="/opt/pdns-auth/sbin/pdns_server" \ + PDNSUTIL="/opt/pdns-auth/bin/pdnsutil" \ + SDIG="/opt/pdns-auth/bin/sdig" \ + ./runtests authoritative gpgsql + + test-auth-api-gsqlite3: + resource_class: small + + docker: + - image: debian:buster + auth: + username: powerdnsreadonly + password: $DOCKERHUB_PASSWORD + environment: + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' + ASAN_OPTIONS: detect_leaks=0 + steps: + - auth-regress-setup + - run: apt-get -y -qq install python3-venv python3-dev default-libmysqlclient-dev libpq-dev + - run: + name: Run API tests gsqlite3 + workdir: ~/project/regression-tests.api + command: | + PDNSSERVER="/opt/pdns-auth/sbin/pdns_server" \ + PDNSUTIL="/opt/pdns-auth/bin/pdnsutil" \ + SDIG="/opt/pdns-auth/bin/sdig" \ + ./runtests authoritative gsqlite3 build-recursor: docker: @@ -1431,7 +1498,7 @@ jobs: ASAN_OPTIONS: detect_leaks=0 steps: - add-auth-repo - - run: apt-get --no-install-recommends install -qq -y python3-venv pdns-tools + - run: apt-get --no-install-recommends install -qq -y python3-venv python3-dev default-libmysqlclient-dev libpq-dev pdns-tools - install-recursor-deps - get-workspace - run: @@ -1887,7 +1954,13 @@ workflows: - build-auth: requires: - checkout - - test-auth-api: + - test-auth-api-gmysql: + requires: + - build-auth + - test-auth-api-gpgsql: + requires: + - build-auth + - test-auth-api-gsqlite3: requires: - build-auth - test-auth-algorithms: diff --git a/regression-tests.api/.gitignore b/regression-tests.api/.gitignore index 11b8420d1c..d443eca5d1 100644 --- a/regression-tests.api/.gitignore +++ b/regression-tests.api/.gitignore @@ -2,6 +2,7 @@ /*.xml /*.pid /pdns.sqlite3 +/pdns.lmdb* /*.controlsocket /named.conf /pdns.conf diff --git a/regression-tests.api/requirements.txt b/regression-tests.api/requirements.txt index ca23d7f08a..b97f1065b1 100644 --- a/regression-tests.api/requirements.txt +++ b/regression-tests.api/requirements.txt @@ -1,3 +1,5 @@ requests==2.20.0 nose==1.3.0 parameterized +mysql-connector +psycopg2 diff --git a/regression-tests.api/runtests.py b/regression-tests.api/runtests.py index 2ff41e5c25..c807fed8e9 100755 --- a/regression-tests.api/runtests.py +++ b/regression-tests.api/runtests.py @@ -4,6 +4,7 @@ from __future__ import print_function import os +import getpass import requests import shutil import subprocess @@ -16,29 +17,63 @@ try: except NameError: raw_input = input +MYSQL_DB='pdnsapi' +MYSQL_USER='root' +MYSQL_HOST=os.environ.get('MYSQL_HOST', 'localhost') +MYSQL_PASSWD='' + +PGSQL_DB='pdnsapi' +PGSQL_USER=getpass.getuser() + SQLITE_DB = 'pdns.sqlite3' + +LMDB_DB = 'pdns.lmdb' + WEBPORT = 5556 DNSPORT = 5300 APIKEY = '1234567890abcdefghijklmnopq-key' WEBPASSWORD = 'something' PDNSUTIL_CMD = [os.environ.get("PDNSUTIL", "../pdns/pdnsutil"), "--config-dir=."] -NAMED_CONF_TPL = """ +ZONES = ["example.com", "powerdnssec.org", "cryptokeys.org"] +ZONE_DIR = "../regression-tests/zones/" + +AUTH_MYSQL_TPL = """ +# Generated by runtests.py +launch=gmysql +gmysql-dnssec=on +gmysql-dbname="""+MYSQL_DB+""" +gmysql-user="""+MYSQL_USER+""" +gmysql-host="""+MYSQL_HOST+""" +gmysql-password="""+MYSQL_PASSWD+""" +""" + +AUTH_PGSQL_TPL = """ # Generated by runtests.py -options { directory "../regression-tests/zones/"; }; -zone "example.com" { type master; file "example.com"; }; -zone "powerdnssec.org" { type master; file "powerdnssec.org"; }; -zone "cryptokeys.org" { type master; file "cryptokeys.org"; }; +launch=gpgsql +gpgsql-dnssec=on +gpgsql-dbname="""+PGSQL_DB+""" +gpgsql-user="""+PGSQL_USER+""" """ -AUTH_CONF_TPL = """ +AUTH_SQLITE_TPL = """ # Generated by runtests.py -launch=gsqlite3,bind +launch=gsqlite3 gsqlite3-dnssec=on gsqlite3-database="""+SQLITE_DB+""" +""" + +AUTH_LMDB_TPL = """ +# Generated by runtests.py +launch=lmdb +lmdb-filename="""+LMDB_DB+""" +""" + +AUTH_COMMON_TPL = """ module-dir=../regression-tests/modules -bind-config=bindbackend.conf default-soa-edit=INCEPTION-INCREMENT +launch+=bind +bind-config=bindbackend.conf """ BINDBACKEND_CONF_TPL = """ @@ -93,9 +128,11 @@ if tests: sys.argv.remove(opt) tests = [opt.split('=', 1)[1] for opt in tests] -daemon = (len(sys.argv) == 2) and sys.argv[1] or None -if daemon not in ('authoritative', 'recursor'): - print("Usage: ./runtests (authoritative|recursor)") +daemon = (len(sys.argv) >= 2) and sys.argv[1] or None +backend = (len(sys.argv) == 3) and sys.argv[2] or 'gsqlite3' + +if daemon not in ('authoritative', 'recursor') or backend not in ('gmysql', 'gpgsql', 'gsqlite3', 'lmdb'): + print("Usage: ./runtests (authoritative|recursor) [gmysql|gpgsql|gsqlite3|lmdb]") sys.exit(2) daemon = sys.argv[1] @@ -122,29 +159,52 @@ if not sdig or not os.path.exists(sdig): if daemon == 'authoritative': zone2sql = os.environ.get("ZONE2SQL", "../pdns/zone2sql") + # Prepare mysql DB with some zones. + if backend == 'gmysql': + subprocess.call(["mysqladmin", "--user=" + MYSQL_USER, "--password=" + MYSQL_PASSWD, "--host=" + MYSQL_HOST, "--force", "drop", MYSQL_DB]) + + run_check_call(["mysqladmin", "--user=" + MYSQL_USER, "--password=" + MYSQL_PASSWD, "--host=" + MYSQL_HOST, "create", MYSQL_DB]) + + with open('../modules/gmysqlbackend/schema.mysql.sql', 'r') as schema_file: + run_check_call(["mysql", "--user=" + MYSQL_USER, "--password=" + MYSQL_PASSWD, "--host=" + MYSQL_HOST, MYSQL_DB], stdin=schema_file) + + with open('pdns.conf', 'w') as pdns_conf: + pdns_conf.write(AUTH_MYSQL_TPL + AUTH_COMMON_TPL) + + # Prepare pgsql DB with some zones. + elif backend == 'gpgsql': + subprocess.call(["dropdb", "--user="+PGSQL_USER, PGSQL_DB]) + + subprocess.check_call(["createdb", "--user="+PGSQL_USER, PGSQL_DB]) + + with open('../modules/gpgsqlbackend/schema.pgsql.sql', 'r') as schema_file: + subprocess.check_call(["psql", "--user="+PGSQL_USER, PGSQL_DB], stdin=schema_file) + + with open('pdns.conf', 'w') as pdns_conf: + pdns_conf.write(AUTH_PGSQL_TPL + AUTH_COMMON_TPL) + # Prepare sqlite DB with some zones. - run_check_call(["rm", "-f", SQLITE_DB]) - if zone2sql == "../pdns/zone2sql": - run_check_call(["make", "-C", "../pdns", "zone2sql"]) - - with open('../modules/gsqlite3backend/schema.sqlite3.sql', 'r') as schema_file: - run_check_call(["sqlite3", SQLITE_DB], stdin=schema_file) - - with open('named.conf', 'w') as named_conf: - named_conf.write(NAMED_CONF_TPL) - with tempfile.TemporaryFile() as tf: - p = subprocess.Popen([zone2sql, "--transactions", "--gsqlite", "--named-conf=named.conf"], stdout=tf) - p.communicate() - if p.returncode != 0: - raise Exception("zone2sql failed") - tf.seek(0, os.SEEK_SET) # rewind - run_check_call(["sqlite3", SQLITE_DB], stdin=tf) + elif backend == 'gsqlite3': + subprocess.call("rm -f " + SQLITE_DB + "*", shell=True) + + with open('../modules/gsqlite3backend/schema.sqlite3.sql', 'r') as schema_file: + run_check_call(["sqlite3", SQLITE_DB], stdin=schema_file) + + with open('pdns.conf', 'w') as pdns_conf: + pdns_conf.write(AUTH_SQLITE_TPL + AUTH_COMMON_TPL) + + # Prepare lmdb DB with some zones. + elif backend == 'lmdb': + subprocess.call("rm -f " + LMDB_DB + "*", shell=True) + + with open('pdns.conf', 'w') as pdns_conf: + pdns_conf.write(AUTH_LMDB_TPL + AUTH_COMMON_TPL) with open('bindbackend.conf', 'w') as bindbackend_conf: bindbackend_conf.write(BINDBACKEND_CONF_TPL) - with open('pdns.conf', 'w') as pdns_conf: - pdns_conf.write(AUTH_CONF_TPL) + for zone in ZONES: + run_check_call(PDNSUTIL_CMD + ["load-zone", zone, ZONE_DIR+zone]) run_check_call(PDNSUTIL_CMD + ["secure-zone", "powerdnssec.org"]) servercmd = [pdns_server] + common_args + ["--no-shuffle", "--dnsupdate=yes", "--cache-ttl=0", "--api=yes"] @@ -195,7 +255,15 @@ test_env.update({ 'WEBPORT': str(WEBPORT), 'APIKEY': APIKEY, 'DAEMON': daemon, + 'BACKEND': backend, + 'MYSQL_DB': MYSQL_DB, + 'MYSQL_USER': MYSQL_USER, + 'MYSQL_HOST': MYSQL_HOST, + 'MYSQL_PASSWD': MYSQL_PASSWD, + 'PGSQL_DB': PGSQL_DB, + 'PGSQL_USER': PGSQL_USER, 'SQLITE_DB': SQLITE_DB, + 'LMDB_DB': LMDB_DB, 'PDNSUTIL_CMD': ' '.join(PDNSUTIL_CMD), 'SDIG': sdig, 'DNSPORT': str(DNSPORT) diff --git a/regression-tests.api/test_helper.py b/regression-tests.api/test_helper.py index c5dbcee20d..92fff44767 100644 --- a/regression-tests.api/test_helper.py +++ b/regression-tests.api/test_helper.py @@ -1,8 +1,11 @@ from __future__ import print_function from datetime import datetime import os +import getpass import requests import unittest +import mysql.connector +import psycopg2 import sqlite3 import subprocess import sys @@ -14,7 +17,15 @@ else: DAEMON = os.environ.get('DAEMON', 'authoritative') PDNSUTIL_CMD = os.environ.get('PDNSUTIL_CMD', 'NOT_SET BUT_THIS MIGHT_BE_A_LIST').split(' ') +BACKEND = os.environ.get('BACKEND', 'gsqlite3') +MYSQL_DB = os.environ.get('MYSQL_DB', 'pdnsapi') +MYSQL_USER = os.environ.get('MYSQL_USER', 'root') +MYSQL_HOST = os.environ.get('MYSQL_HOST', 'localhost') +MYSQL_PASSWD = os.environ.get('MYSQL_PASWORD', '') +PGSQL_DB = os.environ.get('PGSQL_DB', 'pdnsapi') +PGSQL_USER = os.environ.get('PGSQ_USER', getpass.getuser()) SQLITE_DB = os.environ.get('SQLITE_DB', 'pdns.sqlite3') +LMDB_DB = os.environ.get('SQLITE_DB', 'pdns.lmdb') SDIG = os.environ.get('SDIG', 'sdig') DNSPORT = os.environ.get('DNSPORT', '53') @@ -69,20 +80,29 @@ def is_recursor(): def get_auth_db(): """Return Connection to Authoritative backend DB.""" - return sqlite3.Connection(SQLITE_DB) + if BACKEND == 'gmysql': + return mysql.connector.connect(database=MYSQL_DB, user=MYSQL_USER, host=MYSQL_HOST, password=MYSQL_PASSWD), "%s" + elif BACKEND == 'gpgsql': + return psycopg2.connect(database=PGSQL_DB, user=PGSQL_USER), "%s" + else: + return sqlite3.Connection(SQLITE_DB), "?" def get_db_records(zonename, qtype): - with get_auth_db() as db: - rows = db.execute(""" - SELECT name, type, content, ttl, ordername - FROM records - WHERE type = ? AND domain_id = ( - SELECT id FROM domains WHERE name = ? - )""", (qtype, zonename.rstrip('.'))).fetchall() - recs = [{'name': row[0], 'type': row[1], 'content': row[2], 'ttl': row[3], 'ordername': row[4]} for row in rows] - print("DB Records:", recs) - return recs + db, placeholder = get_auth_db() + cur = db.cursor() + cur.execute(""" + SELECT name, type, content, ttl, ordername + FROM records + WHERE type = """+placeholder+""" AND domain_id = ( + SELECT id FROM domains WHERE name = """+placeholder+""" + )""", (qtype, zonename.rstrip('.'))) + rows = cur.fetchall() + cur.close() + db.close() + recs = [{'name': row[0], 'type': row[1], 'content': row[2], 'ttl': row[3], 'ordername': row[4]} for row in rows] + print("DB Records:", recs) + return recs def pdnsutil(subcommand, *args): @@ -102,12 +122,15 @@ def sdig(*args): raise RuntimeError("sdig %s %s failed: %s" % (command, args, except_inst.output.decode('ascii', errors='replace'))) def get_db_tsigkeys(keyname): - with get_auth_db() as db: - rows = db.execute(""" - SELECT name, algorithm, secret - FROM tsigkeys - WHERE name = ?""", (keyname, )).fetchall() - keys = [{'name': row[0], 'algorithm': row[1], 'secret': row[2]} for row in rows] - print("DB TSIG keys:", keys) - return keys - + db, placeholder = get_auth_db() + cur = db.cursor() + cur.execute(""" + SELECT name, algorithm, secret + FROM tsigkeys + WHERE name = """+placeholder, (keyname, )) + rows = cur.fetchall() + cur.close() + db.close() + keys = [{'name': row[0], 'algorithm': row[1], 'secret': row[2]} for row in rows] + print("DB TSIG keys:", keys) + return keys