regression-tests.dnsdist/test_API.py

   1 #!/usr/bin/env python
   2 import os.path
   3
   4 import base64
   5 import dns
   6 import json
   7 import requests
   8 import socket
   9 import time
  10 from dnsdisttests import DNSDistTest, pickAvailablePort
  11
  12 class APITestsBase(DNSDistTest):
  13     __test__ = False
  14     _webTimeout = 5.0
  15     _webServerPort = pickAvailablePort()
  16     _webServerBasicAuthPassword = 'secret'
  17     _webServerBasicAuthPasswordHashed = '$scrypt$ln=10,p=1,r=8$6DKLnvUYEeXWh3JNOd3iwg==$kSrhdHaRbZ7R74q3lGBqO1xetgxRxhmWzYJ2Qvfm7JM='
  18     _webServerAPIKey = 'apisecret'
  19     _webServerAPIKeyHashed = '$scrypt$ln=10,p=1,r=8$9v8JxDfzQVyTpBkTbkUqYg==$bDQzAOHeK1G9UvTPypNhrX48w974ZXbFPtRKS34+aso='
  20     _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed']
  21     _config_template = """
  22     setACL({"127.0.0.1/32", "::1/128"})
  23     newServer{address="127.0.0.1:%s", pool={'', 'mypool'}}
  24     webserver("127.0.0.1:%s")
  25     setWebserverConfig({password="%s", apiKey="%s"})
  26     """
  27     _expectedMetrics = ['responses', 'servfail-responses', 'queries', 'acl-drops',
  28                         'frontend-noerror', 'frontend-nxdomain', 'frontend-servfail',
  29                         'rule-drop', 'rule-nxdomain', 'rule-refused', 'self-answered', 'downstream-timeouts',
  30                         'downstream-send-errors', 'trunc-failures', 'no-policy', 'latency0-1',
  31                         'latency1-10', 'latency10-50', 'latency50-100', 'latency100-1000',
  32                         'latency-slow', 'latency-sum', 'latency-count', 'latency-avg100', 'latency-avg1000',
  33                         'latency-avg10000', 'latency-avg1000000', 'latency-tcp-avg100', 'latency-tcp-avg1000',
  34                         'latency-tcp-avg10000', 'latency-tcp-avg1000000', 'latency-dot-avg100', 'latency-dot-avg1000',
  35                         'latency-dot-avg10000', 'latency-dot-avg1000000', 'latency-doh-avg100', 'latency-doh-avg1000',
  36                         'latency-doh-avg10000', 'latency-doh-avg1000000', 'latency-doq-avg100', 'latency-doq-avg1000',
  37                         'latency-doq-avg10000', 'latency-doq-avg1000000', 'latency-doh3-avg100', 'latency-doh3-avg1000',
  38                         'latency-doh3-avg10000', 'latency-doh3-avg1000000','uptime', 'real-memory-usage', 'noncompliant-queries',
  39                         'noncompliant-responses', 'rdqueries', 'empty-queries', 'cache-hits',
  40                         'cache-misses', 'cpu-iowait', 'cpu-steal', 'cpu-sys-msec', 'cpu-user-msec', 'fd-usage', 'dyn-blocked',
  41                         'dyn-block-nmg-size', 'rule-servfail', 'rule-truncated', 'security-status',
  42                         'udp-in-csum-errors', 'udp-in-errors', 'udp-noport-errors', 'udp-recvbuf-errors', 'udp-sndbuf-errors',
  43                         'udp6-in-errors', 'udp6-recvbuf-errors', 'udp6-sndbuf-errors', 'udp6-noport-errors', 'udp6-in-csum-errors',
  44                         'doh-query-pipe-full', 'doh-response-pipe-full', 'doq-response-pipe-full', 'doh3-response-pipe-full', 'proxy-protocol-invalid', 'tcp-listen-overflows',
  45                         'outgoing-doh-query-pipe-full', 'tcp-query-pipe-full', 'tcp-cross-protocol-query-pipe-full',
  46                         'tcp-cross-protocol-response-pipe-full']
  47     _verboseMode = True
  48
  49     @classmethod
  50     def setUpClass(cls):
  51         cls.startResponders()
  52         cls.startDNSDist()
  53         cls.setUpSockets()
  54         cls.waitForTCPSocket('127.0.0.1', cls._webServerPort)
  55         print("Launching tests..")
  56
  57 class TestAPIBasics(APITestsBase):
  58
  59     # paths accessible using the API key only
  60     _apiOnlyPaths = ['/api/v1/servers/localhost/config', '/api/v1/servers/localhost/config/allow-from', '/api/v1/servers/localhost/statistics']
  61     # paths accessible using an API key or basic auth
  62     _statsPaths = [ '/jsonstat?command=stats', '/jsonstat?command=dynblocklist', '/api/v1/servers/localhost']
  63     # paths accessible using basic auth only (list not exhaustive)
  64     _basicOnlyPaths = ['/', '/index.html']
  65     __test__ = True
  66
  67     def testBasicAuth(self):
  68         """
  69         API: Basic Authentication
  70         """
  71         for path in self._basicOnlyPaths + self._statsPaths:
  72             url = 'http://127.0.0.1:' + str(self._webServerPort) + path
  73             r = requests.get(url, auth=('whatever', "evilsecret"), timeout=self._webTimeout)
  74             self.assertEqual(r.status_code, 401)
  75             r = requests.get(url, auth=('whatever', self._webServerBasicAuthPassword), timeout=self._webTimeout)
  76             self.assertTrue(r)
  77             self.assertEqual(r.status_code, 200)
  78
  79     def testXAPIKey(self):
  80         """
  81         API: X-Api-Key
  82         """
  83         headers = {'x-api-key': self._webServerAPIKey}
  84         for path in self._apiOnlyPaths + self._statsPaths:
  85             url = 'http://127.0.0.1:' + str(self._webServerPort) + path
  86             r = requests.get(url, headers=headers, timeout=self._webTimeout)
  87             self.assertTrue(r)
  88             self.assertEqual(r.status_code, 200)
  89
  90     def testWrongXAPIKey(self):
  91         """
  92         API: Wrong X-Api-Key
  93         """
  94         headers = {'x-api-key': "evilapikey"}
  95         for path in self._apiOnlyPaths + self._statsPaths:
  96             url = 'http://127.0.0.1:' + str(self._webServerPort) + path
  97             r = requests.get(url, headers=headers, timeout=self._webTimeout)
  98             self.assertEqual(r.status_code, 401)
  99
 100     def testBasicAuthOnly(self):
 101         """
 102         API: Basic Authentication Only
 103         """
 104         headers = {'x-api-key': self._webServerAPIKey}
 105         for path in self._basicOnlyPaths:
 106             url = 'http://127.0.0.1:' + str(self._webServerPort) + path
 107             r = requests.get(url, headers=headers, timeout=self._webTimeout)
 108             self.assertEqual(r.status_code, 401)
 109
 110     def testAPIKeyOnly(self):
 111         """
 112         API: API Key Only
 113         """
 114         for path in self._apiOnlyPaths:
 115             url = 'http://127.0.0.1:' + str(self._webServerPort) + path
 116             r = requests.get(url, auth=('whatever', self._webServerBasicAuthPassword), timeout=self._webTimeout)
 117             self.assertEqual(r.status_code, 401)
 118
 119     def testServersLocalhost(self):
 120         """
 121         API: /api/v1/servers/localhost
 122         """
 123         headers = {'x-api-key': self._webServerAPIKey}
 124         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/api/v1/servers/localhost'
 125         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 126         self.assertTrue(r)
 127         self.assertEqual(r.status_code, 200)
 128         self.assertTrue(r.json())
 129         content = r.json()
 130
 131         self.assertEqual(content['daemon_type'], 'dnsdist')
 132
 133         rule_groups = ['response-rules', 'cache-hit-response-rules', 'self-answered-response-rules', 'rules']
 134         for key in ['version', 'acl', 'local', 'servers', 'frontends', 'pools'] + rule_groups:
 135             self.assertIn(key, content)
 136
 137         for rule_group in rule_groups:
 138             for rule in content[rule_group]:
 139                 for key in ['id', 'creationOrder', 'matches', 'rule', 'action', 'uuid']:
 140                     self.assertIn(key, rule)
 141                 for key in ['id', 'creationOrder', 'matches']:
 142                     self.assertTrue(rule[key] >= 0)
 143
 144         for server in content['servers']:
 145             for key in ['id', 'latency', 'name', 'weight', 'outstanding', 'qpsLimit',
 146                         'reuseds', 'state', 'address', 'pools', 'qps', 'queries', 'order', 'sendErrors',
 147                         'dropRate', 'responses', 'nonCompliantResponses', 'tcpDiedSendingQuery', 'tcpDiedReadingResponse',
 148                         'tcpGaveUp', 'tcpReadTimeouts', 'tcpWriteTimeouts', 'tcpCurrentConnections',
 149                         'tcpNewConnections', 'tcpReusedConnections', 'tlsResumptions', 'tcpAvgQueriesPerConnection',
 150                         'tcpAvgConnectionDuration', 'tcpLatency', 'protocol', 'healthCheckFailures', 'healthCheckFailuresParsing', 'healthCheckFailuresTimeout', 'healthCheckFailuresNetwork', 'healthCheckFailuresMismatch', 'healthCheckFailuresInvalid']:
 151                 self.assertIn(key, server)
 152
 153             for key in ['id', 'latency', 'weight', 'outstanding', 'qpsLimit', 'reuseds',
 154                         'qps', 'queries', 'order', 'tcpLatency', 'responses', 'nonCompliantResponses']:
 155                 self.assertTrue(server[key] >= 0)
 156
 157             self.assertTrue(server['state'] in ['up', 'down', 'UP', 'DOWN'])
 158
 159         for frontend in content['frontends']:
 160             for key in ['id', 'address', 'udp', 'tcp', 'type', 'queries', 'nonCompliantQueries']:
 161                 self.assertIn(key, frontend)
 162
 163             for key in ['id', 'queries', 'nonCompliantQueries']:
 164                 self.assertTrue(frontend[key] >= 0)
 165
 166         for pool in content['pools']:
 167             for key in ['id', 'name', 'cacheSize', 'cacheEntries', 'cacheHits', 'cacheMisses', 'cacheDeferredInserts', 'cacheDeferredLookups', 'cacheLookupCollisions', 'cacheInsertCollisions', 'cacheTTLTooShorts', 'cacheCleanupCount']:
 168                 self.assertIn(key, pool)
 169
 170             for key in ['id', 'cacheSize', 'cacheEntries', 'cacheHits', 'cacheMisses', 'cacheDeferredInserts', 'cacheDeferredLookups', 'cacheLookupCollisions', 'cacheInsertCollisions', 'cacheTTLTooShorts', 'cacheCleanupCount']:
 171                 self.assertTrue(pool[key] >= 0)
 172
 173         stats = content['statistics']
 174         for key in self._expectedMetrics:
 175             self.assertIn(key, stats)
 176             self.assertTrue(stats[key] >= 0)
 177         for key in stats:
 178             self.assertIn(key, self._expectedMetrics)
 179
 180     def testServersLocalhostPool(self):
 181         """
 182         API: /api/v1/servers/localhost/pool?name=mypool
 183         """
 184         headers = {'x-api-key': self._webServerAPIKey}
 185         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/api/v1/servers/localhost/pool?name=mypool'
 186         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 187         self.assertTrue(r)
 188         self.assertEqual(r.status_code, 200)
 189         self.assertTrue(r.json())
 190         content = r.json()
 191
 192         self.assertIn('stats', content)
 193         self.assertIn('servers', content)
 194
 195         for key in ['name', 'cacheSize', 'cacheEntries', 'cacheHits', 'cacheMisses', 'cacheDeferredInserts', 'cacheDeferredLookups', 'cacheLookupCollisions', 'cacheInsertCollisions', 'cacheTTLTooShorts']:
 196             self.assertIn(key, content['stats'])
 197
 198         for key in ['cacheSize', 'cacheEntries', 'cacheHits', 'cacheMisses', 'cacheDeferredInserts', 'cacheDeferredLookups', 'cacheLookupCollisions', 'cacheInsertCollisions', 'cacheTTLTooShorts']:
 199             self.assertTrue(content['stats'][key] >= 0)
 200
 201         for server in content['servers']:
 202             for key in ['id', 'latency', 'name', 'weight', 'outstanding', 'qpsLimit',
 203                         'reuseds', 'state', 'address', 'pools', 'qps', 'queries', 'order', 'sendErrors',
 204                         'dropRate', 'responses', 'nonCompliantResponses', 'tcpDiedSendingQuery', 'tcpDiedReadingResponse',
 205                         'tcpGaveUp', 'tcpReadTimeouts', 'tcpWriteTimeouts', 'tcpCurrentConnections',
 206                         'tcpNewConnections', 'tcpReusedConnections', 'tcpAvgQueriesPerConnection',
 207                         'tcpAvgConnectionDuration', 'tcpLatency', 'protocol']:
 208                 self.assertIn(key, server)
 209
 210             for key in ['id', 'latency', 'weight', 'outstanding', 'qpsLimit', 'reuseds',
 211                         'qps', 'queries', 'order', 'tcpLatency', 'responses', 'nonCompliantResponses']:
 212                 self.assertTrue(server[key] >= 0)
 213
 214             self.assertTrue(server['state'] in ['up', 'down', 'UP', 'DOWN'])
 215
 216     def testServersIDontExist(self):
 217         """
 218         API: /api/v1/servers/idonotexist (should be 404)
 219         """
 220         headers = {'x-api-key': self._webServerAPIKey}
 221         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/api/v1/servers/idonotexist'
 222         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 223         self.assertEqual(r.status_code, 404)
 224
 225     def testServersLocalhostConfig(self):
 226         """
 227         API: /api/v1/servers/localhost/config
 228         """
 229         headers = {'x-api-key': self._webServerAPIKey}
 230         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/api/v1/servers/localhost/config'
 231         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 232         self.assertTrue(r)
 233         self.assertEqual(r.status_code, 200)
 234         self.assertTrue(r.json())
 235         content = r.json()
 236         values = {}
 237         for entry in content:
 238             for key in ['type', 'name', 'value']:
 239                 self.assertIn(key, entry)
 240
 241             self.assertEqual(entry['type'], 'ConfigSetting')
 242             values[entry['name']] = entry['value']
 243
 244         for key in ['acl', 'control-socket', 'ecs-override', 'ecs-source-prefix-v4',
 245                     'ecs-source-prefix-v6', 'fixup-case', 'max-outstanding', 'server-policy',
 246                     'stale-cache-entries-ttl', 'tcp-recv-timeout', 'tcp-send-timeout',
 247                     'truncate-tc', 'verbose', 'verbose-health-checks']:
 248             self.assertIn(key, values)
 249
 250         for key in ['max-outstanding', 'stale-cache-entries-ttl', 'tcp-recv-timeout',
 251                     'tcp-send-timeout']:
 252             self.assertTrue(values[key] >= 0)
 253
 254         self.assertTrue(values['ecs-source-prefix-v4'] >= 0 and values['ecs-source-prefix-v4'] <= 32)
 255         self.assertTrue(values['ecs-source-prefix-v6'] >= 0 and values['ecs-source-prefix-v6'] <= 128)
 256
 257     def testServersLocalhostConfigAllowFrom(self):
 258         """
 259         API: /api/v1/servers/localhost/config/allow-from
 260         """
 261         headers = {'x-api-key': self._webServerAPIKey}
 262         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/api/v1/servers/localhost/config/allow-from'
 263         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 264         self.assertTrue(r)
 265         self.assertEqual(r.status_code, 200)
 266         self.assertTrue(r.json())
 267         content = r.json()
 268         for key in ['type', 'name', 'value']:
 269             self.assertIn(key, content)
 270
 271         self.assertEqual(content['name'], 'allow-from')
 272         self.assertEqual(content['type'], 'ConfigSetting')
 273         acl = content['value']
 274         expectedACL = ["127.0.0.1/32", "::1/128"]
 275         acl.sort()
 276         expectedACL.sort()
 277         self.assertEqual(acl, expectedACL)
 278
 279     def testServersLocalhostConfigAllowFromPut(self):
 280         """
 281         API: PUT /api/v1/servers/localhost/config/allow-from (should be refused)
 282
 283         The API is read-only by default, so this should be refused
 284         """
 285         newACL = ["192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24"]
 286         payload = json.dumps({"name": "allow-from",
 287                               "type": "ConfigSetting",
 288                               "value": newACL})
 289         headers = {'x-api-key': self._webServerAPIKey}
 290         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/api/v1/servers/localhost/config/allow-from'
 291         r = requests.put(url, headers=headers, timeout=self._webTimeout, data=payload)
 292         self.assertFalse(r)
 293         self.assertEqual(r.status_code, 405)
 294
 295     def testServersLocalhostStatistics(self):
 296         """
 297         API: /api/v1/servers/localhost/statistics
 298         """
 299         headers = {'x-api-key': self._webServerAPIKey}
 300         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/api/v1/servers/localhost/statistics'
 301         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 302         self.assertTrue(r)
 303         self.assertEqual(r.status_code, 200)
 304         self.assertTrue(r.json())
 305         content = r.json()
 306         values = {}
 307         for entry in content:
 308             self.assertIn('type', entry)
 309             self.assertIn('name', entry)
 310             self.assertIn('value', entry)
 311             self.assertEqual(entry['type'], 'StatisticItem')
 312             values[entry['name']] = entry['value']
 313
 314         for key in self._expectedMetrics:
 315             self.assertIn(key, values)
 316             self.assertTrue(values[key] >= 0)
 317
 318         for key in values:
 319             self.assertIn(key, self._expectedMetrics)
 320
 321     def testJsonstatStats(self):
 322         """
 323         API: /jsonstat?command=stats
 324         """
 325         headers = {'x-api-key': self._webServerAPIKey}
 326         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/jsonstat?command=stats'
 327         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 328         self.assertTrue(r)
 329         self.assertEqual(r.status_code, 200)
 330         self.assertTrue(r.json())
 331         content = r.json()
 332
 333         for key in self._expectedMetrics:
 334             self.assertIn(key, content)
 335             self.assertTrue(content[key] >= 0)
 336
 337     def testJsonstatDynblocklist(self):
 338         """
 339         API: /jsonstat?command=dynblocklist
 340         """
 341         headers = {'x-api-key': self._webServerAPIKey}
 342         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/jsonstat?command=dynblocklist'
 343         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 344         self.assertTrue(r)
 345         self.assertEqual(r.status_code, 200)
 346
 347         content = r.json()
 348
 349         if content:
 350             for key in ['reason', 'seconds', 'blocks', 'action']:
 351                 self.assertIn(key, content)
 352
 353             for key in ['blocks']:
 354                 self.assertTrue(content[key] >= 0)
 355
 356     def testServersLocalhostRings(self):
 357         """
 358         API: /api/v1/servers/localhost/rings
 359         """
 360         headers = {'x-api-key': self._webServerAPIKey}
 361         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/api/v1/servers/localhost/rings'
 362         expectedValues = ['age', 'id', 'name', 'requestor', 'size', 'qtype', 'protocol', 'rd']
 363         expectedResponseValues = expectedValues + ['latency', 'rcode', 'tc', 'aa', 'answers', 'backend']
 364         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 365         self.assertTrue(r)
 366         self.assertEqual(r.status_code, 200)
 367         self.assertTrue(r.json())
 368         content = r.json()
 369         self.assertIn('queries', content)
 370         self.assertIn('responses', content)
 371         self.assertEqual(len(content['queries']), 0)
 372         self.assertEqual(len(content['responses']), 0)
 373
 374         name = 'simple.api.tests.powerdns.com.'
 375         query = dns.message.make_query(name, 'A', 'IN')
 376         response = dns.message.make_response(query)
 377         rrset = dns.rrset.from_text(name,
 378                                     3600,
 379                                     dns.rdataclass.IN,
 380                                     dns.rdatatype.A,
 381                                     '127.0.0.1')
 382         response.answer.append(rrset)
 383
 384         for method in ("sendUDPQuery", "sendTCPQuery"):
 385             sender = getattr(self, method)
 386             (receivedQuery, receivedResponse) = sender(query, response)
 387             self.assertTrue(receivedQuery)
 388             self.assertTrue(receivedResponse)
 389             receivedQuery.id = query.id
 390             self.assertEqual(query, receivedQuery)
 391             self.assertEqual(response, receivedResponse)
 392
 393         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 394         self.assertTrue(r)
 395         self.assertEqual(r.status_code, 200)
 396         self.assertTrue(r.json())
 397         content = r.json()
 398         self.assertIn('queries', content)
 399         self.assertIn('responses', content)
 400         self.assertEqual(len(content['queries']), 2)
 401         self.assertEqual(len(content['responses']), 2)
 402         for entry in content['queries']:
 403             for value in expectedValues:
 404                 self.assertIn(value, entry)
 405         for entry in content['responses']:
 406             for value in expectedResponseValues:
 407                 self.assertIn(value, entry)
 408
 409 class TestAPIServerDown(APITestsBase):
 410     __test__ = True
 411     _config_template = """
 412     setACL({"127.0.0.1/32", "::1/128"})
 413     newServer{address="127.0.0.1:%s"}
 414     getServer(0):setDown()
 415     webserver("127.0.0.1:%s")
 416     setWebserverConfig({password="%s", apiKey="%s"})
 417     """
 418
 419     def testServerDownNoLatencyLocalhost(self):
 420         """
 421         API: /api/v1/servers/localhost, no latency for a down server
 422         """
 423         headers = {'x-api-key': self._webServerAPIKey}
 424         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/api/v1/servers/localhost'
 425         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 426         self.assertTrue(r)
 427         self.assertEqual(r.status_code, 200)
 428         self.assertTrue(r.json())
 429         content = r.json()
 430
 431         self.assertEqual(content['servers'][0]['latency'], None)
 432         self.assertEqual(content['servers'][0]['tcpLatency'], None)
 433
 434 class TestAPIWritable(APITestsBase):
 435     __test__ = True
 436     _APIWriteDir = '/tmp'
 437     _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed', '_APIWriteDir']
 438     _config_template = """
 439     setACL({"127.0.0.1/32", "::1/128"})
 440     newServer{address="127.0.0.1:%s"}
 441     webserver("127.0.0.1:%s")
 442     setWebserverConfig({password="%s", apiKey="%s"})
 443     setAPIWritable(true, "%s")
 444     """
 445
 446     def testSetACL(self):
 447         """
 448         API: Set ACL
 449         """
 450         headers = {'x-api-key': self._webServerAPIKey}
 451         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/api/v1/servers/localhost/config/allow-from'
 452         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 453         self.assertTrue(r)
 454         self.assertEqual(r.status_code, 200)
 455         self.assertTrue(r.json())
 456         content = r.json()
 457         acl = content['value']
 458         expectedACL = ["127.0.0.1/32", "::1/128"]
 459         acl.sort()
 460         expectedACL.sort()
 461         self.assertEqual(acl, expectedACL)
 462
 463         newACL = ["192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24"]
 464         payload = json.dumps({"name": "allow-from",
 465                               "type": "ConfigSetting",
 466                               "value": newACL})
 467         r = requests.put(url, headers=headers, timeout=self._webTimeout, data=payload)
 468         self.assertTrue(r)
 469         self.assertEqual(r.status_code, 200)
 470         self.assertTrue(r.json())
 471         content = r.json()
 472         acl = content['value']
 473         acl.sort()
 474         self.assertEqual(acl, newACL)
 475
 476         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 477         self.assertTrue(r)
 478         self.assertEqual(r.status_code, 200)
 479         self.assertTrue(r.json())
 480         content = r.json()
 481         acl = content['value']
 482         acl.sort()
 483         self.assertEqual(acl, newACL)
 484
 485         configFile = self._APIWriteDir + '/' + 'acl.conf'
 486         self.assertTrue(os.path.isfile(configFile))
 487         fileContent = None
 488         with open(configFile, 'rt') as f:
 489             header = f.readline()
 490             body = f.readline()
 491
 492         self.assertEqual(header, """-- Generated by the REST API, DO NOT EDIT\n""")
 493
 494         self.assertIn(body, {
 495             """setACL({"192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24"})\n""",
 496             """setACL({"192.0.2.0/24", "203.0.113.0/24", "198.51.100.0/24"})\n""",
 497             """setACL({"198.51.100.0/24", "192.0.2.0/24", "203.0.113.0/24"})\n""",
 498             """setACL({"198.51.100.0/24", "203.0.113.0/24", "192.0.2.0/24"})\n""",
 499             """setACL({"203.0.113.0/24", "192.0.2.0/24", "198.51.100.0/24"})\n""",
 500             """setACL({"203.0.113.0/24", "198.51.100.0/24", "192.0.2.0/24"})\n"""
 501         })
 502
 503 class TestAPICustomHeaders(APITestsBase):
 504     __test__ = True
 505     # paths accessible using the API key only
 506     _apiOnlyPath = '/api/v1/servers/localhost/config'
 507     # paths accessible using basic auth only (list not exhaustive)
 508     _basicOnlyPath = '/'
 509     _consoleKey = DNSDistTest.generateConsoleKey()
 510     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
 511     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed']
 512     _config_template = """
 513     setKey("%s")
 514     controlSocket("127.0.0.1:%s")
 515     setACL({"127.0.0.1/32", "::1/128"})
 516     newServer({address="127.0.0.1:%s"})
 517     webserver("127.0.0.1:%s")
 518     setWebserverConfig({password="%s", apiKey="%s", customHeaders={["X-Frame-Options"]="", ["X-Custom"]="custom"} })
 519     """
 520
 521     def testBasicHeaders(self):
 522         """
 523         API: Basic custom headers
 524         """
 525
 526         url = 'http://127.0.0.1:' + str(self._webServerPort) + self._basicOnlyPath
 527
 528         r = requests.get(url, auth=('whatever', self._webServerBasicAuthPassword), timeout=self._webTimeout)
 529         self.assertTrue(r)
 530         self.assertEqual(r.status_code, 200)
 531         self.assertEqual(r.headers.get('x-custom'), "custom")
 532         self.assertFalse("x-frame-options" in r.headers)
 533
 534     def testBasicHeadersUpdate(self):
 535         """
 536         API: Basic update of custom headers
 537         """
 538
 539         url = 'http://127.0.0.1:' + str(self._webServerPort) + self._basicOnlyPath
 540         self.sendConsoleCommand('setWebserverConfig({customHeaders={["x-powered-by"]="dnsdist"}})')
 541         r = requests.get(url, auth=('whatever', self._webServerBasicAuthPassword), timeout=self._webTimeout)
 542         self.assertTrue(r)
 543         self.assertEqual(r.status_code, 200)
 544         self.assertEqual(r.headers.get('x-powered-by'), "dnsdist")
 545         self.assertTrue("x-frame-options" in r.headers)
 546
 547 class TestStatsWithoutAuthentication(APITestsBase):
 548     __test__ = True
 549     # paths accessible using the API key only
 550     _apiOnlyPath = '/api/v1/servers/localhost/config'
 551     # paths accessible using basic auth only (list not exhaustive)
 552     _basicOnlyPath = '/'
 553     _noAuthenticationPaths = [ '/metrics', '/jsonstat?command=dynblocklist' ]
 554     _consoleKey = DNSDistTest.generateConsoleKey()
 555     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
 556     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed']
 557     _config_template = """
 558     setKey("%s")
 559     controlSocket("127.0.0.1:%s")
 560     setACL({"127.0.0.1/32", "::1/128"})
 561     newServer({address="127.0.0.1:%s"})
 562     webserver("127.0.0.1:%s")
 563     setWebserverConfig({password="%s", apiKey="%s", statsRequireAuthentication=false })
 564     """
 565
 566     def testAuth(self):
 567         """
 568         API: Stats do not require authentication
 569         """
 570
 571         for path in self._noAuthenticationPaths:
 572             url = 'http://127.0.0.1:' + str(self._webServerPort) + path
 573
 574             r = requests.get(url, timeout=self._webTimeout)
 575             self.assertTrue(r)
 576             self.assertEqual(r.status_code, 200)
 577
 578         # these should still require basic authentication
 579         for path in [self._basicOnlyPath]:
 580             url = 'http://127.0.0.1:' + str(self._webServerPort) + path
 581
 582             r = requests.get(url, timeout=self._webTimeout)
 583             self.assertEqual(r.status_code, 401)
 584
 585             r = requests.get(url, auth=('whatever', self._webServerBasicAuthPassword), timeout=self._webTimeout)
 586             self.assertTrue(r)
 587             self.assertEqual(r.status_code, 200)
 588
 589         # these should still require API authentication
 590         for path in [self._apiOnlyPath]:
 591             url = 'http://127.0.0.1:' + str(self._webServerPort) + path
 592
 593             r = requests.get(url, timeout=self._webTimeout)
 594             self.assertEqual(r.status_code, 401)
 595
 596             headers = {'x-api-key': self._webServerAPIKey}
 597             r = requests.get(url, headers=headers, timeout=self._webTimeout)
 598             self.assertTrue(r)
 599             self.assertEqual(r.status_code, 200)
 600
 601 class TestAPIAuth(APITestsBase):
 602     __test__ = True
 603     _webServerBasicAuthPasswordNew = 'password'
 604     _webServerBasicAuthPasswordNewHashed = '$scrypt$ln=10,p=1,r=8$yefz8SAuT3lj3moXqUYvmw==$T98/RYMp76ZYNjd7MpAkcVXZEDqpLtrc3tQ52QflVBA='
 605     _webServerAPIKeyNew = 'apipassword'
 606     _webServerAPIKeyNewHashed = '$scrypt$ln=9,p=1,r=8$y96I9nfkY0LWDQEdSUzWgA==$jiyn9QD36o9d0ADrlqiIBk4AKyQrkD1KYw3CexwtHp4='
 607     # paths accessible using the API key only
 608     _apiOnlyPath = '/api/v1/servers/localhost/config'
 609     # paths accessible using basic auth only (list not exhaustive)
 610     _basicOnlyPath = '/'
 611     _consoleKey = DNSDistTest.generateConsoleKey()
 612     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
 613     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed']
 614     _config_template = """
 615     setKey("%s")
 616     controlSocket("127.0.0.1:%s")
 617     setACL({"127.0.0.1/32", "::1/128"})
 618     newServer{address="127.0.0.1:%s"}
 619     webserver("127.0.0.1:%s")
 620     setWebserverConfig({password="%s", apiKey="%s"})
 621     """
 622
 623     def testBasicAuthChange(self):
 624         """
 625         API: Basic Authentication updating credentials
 626         """
 627
 628         url = 'http://127.0.0.1:' + str(self._webServerPort) + self._basicOnlyPath
 629         self.sendConsoleCommand('setWebserverConfig({{password="{}"}})'.format(self._webServerBasicAuthPasswordNewHashed))
 630
 631         r = requests.get(url, auth=('whatever', self._webServerBasicAuthPasswordNew), timeout=self._webTimeout)
 632         self.assertTrue(r)
 633         self.assertEqual(r.status_code, 200)
 634
 635         # Make sure the old password is not usable any more
 636         r = requests.get(url, auth=('whatever', self._webServerBasicAuthPassword), timeout=self._webTimeout)
 637         self.assertEqual(r.status_code, 401)
 638
 639     def testXAPIKeyChange(self):
 640         """
 641         API: X-Api-Key updating credentials
 642         """
 643
 644         url = 'http://127.0.0.1:' + str(self._webServerPort) + self._apiOnlyPath
 645         self.sendConsoleCommand('setWebserverConfig({{apiKey="{}"}})'.format(self._webServerAPIKeyNewHashed))
 646
 647         headers = {'x-api-key': self._webServerAPIKeyNew}
 648         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 649         self.assertTrue(r)
 650         self.assertEqual(r.status_code, 200)
 651
 652         # Make sure the old password is not usable any more
 653         headers = {'x-api-key': self._webServerAPIKey}
 654         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 655         self.assertEqual(r.status_code, 401)
 656
 657     def testBasicAuthOnlyChange(self):
 658         """
 659         API: X-Api-Key updated to none (disabled)
 660         """
 661
 662         url = 'http://127.0.0.1:' + str(self._webServerPort) + self._apiOnlyPath
 663         self.sendConsoleCommand('setWebserverConfig({{apiKey="{}"}})'.format(self._webServerAPIKeyNewHashed))
 664
 665         headers = {'x-api-key': self._webServerAPIKeyNew}
 666         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 667         self.assertTrue(r)
 668         self.assertEqual(r.status_code, 200)
 669
 670         # now disable apiKey
 671         self.sendConsoleCommand('setWebserverConfig({apiKey=""})')
 672
 673         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 674         self.assertEqual(r.status_code, 401)
 675
 676 class TestAPIACL(APITestsBase):
 677     __test__ = True
 678     _consoleKey = DNSDistTest.generateConsoleKey()
 679     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
 680     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed']
 681     _config_template = """
 682     setKey("%s")
 683     controlSocket("127.0.0.1:%s")
 684     setACL({"127.0.0.1/32", "::1/128"})
 685     newServer{address="127.0.0.1:%s"}
 686     webserver("127.0.0.1:%s")
 687     setWebserverConfig({password="%s", apiKey="%s", acl="192.0.2.1"})
 688     """
 689
 690     def testACLChange(self):
 691         """
 692         API: Should be denied by ACL then allowed
 693         """
 694
 695         url = 'http://127.0.0.1:' + str(self._webServerPort) + "/"
 696         try:
 697             r = requests.get(url, auth=('whatever', self._webServerBasicAuthPassword), timeout=self._webTimeout)
 698             self.assertTrue(False)
 699         except requests.exceptions.ConnectionError as exp:
 700             pass
 701
 702         # reset the ACL
 703         self.sendConsoleCommand('setWebserverConfig({acl="127.0.0.1"})')
 704
 705         r = requests.get(url, auth=('whatever', self._webServerBasicAuthPassword), timeout=self._webTimeout)
 706         self.assertTrue(r)
 707         self.assertEqual(r.status_code, 200)
 708
 709 class TestAPIWithoutAuthentication(APITestsBase):
 710     __test__ = True
 711     _apiPath = '/api/v1/servers/localhost/config'
 712     # paths accessible using basic auth only (list not exhaustive)
 713     _basicOnlyPath = '/'
 714     _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed']
 715     _config_template = """
 716     setACL({"127.0.0.1/32", "::1/128"})
 717     newServer({address="127.0.0.1:%s"})
 718     webserver("127.0.0.1:%s")
 719     setWebserverConfig({password="%s", apiRequiresAuthentication=false })
 720     """
 721
 722     def testAuth(self):
 723         """
 724         API: API do not require authentication
 725         """
 726
 727         for path in [self._apiPath]:
 728             url = 'http://127.0.0.1:' + str(self._webServerPort) + path
 729
 730             r = requests.get(url, timeout=self._webTimeout)
 731             self.assertTrue(r)
 732             self.assertEqual(r.status_code, 200)
 733
 734         # these should still require basic authentication
 735         for path in [self._basicOnlyPath]:
 736             url = 'http://127.0.0.1:' + str(self._webServerPort) + path
 737
 738             r = requests.get(url, timeout=self._webTimeout)
 739             self.assertEqual(r.status_code, 401)
 740
 741             r = requests.get(url, auth=('whatever', self._webServerBasicAuthPassword), timeout=self._webTimeout)
 742             self.assertTrue(r)
 743             self.assertEqual(r.status_code, 200)
 744
 745 class TestDashboardWithoutAuthentication(APITestsBase):
 746     __test__ = True
 747     _basicPath = '/'
 748     _config_params = ['_testServerPort', '_webServerPort']
 749     _config_template = """
 750     setACL({"127.0.0.1/32", "::1/128"})
 751     newServer({address="127.0.0.1:%d"})
 752     webserver("127.0.0.1:%d")
 753     setWebserverConfig({ dashboardRequiresAuthentication=false })
 754     """
 755     _verboseMode=True
 756
 757     def testDashboard(self):
 758         """
 759         API: Dashboard do not require authentication
 760         """
 761
 762         for path in [self._basicPath]:
 763             url = 'http://127.0.0.1:' + str(self._webServerPort) + path
 764
 765             r = requests.get(url, timeout=self._webTimeout)
 766             self.assertTrue(r)
 767             self.assertEqual(r.status_code, 200)
 768
 769 class TestCustomLuaEndpoint(APITestsBase):
 770     __test__ = True
 771     _config_template = """
 772     setACL({"127.0.0.1/32", "::1/128"})
 773     newServer{address="127.0.0.1:%s"}
 774     webserver("127.0.0.1:%s")
 775     setWebserverConfig({password="%s"})
 776
 777     function customHTTPHandler(req, resp)
 778       if req.path ~= '/foo' then
 779         resp.status = 500
 780         return
 781       end
 782
 783       if req.version ~= 11 then
 784         resp.status = 501
 785         return
 786       end
 787
 788       if req.method ~= 'GET' then
 789         resp.status = 502
 790         return
 791       end
 792
 793       local get = req.getvars
 794       if get['param'] ~= '42' then
 795         resp.status = 503
 796         return
 797       end
 798
 799       local headers = req.headers
 800       if headers['customheader'] ~= 'foobar' then
 801         resp.status = 504
 802         return
 803       end
 804
 805       resp.body = 'It works!'
 806       resp.status = 200
 807       resp.headers = { ['Foo']='Bar'}
 808     end
 809     registerWebHandler('/foo', customHTTPHandler)
 810     """
 811     _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed']
 812
 813     def testBasic(self):
 814         """
 815         Custom Web Handler
 816         """
 817         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/foo?param=42'
 818         headers = {'customheader': 'foobar'}
 819         r = requests.get(url, auth=('whatever', self._webServerBasicAuthPassword), timeout=self._webTimeout, headers=headers)
 820         self.assertTrue(r)
 821         self.assertEqual(r.status_code, 200)
 822         self.assertEqual(r.content, b'It works!')
 823         self.assertEqual(r.headers.get('foo'), "Bar")
 824
 825 class TestWebConcurrentConnections(APITestsBase):
 826     __test__ = True
 827     _maxConns = 2
 828
 829     _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed', '_maxConns']
 830     _config_template = """
 831     newServer{address="127.0.0.1:%s"}
 832     webserver("127.0.0.1:%s")
 833     setWebserverConfig({password="%s", apiKey="%s", maxConcurrentConnections=%d})
 834     """
 835
 836     @classmethod
 837     def setUpClass(cls):
 838         cls.startResponders()
 839         cls.startDNSDist()
 840         cls.setUpSockets()
 841         # do no check if the web server socket is up, because this
 842         # might mess up the concurrent connections counter
 843
 844     def testConcurrentConnections(self):
 845         """
 846         Web: Concurrent connections
 847         """
 848
 849         conns = []
 850         # open the maximum number of connections
 851         for _ in range(self._maxConns):
 852             conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 853             conn.connect(("127.0.0.1", self._webServerPort))
 854             conns.append(conn)
 855
 856         # we now hold all the slots, let's try to establish a new connection
 857         url = 'http://127.0.0.1:' + str(self._webServerPort) + "/"
 858         self.assertRaises(requests.exceptions.ConnectionError, requests.get, url, auth=('whatever', self._webServerBasicAuthPassword), timeout=self._webTimeout)
 859
 860         # free one slot
 861         conns[0].close()
 862         conns[0] = None
 863         time.sleep(1)
 864
 865         # this should work
 866         r = requests.get(url, auth=('whatever', self._webServerBasicAuthPassword), timeout=self._webTimeout)
 867         self.assertTrue(r)
 868         self.assertEqual(r.status_code, 200)
 869
 870 class TestAPICustomStatistics(APITestsBase):
 871     __test__ = True
 872     _maxConns = 2
 873
 874     _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed']
 875     _config_template = """
 876     newServer{address="127.0.0.1:%s"}
 877     webserver("127.0.0.1:%s")
 878     declareMetric("my-custom-metric", "counter", "Number of statistics")
 879     declareMetric("my-other-metric", "counter", "Another number of statistics")
 880     declareMetric("my-gauge", "gauge", "Current memory usage")
 881     setWebserverConfig({password="%s", apiKey="%s"})
 882     """
 883
 884     def testCustomStats(self):
 885         """
 886         API: /jsonstat?command=stats
 887         Test custom statistics are exposed
 888         """
 889         headers = {'x-api-key': self._webServerAPIKey}
 890         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/jsonstat?command=stats'
 891         r = requests.get(url, headers=headers, timeout=self._webTimeout)
 892         self.assertTrue(r)
 893         self.assertEqual(r.status_code, 200)
 894         self.assertTrue(r.json())
 895         content = r.json()
 896
 897         expected = ['my-custom-metric', 'my-other-metric', 'my-gauge']
 898
 899         for key in expected:
 900             self.assertIn(key, content)
 901             self.assertTrue(content[key] >= 0)