regression-tests.dnsdist/test_Protobuf.py

   1 #!/usr/bin/env python
   2 import base64
   3 import threading
   4 import socket
   5 import struct
   6 import sys
   7 import time
   8 from dnsdisttests import DNSDistTest, pickAvailablePort, Queue
   9 from proxyprotocol import ProxyProtocol
  10
  11 import dns
  12 import dnsmessage_pb2
  13
  14 class DNSDistProtobufTest(DNSDistTest):
  15     _protobufServerPort = pickAvailablePort()
  16     _protobufQueue = Queue()
  17     _protobufServerID = 'dnsdist-server-1'
  18     _protobufCounter = 0
  19
  20     @classmethod
  21     def ProtobufListener(cls, port):
  22         cls._backgroundThreads[threading.get_native_id()] = True
  23         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  24         sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
  25         try:
  26             sock.bind(("127.0.0.1", port))
  27         except socket.error as e:
  28             print("Error binding in the protbuf listener: %s" % str(e))
  29             sys.exit(1)
  30
  31         sock.listen(100)
  32         sock.settimeout(1.0)
  33         while True:
  34             try:
  35                 (conn, _) = sock.accept()
  36             except socket.timeout:
  37                 if cls._backgroundThreads.get(threading.get_native_id(), False) == False:
  38                     del cls._backgroundThreads[threading.get_native_id()]
  39                     break
  40                 else:
  41                     continue
  42
  43             data = None
  44             while True:
  45                 data = conn.recv(2)
  46                 if not data:
  47                     break
  48                 (datalen,) = struct.unpack("!H", data)
  49                 data = conn.recv(datalen)
  50                 if not data:
  51                     break
  52
  53                 cls._protobufQueue.put(data, True, timeout=2.0)
  54
  55             conn.close()
  56         sock.close()
  57
  58     @classmethod
  59     def startResponders(cls):
  60         cls._UDPResponder = threading.Thread(name='UDP Responder', target=cls.UDPResponder, args=[cls._testServerPort, cls._toResponderQueue, cls._fromResponderQueue])
  61         cls._UDPResponder.daemon = True
  62         cls._UDPResponder.start()
  63
  64         cls._TCPResponder = threading.Thread(name='TCP Responder', target=cls.TCPResponder, args=[cls._testServerPort, cls._toResponderQueue, cls._fromResponderQueue])
  65         cls._TCPResponder.daemon = True
  66         cls._TCPResponder.start()
  67
  68         cls._protobufListener = threading.Thread(name='Protobuf Listener', target=cls.ProtobufListener, args=[cls._protobufServerPort])
  69         cls._protobufListener.daemon = True
  70         cls._protobufListener.start()
  71
  72     def getFirstProtobufMessage(self):
  73         self.assertFalse(self._protobufQueue.empty())
  74         data = self._protobufQueue.get(False)
  75         self.assertTrue(data)
  76         msg = dnsmessage_pb2.PBDNSMessage()
  77         msg.ParseFromString(data)
  78         return msg
  79
  80     def checkProtobufBase(self, msg, protocol, query, initiator, normalQueryResponse=True, v6=False):
  81         self.assertTrue(msg)
  82         self.assertTrue(msg.HasField('timeSec'))
  83         self.assertTrue(msg.HasField('socketFamily'))
  84         if v6:
  85             self.assertEqual(msg.socketFamily, dnsmessage_pb2.PBDNSMessage.INET6)
  86         else:
  87             self.assertEqual(msg.socketFamily, dnsmessage_pb2.PBDNSMessage.INET)
  88         self.assertTrue(msg.HasField('from'))
  89         fromvalue = getattr(msg, 'from')
  90         if v6:
  91             self.assertEqual(socket.inet_ntop(socket.AF_INET6, fromvalue), initiator)
  92         else:
  93             self.assertEqual(socket.inet_ntop(socket.AF_INET, fromvalue), initiator)
  94         self.assertTrue(msg.HasField('socketProtocol'))
  95         self.assertEqual(msg.socketProtocol, protocol)
  96         self.assertTrue(msg.HasField('messageId'))
  97         self.assertTrue(msg.HasField('id'))
  98         self.assertEqual(msg.id, query.id)
  99         self.assertTrue(msg.HasField('inBytes'))
 100         self.assertTrue(msg.HasField('serverIdentity'))
 101         self.assertEqual(msg.serverIdentity, self._protobufServerID.encode('utf-8'))
 102
 103         if normalQueryResponse and (protocol == dnsmessage_pb2.PBDNSMessage.UDP or protocol == dnsmessage_pb2.PBDNSMessage.TCP):
 104           # compare inBytes with length of query/response
 105           self.assertEqual(msg.inBytes, len(query.to_wire()))
 106         # dnsdist doesn't set the existing EDNS Subnet for now,
 107         # although it might be set from Lua
 108         # self.assertTrue(msg.HasField('originalRequestorSubnet'))
 109         # self.assertEqual(len(msg.originalRequestorSubnet), 4)
 110         # self.assertEqual(socket.inet_ntop(socket.AF_INET, msg.originalRequestorSubnet), '127.0.0.1')
 111
 112     def checkProtobufQuery(self, msg, protocol, query, qclass, qtype, qname, initiator='127.0.0.1', v6=False):
 113         self.assertEqual(msg.type, dnsmessage_pb2.PBDNSMessage.DNSQueryType)
 114         self.checkProtobufBase(msg, protocol, query, initiator, v6=v6)
 115         # dnsdist doesn't fill the responder field for responses
 116         # because it doesn't keep the information around.
 117         self.assertTrue(msg.HasField('to'))
 118         if not v6:
 119             self.assertEqual(socket.inet_ntop(socket.AF_INET, msg.to), '127.0.0.1')
 120         self.assertTrue(msg.HasField('question'))
 121         self.assertTrue(msg.question.HasField('qClass'))
 122         self.assertEqual(msg.question.qClass, qclass)
 123         self.assertTrue(msg.question.HasField('qType'))
 124         self.assertEqual(msg.question.qClass, qtype)
 125         self.assertTrue(msg.question.HasField('qName'))
 126         self.assertEqual(msg.question.qName, qname)
 127
 128     def checkProtobufTags(self, tags, expectedTags):
 129         # only differences will be in new list
 130         listx = set(tags) ^ set(expectedTags)
 131         # exclusive or of lists should be empty
 132         self.assertEqual(len(listx), 0, "Protobuf tags don't match")
 133
 134     def checkProtobufQueryConvertedToResponse(self, msg, protocol, response, initiator='127.0.0.0'):
 135         self.assertEqual(msg.type, dnsmessage_pb2.PBDNSMessage.DNSResponseType)
 136         # skip comparing inBytes (size of the query) with the length of the generated response
 137         self.checkProtobufBase(msg, protocol, response, initiator, False)
 138         self.assertTrue(msg.HasField('response'))
 139         self.assertTrue(msg.response.HasField('queryTimeSec'))
 140
 141     def checkProtobufResponse(self, msg, protocol, response, initiator='127.0.0.1', v6=False):
 142         self.assertEqual(msg.type, dnsmessage_pb2.PBDNSMessage.DNSResponseType)
 143         self.checkProtobufBase(msg, protocol, response, initiator, v6=v6)
 144         self.assertTrue(msg.HasField('response'))
 145         self.assertTrue(msg.response.HasField('queryTimeSec'))
 146
 147     def checkProtobufResponseRecord(self, record, rclass, rtype, rname, rttl):
 148         self.assertTrue(record.HasField('class'))
 149         self.assertEqual(getattr(record, 'class'), rclass)
 150         self.assertTrue(record.HasField('type'))
 151         self.assertEqual(record.type, rtype)
 152         self.assertTrue(record.HasField('name'))
 153         self.assertEqual(record.name, rname)
 154         self.assertTrue(record.HasField('ttl'))
 155         self.assertEqual(record.ttl, rttl)
 156         self.assertTrue(record.HasField('rdata'))
 157
 158 class TestProtobuf(DNSDistProtobufTest):
 159     _config_params = ['_testServerPort', '_protobufServerPort', '_protobufServerID', '_protobufServerID']
 160     _config_template = """
 161     luasmn = newSuffixMatchNode()
 162     luasmn:add(newDNSName('lua.protobuf.tests.powerdns.com.'))
 163
 164     function alterProtobufResponse(dq, protobuf)
 165       if luasmn:check(dq.qname) then
 166         requestor = newCA(tostring(dq.remoteaddr))              -- called by testLuaProtobuf()
 167         if requestor:isIPv4() then
 168           requestor:truncate(24)
 169         else
 170           requestor:truncate(56)
 171         end
 172         protobuf:setRequestor(requestor)
 173
 174         local tableTags = {}
 175         table.insert(tableTags, "TestLabel1,TestData1")
 176         table.insert(tableTags, "TestLabel2,TestData2")
 177
 178         protobuf:setTagArray(tableTags)
 179
 180         protobuf:setTag('TestLabel3,TestData3')
 181
 182         protobuf:setTag("Response,456")
 183
 184       else
 185
 186         local tableTags = {}                                    -- called by testProtobuf()
 187         table.insert(tableTags, "TestLabel1,TestData1")
 188         table.insert(tableTags, "TestLabel2,TestData2")
 189         protobuf:setTagArray(tableTags)
 190
 191         protobuf:setTag('TestLabel3,TestData3')
 192
 193         protobuf:setTag("Response,456")
 194
 195       end
 196     end
 197
 198     function alterProtobufQuery(dq, protobuf)
 199
 200       if luasmn:check(dq.qname) then
 201         requestor = newCA(tostring(dq.remoteaddr))              -- called by testLuaProtobuf()
 202         if requestor:isIPv4() then
 203           requestor:truncate(24)
 204         else
 205           requestor:truncate(56)
 206         end
 207         protobuf:setRequestor(requestor)
 208
 209         local tableTags = {}
 210         tableTags = dq:getTagArray()                            -- get table from DNSQuery
 211
 212         local tablePB = {}
 213           for k, v in pairs( tableTags) do
 214           table.insert(tablePB, k .. "," .. v)
 215         end
 216
 217         protobuf:setTagArray(tablePB)                           -- store table in protobuf
 218         protobuf:setTag("Query,123")                            -- add another tag entry in protobuf
 219
 220         protobuf:setResponseCode(DNSRCode.NXDOMAIN)             -- set protobuf response code to be NXDOMAIN
 221
 222         local strReqName = tostring(dq.qname)                   -- get request dns name
 223
 224         protobuf:setProtobufResponseType()                      -- set protobuf to look like a response and not a query, with 0 default time
 225
 226         blobData = '\127' .. '\000' .. '\000' .. '\002'         -- 127.0.0.2, note: lua 5.1 can only embed decimal not hex
 227
 228         protobuf:addResponseRR(strReqName, 1, 1, 123, blobData) -- add a RR to the protobuf
 229
 230         protobuf:setBytes(65)                                   -- set the size of the query to confirm in checkProtobufBase
 231
 232       else
 233
 234         local tableTags = {}                                    -- called by testProtobuf()
 235         table.insert(tableTags, "TestLabel1,TestData1")
 236         table.insert(tableTags, "TestLabel2,TestData2")
 237
 238         protobuf:setTagArray(tableTags)
 239         protobuf:setTag('TestLabel3,TestData3')
 240         protobuf:setTag("Query,123")
 241
 242       end
 243     end
 244
 245     function alterLuaFirst(dq)                                  -- called when dnsdist receives new request
 246       local tt = {}
 247       tt["TestLabel1"] = "TestData1"
 248       tt["TestLabel2"] = "TestData2"
 249
 250       dq:setTagArray(tt)
 251
 252       dq:setTag("TestLabel3","TestData3")
 253       return DNSAction.None, ""                         -- continue to the next rule
 254     end
 255
 256     newServer{address="127.0.0.1:%s", useClientSubnet=true}
 257     rl = newRemoteLogger('127.0.0.1:%s')
 258
 259     addAction(AllRule(), LuaAction(alterLuaFirst))                                                      -- Add tags to DNSQuery first
 260
 261     addAction(AllRule(), RemoteLogAction(rl, alterProtobufQuery, {serverID='%s'}))                              -- Send protobuf message before lookup
 262
 263     addResponseAction(AllRule(), RemoteLogResponseAction(rl, alterProtobufResponse, true, {serverID='%s'}))     -- Send protobuf message after lookup
 264
 265     """
 266
 267     def testProtobuf(self):
 268         """
 269         Protobuf: Send data to a protobuf server
 270         """
 271         name = 'query.protobuf.tests.powerdns.com.'
 272
 273         target = 'target.protobuf.tests.powerdns.com.'
 274         query = dns.message.make_query(name, 'A', 'IN')
 275         response = dns.message.make_response(query)
 276
 277         rrset = dns.rrset.from_text(name,
 278                                     3600,
 279                                     dns.rdataclass.IN,
 280                                     dns.rdatatype.CNAME,
 281                                     target)
 282         response.answer.append(rrset)
 283
 284         rrset = dns.rrset.from_text(target,
 285                                     3600,
 286                                     dns.rdataclass.IN,
 287                                     dns.rdatatype.A,
 288                                     '127.0.0.1')
 289         response.answer.append(rrset)
 290
 291         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 292         self.assertTrue(receivedQuery)
 293         self.assertTrue(receivedResponse)
 294         receivedQuery.id = query.id
 295         self.assertEqual(query, receivedQuery)
 296         self.assertEqual(response, receivedResponse)
 297
 298         # let the protobuf messages the time to get there
 299         time.sleep(1)
 300
 301         # check the protobuf message corresponding to the UDP query
 302         msg = self.getFirstProtobufMessage()
 303
 304         self.checkProtobufQuery(msg, dnsmessage_pb2.PBDNSMessage.UDP, query, dns.rdataclass.IN, dns.rdatatype.A, name)
 305         self.checkProtobufTags(msg.response.tags, [u"TestLabel1,TestData1", u"TestLabel2,TestData2", u"TestLabel3,TestData3", u"Query,123"])
 306
 307         # check the protobuf message corresponding to the UDP response
 308         msg = self.getFirstProtobufMessage()
 309         self.checkProtobufResponse(msg, dnsmessage_pb2.PBDNSMessage.UDP, response)
 310         self.checkProtobufTags(msg.response.tags, [ u"TestLabel1,TestData1", u"TestLabel2,TestData2", u"TestLabel3,TestData3", u"Response,456"])
 311         self.assertEqual(len(msg.response.rrs), 2)
 312         rr = msg.response.rrs[0]
 313         self.checkProtobufResponseRecord(rr, dns.rdataclass.IN, dns.rdatatype.CNAME, name, 3600)
 314         self.assertEqual(rr.rdata.decode('utf-8'), target)
 315         rr = msg.response.rrs[1]
 316         self.checkProtobufResponseRecord(rr, dns.rdataclass.IN, dns.rdatatype.A, target, 3600)
 317         self.assertEqual(socket.inet_ntop(socket.AF_INET, rr.rdata), '127.0.0.1')
 318
 319         (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response)
 320         self.assertTrue(receivedQuery)
 321         self.assertTrue(receivedResponse)
 322         receivedQuery.id = query.id
 323         self.assertEqual(query, receivedQuery)
 324         self.assertEqual(response, receivedResponse)
 325
 326         # let the protobuf messages the time to get there
 327         time.sleep(1)
 328
 329         # check the protobuf message corresponding to the TCP query
 330         msg = self.getFirstProtobufMessage()
 331
 332         self.checkProtobufQuery(msg, dnsmessage_pb2.PBDNSMessage.TCP, query, dns.rdataclass.IN, dns.rdatatype.A, name)
 333         self.checkProtobufTags(msg.response.tags, [u"TestLabel1,TestData1", u"TestLabel2,TestData2", u"TestLabel3,TestData3", u"Query,123"])
 334
 335         # check the protobuf message corresponding to the TCP response
 336         msg = self.getFirstProtobufMessage()
 337         self.checkProtobufResponse(msg, dnsmessage_pb2.PBDNSMessage.TCP, response)
 338         self.checkProtobufTags(msg.response.tags, [ u"TestLabel1,TestData1", u"TestLabel2,TestData2", u"TestLabel3,TestData3", u"Response,456"])
 339         self.assertEqual(len(msg.response.rrs), 2)
 340         rr = msg.response.rrs[0]
 341         self.checkProtobufResponseRecord(rr, dns.rdataclass.IN, dns.rdatatype.CNAME, name, 3600)
 342         self.assertEqual(rr.rdata.decode('utf-8'), target)
 343         rr = msg.response.rrs[1]
 344         self.checkProtobufResponseRecord(rr, dns.rdataclass.IN, dns.rdatatype.A, target, 3600)
 345         self.assertEqual(socket.inet_ntop(socket.AF_INET, rr.rdata), '127.0.0.1')
 346
 347     def testLuaProtobuf(self):
 348
 349         """
 350         Protobuf: Check that the Lua callback rewrote the initiator
 351         """
 352         name = 'lua.protobuf.tests.powerdns.com.'
 353         query = dns.message.make_query(name, 'A', 'IN')
 354         response = dns.message.make_response(query)
 355         rrset = dns.rrset.from_text(name,
 356                                     3600,
 357                                     dns.rdataclass.IN,
 358                                     dns.rdatatype.A,
 359                                     '127.0.0.1')
 360         response.answer.append(rrset)
 361
 362
 363         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 364
 365         self.assertTrue(receivedQuery)
 366         self.assertTrue(receivedResponse)
 367         receivedQuery.id = query.id
 368         self.assertEqual(query, receivedQuery)
 369         self.assertEqual(response, receivedResponse)
 370
 371
 372         # let the protobuf messages the time to get there
 373         time.sleep(1)
 374
 375         # check the protobuf message corresponding to the UDP query
 376         msg = self.getFirstProtobufMessage()
 377
 378         self.checkProtobufQueryConvertedToResponse(msg, dnsmessage_pb2.PBDNSMessage.UDP, response, '127.0.0.0')
 379         self.checkProtobufTags(msg.response.tags, [ u"TestLabel1,TestData1", u"TestLabel2,TestData2", u"TestLabel3,TestData3", u"Query,123"])
 380
 381         # check the protobuf message corresponding to the UDP response
 382         msg = self.getFirstProtobufMessage()
 383         self.checkProtobufResponse(msg, dnsmessage_pb2.PBDNSMessage.UDP, response, '127.0.0.0')
 384         self.checkProtobufTags(msg.response.tags, [ u"TestLabel1,TestData1", u"TestLabel2,TestData2", u"TestLabel3,TestData3", u"Response,456"])
 385         self.assertEqual(len(msg.response.rrs), 1)
 386         for rr in msg.response.rrs:
 387             self.checkProtobufResponseRecord(rr, dns.rdataclass.IN, dns.rdatatype.A, name, 3600)
 388             self.assertEqual(socket.inet_ntop(socket.AF_INET, rr.rdata), '127.0.0.1')
 389
 390         (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response)
 391         self.assertTrue(receivedQuery)
 392         self.assertTrue(receivedResponse)
 393         receivedQuery.id = query.id
 394         self.assertEqual(query, receivedQuery)
 395         self.assertEqual(response, receivedResponse)
 396
 397         # let the protobuf messages the time to get there
 398         time.sleep(1)
 399
 400         # check the protobuf message corresponding to the TCP query
 401         msg = self.getFirstProtobufMessage()
 402         self.checkProtobufQueryConvertedToResponse(msg, dnsmessage_pb2.PBDNSMessage.TCP, response, '127.0.0.0')
 403         self.checkProtobufTags(msg.response.tags, [ u"TestLabel1,TestData1", u"TestLabel2,TestData2", u"TestLabel3,TestData3", u"Query,123"])
 404
 405         # check the protobuf message corresponding to the TCP response
 406         msg = self.getFirstProtobufMessage()
 407         self.checkProtobufResponse(msg, dnsmessage_pb2.PBDNSMessage.TCP, response, '127.0.0.0')
 408         self.checkProtobufTags(msg.response.tags, [ u"TestLabel1,TestData1", u"TestLabel2,TestData2", u"TestLabel3,TestData3", u"Response,456"])
 409         self.assertEqual(len(msg.response.rrs), 1)
 410         for rr in msg.response.rrs:
 411             self.checkProtobufResponseRecord(rr, dns.rdataclass.IN, dns.rdatatype.A, name, 3600)
 412             self.assertEqual(socket.inet_ntop(socket.AF_INET, rr.rdata), '127.0.0.1')
 413
 414 class TestProtobufMetaTags(DNSDistProtobufTest):
 415     _config_params = ['_testServerPort', '_protobufServerPort']
 416     _config_template = """
 417     newServer{address="127.0.0.1:%s"}
 418     rl = newRemoteLogger('127.0.0.1:%d')
 419
 420     addAction(AllRule(), SetTagAction('my-tag-key', 'my-tag-value'))
 421     addAction(AllRule(), SetTagAction('my-empty-key', ''))
 422     addAction(AllRule(), RemoteLogAction(rl, nil, {serverID='dnsdist-server-1', exportTags='*'}, {b64='b64-content', ['my-tag-export-name']='tag:my-tag-key'}))
 423     addResponseAction(AllRule(), SetTagResponseAction('my-tag-key2', 'my-tag-value2'))
 424     addResponseAction(AllRule(), RemoteLogResponseAction(rl, nil, false, {serverID='dnsdist-server-1', exportTags='my-empty-key,my-tag-key2'}, {['my-tag-export-name']='tags'}))
 425     """
 426
 427     def testProtobufMeta(self):
 428         """
 429         Protobuf: Meta values
 430         """
 431         name = 'meta.protobuf.tests.powerdns.com.'
 432         query = dns.message.make_query(name, 'A', 'IN')
 433         response = dns.message.make_response(query)
 434         rrset = dns.rrset.from_text(name,
 435                                     3600,
 436                                     dns.rdataclass.IN,
 437                                     dns.rdatatype.A,
 438                                     '127.0.0.1')
 439         response.answer.append(rrset)
 440
 441         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 442         self.assertTrue(receivedQuery)
 443         self.assertTrue(receivedResponse)
 444         receivedQuery.id = query.id
 445         self.assertEqual(query, receivedQuery)
 446         self.assertEqual(response, receivedResponse)
 447
 448         # let the protobuf messages the time to get there
 449         time.sleep(1)
 450
 451         # check the protobuf message corresponding to the UDP query
 452         msg = self.getFirstProtobufMessage()
 453
 454         self.checkProtobufQuery(msg, dnsmessage_pb2.PBDNSMessage.UDP, query, dns.rdataclass.IN, dns.rdatatype.A, name)
 455         # regular tags
 456         self.assertEqual(len(msg.response.tags), 2)
 457         self.assertIn('my-tag-key:my-tag-value', msg.response.tags)
 458         self.assertIn('my-empty-key', msg.response.tags)
 459         # meta tags
 460         self.assertEqual(len(msg.meta), 2)
 461         tags = {}
 462         for entry in msg.meta:
 463             tags[entry.key] = entry.value.stringVal
 464
 465         self.assertIn('b64', tags)
 466         self.assertIn('my-tag-export-name', tags)
 467
 468         b64EncodedQuery = base64.b64encode(query.to_wire()).decode('ascii')
 469         self.assertEqual(tags['b64'], [b64EncodedQuery])
 470         self.assertEqual(tags['my-tag-export-name'], ['my-tag-value'])
 471
 472         # check the protobuf message corresponding to the UDP response
 473         msg = self.getFirstProtobufMessage()
 474         self.checkProtobufResponse(msg, dnsmessage_pb2.PBDNSMessage.UDP, response)
 475         # regular tags
 476         self.assertEqual(len(msg.response.tags), 2)
 477         self.assertIn('my-tag-key2:my-tag-value2', msg.response.tags)
 478         self.assertIn('my-empty-key', msg.response.tags)
 479         # meta tags
 480         self.assertEqual(len(msg.meta), 1)
 481         self.assertEqual(msg.meta[0].key, 'my-tag-export-name')
 482         self.assertEqual(len(msg.meta[0].value.stringVal), 3)
 483         self.assertIn('my-tag-key:my-tag-value', msg.meta[0].value.stringVal)
 484         self.assertIn('my-tag-key2:my-tag-value2', msg.meta[0].value.stringVal)
 485         # no ':' when the value is empty
 486         self.assertIn('my-empty-key', msg.meta[0].value.stringVal)
 487
 488 class TestProtobufMetaDOH(DNSDistProtobufTest):
 489
 490     _serverKey = 'server.key'
 491     _serverCert = 'server.chain'
 492     _serverName = 'tls.tests.dnsdist.org'
 493     _caCert = 'ca.pem'
 494     _tlsServerPort = pickAvailablePort()
 495     _dohWithNGHTTP2ServerPort = pickAvailablePort()
 496     _dohWithNGHTTP2BaseURL = ("https://%s:%d/dns-query" % (_serverName, _dohWithNGHTTP2ServerPort))
 497     _dohWithH2OServerPort = pickAvailablePort()
 498     _dohWithH2OBaseURL = ("https://%s:%d/dns-query" % (_serverName, _dohWithH2OServerPort))
 499     _config_template = """
 500     newServer{address="127.0.0.1:%d"}
 501     rl = newRemoteLogger('127.0.0.1:%d')
 502
 503     addTLSLocal("127.0.0.1:%s", "%s", "%s", { provider="openssl" })
 504     addDOHLocal("127.0.0.1:%d", "%s", "%s", { '/dns-query' }, { keepIncomingHeaders=true, library='nghttp2' })
 505     addDOHLocal("127.0.0.1:%d", "%s", "%s", { '/dns-query' }, { keepIncomingHeaders=true, library='h2o' })
 506
 507     local mytags = {path='doh-path', host='doh-host', ['query-string']='doh-query-string', scheme='doh-scheme', agent='doh-header:user-agent'}
 508     addAction(AllRule(), RemoteLogAction(rl, nil, {serverID='dnsdist-server-1'}, mytags))
 509     addResponseAction(AllRule(), RemoteLogResponseAction(rl, nil, false, {serverID='dnsdist-server-1'}, mytags))
 510     """
 511     _config_params = ['_testServerPort', '_protobufServerPort', '_tlsServerPort', '_serverCert', '_serverKey', '_dohWithNGHTTP2ServerPort', '_serverCert', '_serverKey', '_dohWithH2OServerPort', '_serverCert', '_serverKey']
 512
 513     def testProtobufMetaDoH(self):
 514         """
 515         Protobuf: Meta values - DoH
 516         """
 517         name = 'meta-doh.protobuf.tests.powerdns.com.'
 518         query = dns.message.make_query(name, 'A', 'IN')
 519         response = dns.message.make_response(query)
 520         rrset = dns.rrset.from_text(name,
 521                                     3600,
 522                                     dns.rdataclass.IN,
 523                                     dns.rdatatype.A,
 524                                     '127.0.0.1')
 525         response.answer.append(rrset)
 526
 527         for method in ("sendUDPQuery", "sendTCPQuery", "sendDOTQueryWrapper", "sendDOHWithNGHTTP2QueryWrapper", "sendDOHWithH2OQueryWrapper"):
 528             sender = getattr(self, method)
 529             (receivedQuery, receivedResponse) = sender(query, response)
 530
 531             self.assertTrue(receivedQuery)
 532             self.assertTrue(receivedResponse)
 533             receivedQuery.id = query.id
 534             self.assertEqual(query, receivedQuery)
 535             self.assertEqual(response, receivedResponse)
 536
 537             # let the protobuf messages the time to get there
 538             time.sleep(1)
 539
 540             # check the protobuf message corresponding to the query
 541             msg = self.getFirstProtobufMessage()
 542
 543             if method == "sendUDPQuery":
 544                 pbMessageType = dnsmessage_pb2.PBDNSMessage.UDP
 545             elif method == "sendTCPQuery":
 546                 pbMessageType = dnsmessage_pb2.PBDNSMessage.TCP
 547             elif method == "sendDOTQueryWrapper":
 548                 pbMessageType = dnsmessage_pb2.PBDNSMessage.DOT
 549             elif method == "sendDOHWithNGHTTP2QueryWrapper" or method == "sendDOHWithH2OQueryWrapper":
 550                 pbMessageType = dnsmessage_pb2.PBDNSMessage.DOH
 551
 552             self.checkProtobufQuery(msg, pbMessageType, query, dns.rdataclass.IN, dns.rdatatype.A, name)
 553             self.assertEqual(len(msg.meta), 5)
 554             tags = {}
 555             for entry in msg.meta:
 556                 self.assertEqual(len(entry.value.stringVal), 1)
 557                 tags[entry.key] = entry.value.stringVal[0]
 558
 559             self.assertIn('agent', tags)
 560             if method == "sendDOHWithNGHTTP2QueryWrapper" or method == "sendDOHWithH2OQueryWrapper":
 561                 self.assertIn('PycURL', tags['agent'])
 562                 self.assertIn('host', tags)
 563                 if method == "sendDOHWithNGHTTP2QueryWrapper":
 564                     self.assertEqual(tags['host'], self._serverName + ':' + str(self._dohWithNGHTTP2ServerPort))
 565                 elif method == "sendDOHWithH2OQueryWrapper":
 566                     self.assertEqual(tags['host'], self._serverName + ':' + str(self._dohWithH2OServerPort))
 567                 self.assertIn('path', tags)
 568                 self.assertEqual(tags['path'], '/dns-query')
 569                 self.assertIn('query-string', tags)
 570                 self.assertIn('?dns=', tags['query-string'])
 571                 self.assertIn('scheme', tags)
 572                 self.assertEqual(tags['scheme'], 'https')
 573
 574             # check the protobuf message corresponding to the response
 575             msg = self.getFirstProtobufMessage()
 576             self.checkProtobufResponse(msg, pbMessageType, response)
 577             self.assertEqual(len(msg.meta), 5)
 578             tags = {}
 579             for entry in msg.meta:
 580                 self.assertEqual(len(entry.value.stringVal), 1)
 581                 tags[entry.key] = entry.value.stringVal[0]
 582
 583             self.assertIn('agent', tags)
 584             if method == "sendDOHWithNGHTTP2QueryWrapper" or method == "sendDOHWithH2OQueryWrapper":
 585                 self.assertIn('PycURL', tags['agent'])
 586                 self.assertIn('host', tags)
 587                 if method == "sendDOHWithNGHTTP2QueryWrapper":
 588                     self.assertEqual(tags['host'], self._serverName + ':' + str(self._dohWithNGHTTP2ServerPort))
 589                 elif method == "sendDOHWithH2OQueryWrapper":
 590                     self.assertEqual(tags['host'], self._serverName + ':' + str(self._dohWithH2OServerPort))
 591                 self.assertIn('path', tags)
 592                 self.assertEqual(tags['path'], '/dns-query')
 593                 self.assertIn('query-string', tags)
 594                 self.assertIn('?dns=', tags['query-string'])
 595                 self.assertIn('scheme', tags)
 596                 self.assertEqual(tags['scheme'], 'https')
 597
 598 class TestProtobufMetaProxy(DNSDistProtobufTest):
 599
 600     _config_params = ['_testServerPort', '_protobufServerPort']
 601     _config_template = """
 602     setProxyProtocolACL( { "127.0.0.1/32" } )
 603
 604     newServer{address="127.0.0.1:%d"}
 605     rl = newRemoteLogger('127.0.0.1:%d')
 606
 607     local mytags = {pp='proxy-protocol-values', pp42='proxy-protocol-value:42'}
 608     addAction(AllRule(), RemoteLogAction(rl, nil, {serverID='dnsdist-server-1'}, mytags))
 609
 610     -- proxy protocol values are NOT passed to the response
 611     """
 612
 613     def testProtobufMetaProxy(self):
 614         """
 615         Protobuf: Meta values - Proxy
 616         """
 617         name = 'meta-proxy.protobuf.tests.powerdns.com.'
 618         query = dns.message.make_query(name, 'A', 'IN')
 619         response = dns.message.make_response(query)
 620         rrset = dns.rrset.from_text(name,
 621                                     3600,
 622                                     dns.rdataclass.IN,
 623                                     dns.rdatatype.A,
 624                                     '127.0.0.1')
 625         response.answer.append(rrset)
 626
 627         destAddr = "2001:db8::9"
 628         destPort = 9999
 629         srcAddr = "2001:db8::8"
 630         srcPort = 8888
 631         udpPayload = ProxyProtocol.getPayload(False, False, True, srcAddr, destAddr, srcPort, destPort, [ [ 2, b'foo'], [ 42, b'proxy'] ])
 632         (receivedQuery, receivedResponse) = self.sendUDPQuery(udpPayload + query.to_wire(), response, rawQuery=True)
 633
 634         self.assertTrue(receivedQuery)
 635         self.assertTrue(receivedResponse)
 636         receivedQuery.id = query.id
 637         self.assertEqual(query, receivedQuery)
 638         self.assertEqual(response, receivedResponse)
 639
 640         # let the protobuf messages the time to get there
 641         time.sleep(1)
 642
 643         # check the protobuf message corresponding to the UDP query
 644         msg = self.getFirstProtobufMessage()
 645
 646         self.checkProtobufQuery(msg, dnsmessage_pb2.PBDNSMessage.UDP, query, dns.rdataclass.IN, dns.rdatatype.A, name, initiator='2001:db8::8', v6=True)
 647         self.assertEqual(len(msg.meta), 2)
 648         tags = {}
 649         for entry in msg.meta:
 650             tags[entry.key] = entry.value.stringVal
 651
 652         self.assertIn('pp42', tags)
 653         self.assertEqual(tags['pp42'], ['proxy'])
 654         self.assertIn('pp', tags)
 655         self.assertEqual(len(tags['pp']), 2)
 656         self.assertIn('2:foo', tags['pp'])
 657         self.assertIn('42:proxy', tags['pp'])
 658
 659 class TestProtobufIPCipher(DNSDistProtobufTest):
 660     _config_params = ['_testServerPort', '_protobufServerPort', '_protobufServerID', '_protobufServerID']
 661     _config_template = """
 662     newServer{address="127.0.0.1:%s", useClientSubnet=true}
 663     key = makeIPCipherKey("some 16-byte key")
 664     rl = newRemoteLogger('127.0.0.1:%s')
 665     addAction(AllRule(), RemoteLogAction(rl, nil, {serverID='%s', ipEncryptKey=key})) -- Send protobuf message before lookup
 666     addResponseAction(AllRule(), RemoteLogResponseAction(rl, nil, true, {serverID='%s', ipEncryptKey=key})) -- Send protobuf message after lookup
 667
 668     """
 669
 670     def testProtobuf(self):
 671         """
 672         Protobuf: Send data to a protobuf server, with pseudonymization
 673         """
 674         name = 'query.protobuf-ipcipher.tests.powerdns.com.'
 675
 676         target = 'target.protobuf-ipcipher.tests.powerdns.com.'
 677         query = dns.message.make_query(name, 'A', 'IN')
 678         response = dns.message.make_response(query)
 679
 680         rrset = dns.rrset.from_text(name,
 681                                     3600,
 682                                     dns.rdataclass.IN,
 683                                     dns.rdatatype.CNAME,
 684                                     target)
 685         response.answer.append(rrset)
 686
 687         rrset = dns.rrset.from_text(target,
 688                                     3600,
 689                                     dns.rdataclass.IN,
 690                                     dns.rdatatype.A,
 691                                     '127.0.0.1')
 692         response.answer.append(rrset)
 693
 694         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 695         self.assertTrue(receivedQuery)
 696         self.assertTrue(receivedResponse)
 697         receivedQuery.id = query.id
 698         self.assertEqual(query, receivedQuery)
 699         self.assertEqual(response, receivedResponse)
 700
 701         # let the protobuf messages the time to get there
 702         time.sleep(1)
 703
 704         # check the protobuf message corresponding to the UDP query
 705         msg = self.getFirstProtobufMessage()
 706
 707         # 108.41.239.98 is 127.0.0.1 pseudonymized with ipcipher and the current key
 708         self.checkProtobufQuery(msg, dnsmessage_pb2.PBDNSMessage.UDP, query, dns.rdataclass.IN, dns.rdatatype.A, name, '108.41.239.98')
 709
 710         # check the protobuf message corresponding to the UDP response
 711         msg = self.getFirstProtobufMessage()
 712         self.checkProtobufResponse(msg, dnsmessage_pb2.PBDNSMessage.UDP, response, '108.41.239.98')
 713
 714         self.assertEqual(len(msg.response.rrs), 2)
 715         rr = msg.response.rrs[0]
 716         self.checkProtobufResponseRecord(rr, dns.rdataclass.IN, dns.rdatatype.CNAME, name, 3600)
 717         self.assertEqual(rr.rdata.decode('ascii'), target)
 718         rr = msg.response.rrs[1]
 719         self.checkProtobufResponseRecord(rr, dns.rdataclass.IN, dns.rdatatype.A, target, 3600)
 720         self.assertEqual(socket.inet_ntop(socket.AF_INET, rr.rdata), '127.0.0.1')
 721
 722         (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response)
 723         self.assertTrue(receivedQuery)
 724         self.assertTrue(receivedResponse)
 725         receivedQuery.id = query.id
 726         self.assertEqual(query, receivedQuery)
 727         self.assertEqual(response, receivedResponse)
 728
 729         # let the protobuf messages the time to get there
 730         time.sleep(1)
 731
 732         # check the protobuf message corresponding to the TCP query
 733         msg = self.getFirstProtobufMessage()
 734         # 108.41.239.98 is 127.0.0.1 pseudonymized with ipcipher and the current key
 735         self.checkProtobufQuery(msg, dnsmessage_pb2.PBDNSMessage.TCP, query, dns.rdataclass.IN, dns.rdatatype.A, name, '108.41.239.98')
 736
 737         # check the protobuf message corresponding to the TCP response
 738         msg = self.getFirstProtobufMessage()
 739         self.checkProtobufResponse(msg, dnsmessage_pb2.PBDNSMessage.TCP, response, '108.41.239.98')
 740         self.assertEqual(len(msg.response.rrs), 2)
 741         rr = msg.response.rrs[0]
 742         self.checkProtobufResponseRecord(rr, dns.rdataclass.IN, dns.rdatatype.CNAME, name, 3600)
 743         self.assertEqual(rr.rdata.decode('ascii'), target)
 744         rr = msg.response.rrs[1]
 745         self.checkProtobufResponseRecord(rr, dns.rdataclass.IN, dns.rdatatype.A, target, 3600)
 746         self.assertEqual(socket.inet_ntop(socket.AF_INET, rr.rdata), '127.0.0.1')