2 * This file is part of PowerDNS or dnsdist.
3 * Copyright -- PowerDNS.COM B.V. and its contributors
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of version 2 of the GNU General Public License as
7 * published by the Free Software Foundation.
9 * In addition, for the avoidance of any doubt, permission is granted to
10 * link this program with OpenSSL and to (re)distribute the binaries
11 * produced as the result of such linking.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23 #include "ext/lmdb-safe/lmdb-safe.hh"
30 #include "pdns/utility.hh"
31 #include "pdns/dnsbackend.hh"
32 #include "pdns/dns.hh"
33 #include "pdns/dnspacket.hh"
34 #include "pdns/base32.hh"
35 #include "pdns/dnssecinfra.hh"
36 #include "pdns/pdnsexception.hh"
37 #include "pdns/logger.hh"
38 #include "pdns/version.hh"
39 #include "pdns/arguments.hh"
40 #include "pdns/lock.hh"
41 #include "pdns/uuid-utils.hh"
42 #include <boost/archive/binary_oarchive.hpp>
43 #include <boost/archive/binary_iarchive.hpp>
44 #include <boost/serialization/vector.hpp>
45 #include <boost/serialization/string.hpp>
46 #include <boost/serialization/utility.hpp>
47 #include <boost/uuid/uuid_serialize.hpp>
49 #include <boost/iostreams/device/back_inserter.hpp>
54 #include "lmdbbackend.hh"
56 #define SCHEMAVERSION 5
58 // List the class version here. Default is 0
59 BOOST_CLASS_VERSION(LMDBBackend::KeyDataDB
, 1)
60 BOOST_CLASS_VERSION(DomainInfo
, 1)
62 static bool s_first
= true;
63 static int s_shards
= 0;
64 static std::mutex s_lmdbStartupLock
;
66 std::pair
<uint32_t, uint32_t> LMDBBackend::getSchemaVersionAndShards(std::string
& filename
)
68 // cerr << "getting schema version for path " << filename << endl;
70 uint32_t schemaversion
;
73 MDB_env
* env
= nullptr;
75 if ((rc
= mdb_env_create(&env
)) != 0) {
76 throw std::runtime_error("mdb_env_create failed");
79 if ((rc
= mdb_env_set_mapsize(env
, 0)) != 0) {
80 throw std::runtime_error("mdb_env_set_mapsize failed");
83 if ((rc
= mdb_env_set_maxdbs(env
, 20)) != 0) { // we need 17: 1 {"pdns"} + 4 {"domains", "keydata", "tsig", "metadata"} * 2 {v4, v5} * 2 {main, index in _0}
85 throw std::runtime_error("mdb_env_set_maxdbs failed");
88 if ((rc
= mdb_env_open(env
, filename
.c_str(), MDB_NOSUBDIR
| MDB_RDONLY
, 0600)) != 0) {
90 // we don't have a database yet! report schema 0, with 0 shards
94 throw std::runtime_error("mdb_env_open failed");
97 MDB_txn
* txn
= nullptr;
99 if ((rc
= mdb_txn_begin(env
, NULL
, MDB_RDONLY
, &txn
)) != 0) {
101 throw std::runtime_error("mdb_txn_begin failed");
106 if ((rc
= mdb_dbi_open(txn
, "pdns", 0, &dbi
)) != 0) {
107 if (rc
== MDB_NOTFOUND
) {
108 // this means nothing has been inited yet
109 // we pretend this means 5
116 throw std::runtime_error("mdb_dbi_open failed");
121 key
.mv_data
= (char*)"schemaversion";
122 key
.mv_size
= strlen((char*)key
.mv_data
);
124 if ((rc
= mdb_get(txn
, dbi
, &key
, &data
)) != 0) {
125 if (rc
== MDB_NOTFOUND
) {
126 // this means nothing has been inited yet
127 // we pretend this means 5
133 throw std::runtime_error("mdb_get pdns.schemaversion failed");
136 if (data
.mv_size
== 4) {
137 // schemaversion is < 5 and is stored in 32 bits, in host order
139 memcpy(&schemaversion
, data
.mv_data
, data
.mv_size
);
141 else if (data
.mv_size
>= LMDBLS::LS_MIN_HEADER_SIZE
+ sizeof(schemaversion
)) {
142 // schemaversion presumably is 5, stored in 32 bits, network order, after the LS header
144 // FIXME: get actual header size (including extension blocks) instead of just reading from the back
145 // FIXME: add a test for reading schemaversion and shards (and actual data, later) when there are variably sized headers
146 memcpy(&schemaversion
, (char*)data
.mv_data
+ data
.mv_size
- sizeof(schemaversion
), sizeof(schemaversion
));
147 schemaversion
= ntohl(schemaversion
);
150 throw std::runtime_error("pdns.schemaversion had unexpected size");
155 key
.mv_data
= (char*)"shards";
156 key
.mv_size
= strlen((char*)key
.mv_data
);
158 if ((rc
= mdb_get(txn
, dbi
, &key
, &data
)) != 0) {
159 if (rc
== MDB_NOTFOUND
) {
160 cerr
<< "schemaversion was set, but shards was not. Dazed and confused, trying to exit." << endl
;
166 throw std::runtime_error("mdb_get pdns.shards failed");
169 if (data
.mv_size
== 4) {
170 // 'shards' is stored in 32 bits, in host order
172 memcpy(&shards
, data
.mv_data
, data
.mv_size
);
174 else if (data
.mv_size
>= LMDBLS::LS_MIN_HEADER_SIZE
+ sizeof(shards
)) {
175 // FIXME: get actual header size (including extension blocks) instead of just reading from the back
176 memcpy(&shards
, (char*)data
.mv_data
+ data
.mv_size
- sizeof(shards
), sizeof(shards
));
177 shards
= ntohl(shards
);
180 throw std::runtime_error("pdns.shards had unexpected size");
186 return {schemaversion
, shards
};
191 // copy sdbi to tdbi, prepending an empty LS header (24 bytes of '\0') to all values
192 void copyDBIAndAddLSHeader(MDB_txn
* txn
, MDB_dbi sdbi
, MDB_dbi tdbi
)
194 // FIXME: clear out target dbi first
196 std::string
header(LMDBLS::LS_MIN_HEADER_SIZE
, '\0');
201 if ((rc
= mdb_cursor_open(txn
, sdbi
, &cur
)) != 0) {
202 throw std::runtime_error("mdb_cursur_open failed");
207 rc
= mdb_cursor_get(cur
, &key
, &data
, MDB_FIRST
);
210 std::string
skey(reinterpret_cast<const char*>(key
.mv_data
), key
.mv_size
);
211 std::string
sdata(reinterpret_cast<const char*>(data
.mv_data
), data
.mv_size
);
213 std::string stdata
= header
+ sdata
;
215 // cerr<<"got key="<<makeHexDump(skey)<<", data="<<makeHexDump(sdata)<<", sdata="<<makeHexDump(stdata)<<endl;
220 tkey
.mv_data
= const_cast<char*>(skey
.c_str());
221 tkey
.mv_size
= skey
.size();
222 tdata
.mv_data
= const_cast<char*>(stdata
.c_str());
223 tdata
.mv_size
= stdata
.size();
225 if ((rc
= mdb_put(txn
, tdbi
, &tkey
, &tdata
, 0)) != 0) {
226 throw std::runtime_error("mdb_put failed");
229 rc
= mdb_cursor_get(cur
, &key
, &data
, MDB_NEXT
);
231 if (rc
!= MDB_NOTFOUND
) {
232 cerr
<< "rc=" << rc
<< endl
;
233 throw std::runtime_error("error while iterating dbi");
237 // migrated a typed DBI:
238 // 1. change keys (uint32_t) from host to network order
239 // 2. prepend empty LS header to values
240 void copyTypedDBI(MDB_txn
* txn
, MDB_dbi sdbi
, MDB_dbi tdbi
)
242 // FIXME: clear out target dbi first
244 std::string
header(LMDBLS::LS_MIN_HEADER_SIZE
, '\0');
249 if ((rc
= mdb_cursor_open(txn
, sdbi
, &cur
)) != 0) {
250 throw std::runtime_error("mdb_cursur_open failed");
255 rc
= mdb_cursor_get(cur
, &key
, &data
, MDB_FIRST
);
258 // std::string skey((char*) key.mv_data, key.mv_size);
259 std::string
sdata(reinterpret_cast<const char*>(data
.mv_data
), data
.mv_size
);
261 std::string stdata
= header
+ sdata
;
265 if (key
.mv_size
!= sizeof(uint32_t)) {
266 throw std::runtime_error("got non-uint32_t key in TypedDBI");
269 memcpy(&id
, key
.mv_data
, sizeof(uint32_t));
273 // cerr<<"got key="<<makeHexDump(skey)<<", data="<<makeHexDump(sdata)<<", sdata="<<makeHexDump(stdata)<<endl;
278 tkey
.mv_data
= reinterpret_cast<char*>(&id
);
279 tkey
.mv_size
= sizeof(uint32_t);
280 tdata
.mv_data
= const_cast<char*>(stdata
.c_str());
281 tdata
.mv_size
= stdata
.size();
283 if ((rc
= mdb_put(txn
, tdbi
, &tkey
, &tdata
, 0)) != 0) {
284 throw std::runtime_error("mdb_put failed");
287 rc
= mdb_cursor_get(cur
, &key
, &data
, MDB_NEXT
);
289 if (rc
!= MDB_NOTFOUND
) {
290 cerr
<< "rc=" << rc
<< endl
;
291 throw std::runtime_error("error while iterating dbi");
295 // migrating an index DBI:
296 // newkey = oldkey.len(), oldkey, htonl(oldvalue)
297 // newvalue = empty lsheader
298 void copyIndexDBI(MDB_txn
* txn
, MDB_dbi sdbi
, MDB_dbi tdbi
)
300 // FIXME: clear out target dbi first
302 std::string
header(LMDBLS::LS_MIN_HEADER_SIZE
, '\0');
307 if ((rc
= mdb_cursor_open(txn
, sdbi
, &cur
)) != 0) {
308 throw std::runtime_error("mdb_cursur_open failed");
313 rc
= mdb_cursor_get(cur
, &key
, &data
, MDB_FIRST
);
316 std::string
lenprefix(sizeof(uint16_t), '\0');
317 std::string
skey((char*)key
.mv_data
, key
.mv_size
);
321 if (data
.mv_size
!= sizeof(uint32_t)) {
322 throw std::runtime_error("got non-uint32_t ID value in IndexDBI");
325 memcpy((void*)&id
, data
.mv_data
, sizeof(uint32_t));
328 uint16_t len
= htons(skey
.size());
329 memcpy((void*)lenprefix
.data(), &len
, sizeof(len
));
330 std::string stkey
= lenprefix
+ skey
+ std::string((char*)&id
, sizeof(uint32_t));
335 tkey
.mv_data
= (char*)stkey
.c_str();
336 tkey
.mv_size
= stkey
.size();
337 tdata
.mv_data
= (char*)header
.c_str();
338 tdata
.mv_size
= header
.size();
340 if ((rc
= mdb_put(txn
, tdbi
, &tkey
, &tdata
, 0)) != 0) {
341 throw std::runtime_error("mdb_put failed");
344 rc
= mdb_cursor_get(cur
, &key
, &data
, MDB_NEXT
);
346 if (rc
!= MDB_NOTFOUND
) {
347 throw std::runtime_error("error while iterating dbi");
353 bool LMDBBackend::upgradeToSchemav5(std::string
& filename
)
357 auto currentSchemaVersionAndShards
= getSchemaVersionAndShards(filename
);
358 uint32_t currentSchemaVersion
= currentSchemaVersionAndShards
.first
;
359 uint32_t shards
= currentSchemaVersionAndShards
.second
;
361 if (currentSchemaVersion
!= 3 && currentSchemaVersion
!= 4) {
362 throw std::runtime_error("upgrade to v5 requested but current schema is not v3 or v4, stopping");
365 MDB_env
* env
= nullptr;
367 if ((rc
= mdb_env_create(&env
)) != 0) {
368 throw std::runtime_error("mdb_env_create failed");
371 if ((rc
= mdb_env_set_maxdbs(env
, 20)) != 0) {
373 throw std::runtime_error("mdb_env_set_maxdbs failed");
376 if ((rc
= mdb_env_open(env
, filename
.c_str(), MDB_NOSUBDIR
, 0600)) != 0) {
378 throw std::runtime_error("mdb_env_open failed");
381 MDB_txn
* txn
= nullptr;
383 if ((rc
= mdb_txn_begin(env
, NULL
, 0, &txn
)) != 0) {
385 throw std::runtime_error("mdb_txn_begin failed");
388 std::cerr
<< "migrating shards" << std::endl
;
389 for (uint32_t i
= 0; i
< shards
; i
++) {
390 string shardfile
= filename
+ "-" + std::to_string(i
);
391 if (access(shardfile
.c_str(), F_OK
) < 0) {
392 if (errno
== ENOENT
) {
393 // apparently this shard doesn't exist yet, moving on
394 std::cerr
<< "shard " << shardfile
<< " not found, continuing" << std::endl
;
399 std::cerr
<< "migrating shard " << shardfile
<< std::endl
;
400 MDB_env
* shenv
= nullptr;
402 if ((rc
= mdb_env_create(&shenv
)) != 0) {
403 throw std::runtime_error("mdb_env_create failed");
406 if ((rc
= mdb_env_set_maxdbs(shenv
, 8)) != 0) {
408 throw std::runtime_error("mdb_env_set_maxdbs failed");
411 if ((rc
= mdb_env_open(shenv
, shardfile
.c_str(), MDB_NOSUBDIR
, 0600)) != 0) {
413 throw std::runtime_error("mdb_env_open failed");
416 MDB_txn
* shtxn
= nullptr;
418 if ((rc
= mdb_txn_begin(shenv
, NULL
, 0, &shtxn
)) != 0) {
420 throw std::runtime_error("mdb_txn_begin failed");
425 if ((rc
= mdb_dbi_open(shtxn
, "records", 0, &shdbi
)) != 0) {
426 if (rc
== MDB_NOTFOUND
) {
427 mdb_txn_abort(shtxn
);
428 mdb_env_close(shenv
);
431 mdb_txn_abort(shtxn
);
432 mdb_env_close(shenv
);
433 throw std::runtime_error("mdb_dbi_open shard records failed");
438 if ((rc
= mdb_dbi_open(shtxn
, "records_v5", MDB_CREATE
, &shdbi2
)) != 0) {
439 mdb_dbi_close(shenv
, shdbi
);
440 mdb_txn_abort(shtxn
);
441 mdb_env_close(shenv
);
442 throw std::runtime_error("mdb_dbi_open shard records_v5 failed");
446 copyDBIAndAddLSHeader(shtxn
, shdbi
, shdbi2
);
448 catch (std::exception
& e
) {
449 mdb_dbi_close(shenv
, shdbi2
);
450 mdb_dbi_close(shenv
, shdbi
);
451 mdb_txn_abort(shtxn
);
452 mdb_env_close(shenv
);
453 throw std::runtime_error("copyDBIAndAddLSHeader failed");
456 cerr
<< "shard mbd_drop=" << mdb_drop(shtxn
, shdbi
, 1) << endl
;
457 mdb_txn_commit(shtxn
);
458 mdb_dbi_close(shenv
, shdbi2
);
459 mdb_env_close(shenv
);
462 std::array
<MDB_dbi
, 4> fromtypeddbi
;
463 std::array
<MDB_dbi
, 4> totypeddbi
;
467 for (const std::string dbname
: {"domains", "keydata", "tsig", "metadata"}) {
468 std::cerr
<< "migrating " << dbname
<< std::endl
;
469 std::string tdbname
= dbname
+ "_v5";
471 if ((rc
= mdb_dbi_open(txn
, dbname
.c_str(), 0, &fromtypeddbi
[index
])) != 0) {
474 throw std::runtime_error("mdb_dbi_open typeddbi failed");
477 if ((rc
= mdb_dbi_open(txn
, tdbname
.c_str(), MDB_CREATE
, &totypeddbi
[index
])) != 0) {
478 mdb_dbi_close(env
, fromtypeddbi
[index
]);
481 throw std::runtime_error("mdb_dbi_open typeddbi target failed");
485 copyTypedDBI(txn
, fromtypeddbi
[index
], totypeddbi
[index
]);
487 catch (std::exception
& e
) {
488 mdb_dbi_close(env
, totypeddbi
[index
]);
489 mdb_dbi_close(env
, fromtypeddbi
[index
]);
492 throw std::runtime_error("copyTypedDBI failed");
495 // mdb_dbi_close(env, dbi2);
496 // mdb_dbi_close(env, dbi);
497 std::cerr
<< "migrated " << dbname
<< std::endl
;
502 std::array
<MDB_dbi
, 4> fromindexdbi
;
503 std::array
<MDB_dbi
, 4> toindexdbi
;
507 for (const std::string dbname
: {"domains", "keydata", "tsig", "metadata"}) {
508 std::string fdbname
= dbname
+ "_0";
509 std::cerr
<< "migrating " << dbname
<< std::endl
;
510 std::string tdbname
= dbname
+ "_v5_0";
512 if ((rc
= mdb_dbi_open(txn
, fdbname
.c_str(), 0, &fromindexdbi
[index
])) != 0) {
515 throw std::runtime_error("mdb_dbi_open indexdbi failed");
518 if ((rc
= mdb_dbi_open(txn
, tdbname
.c_str(), MDB_CREATE
, &toindexdbi
[index
])) != 0) {
519 mdb_dbi_close(env
, fromindexdbi
[index
]);
522 throw std::runtime_error("mdb_dbi_open indexdbi target failed");
526 copyIndexDBI(txn
, fromindexdbi
[index
], toindexdbi
[index
]);
528 catch (std::exception
& e
) {
529 mdb_dbi_close(env
, toindexdbi
[index
]);
530 mdb_dbi_close(env
, fromindexdbi
[index
]);
533 throw std::runtime_error("copyIndexDBI failed");
536 // mdb_dbi_close(env, dbi2);
537 // mdb_dbi_close(env, dbi);
538 std::cerr
<< "migrated " << dbname
<< std::endl
;
545 // finally, migrate the pdns db
546 if ((rc
= mdb_dbi_open(txn
, "pdns", 0, &dbi
)) != 0) {
549 throw std::runtime_error("mdb_dbi_open pdns failed");
554 std::string
header(LMDBLS::LS_MIN_HEADER_SIZE
, '\0');
556 for (const std::string keyname
: {"schemaversion", "shards"}) {
557 cerr
<< "migrating pdns." << keyname
<< endl
;
559 key
.mv_data
= (char*)keyname
.c_str();
560 key
.mv_size
= keyname
.size();
562 if ((rc
= mdb_get(txn
, dbi
, &key
, &data
))) {
563 throw std::runtime_error("mdb_get pdns.shards failed");
568 if (data
.mv_size
!= sizeof(uint32_t)) {
569 throw std::runtime_error("got non-uint32_t key");
572 memcpy((void*)&value
, data
.mv_data
, sizeof(uint32_t));
574 value
= htonl(value
);
575 if (keyname
== "schemaversion") {
579 std::string
sdata((char*)data
.mv_data
, data
.mv_size
);
581 std::string stdata
= header
+ std::string((char*)&value
, sizeof(uint32_t));
586 tdata
.mv_data
= (char*)stdata
.c_str();
587 tdata
.mv_size
= stdata
.size();
589 if ((rc
= mdb_put(txn
, dbi
, &key
, &tdata
, 0)) != 0) {
590 throw std::runtime_error("mdb_put failed");
594 for (const std::string keyname
: {"uuid"}) {
595 cerr
<< "migrating pdns." << keyname
<< endl
;
597 key
.mv_data
= (char*)keyname
.c_str();
598 key
.mv_size
= keyname
.size();
600 if ((rc
= mdb_get(txn
, dbi
, &key
, &data
))) {
601 throw std::runtime_error("mdb_get pdns.shards failed");
604 std::string
sdata((char*)data
.mv_data
, data
.mv_size
);
606 std::string stdata
= header
+ sdata
;
610 tdata
.mv_data
= (char*)stdata
.c_str();
611 tdata
.mv_size
= stdata
.size();
613 if ((rc
= mdb_put(txn
, dbi
, &key
, &tdata
, 0)) != 0) {
614 throw std::runtime_error("mdb_put failed");
618 for (int i
= 0; i
< 4; i
++) {
619 mdb_drop(txn
, fromtypeddbi
[i
], 1);
620 mdb_drop(txn
, fromindexdbi
[i
], 1);
623 cerr
<< "txn commit=" << mdb_txn_commit(txn
) << endl
;
625 for (int i
= 0; i
< 4; i
++) {
626 mdb_dbi_close(env
, totypeddbi
[i
]);
627 mdb_dbi_close(env
, toindexdbi
[i
]);
631 // throw std::runtime_error("migration done");
632 cerr
<< "migration done" << endl
;
637 LMDBBackend::LMDBBackend(const std::string
& suffix
)
639 // overlapping domain ids in combination with relative names are a recipe for disaster
640 if (!suffix
.empty()) {
641 throw std::runtime_error("LMDB backend does not support multiple instances");
644 setArgPrefix("lmdb" + suffix
);
646 string syncMode
= toLower(getArg("sync-mode"));
648 d_random_ids
= mustDo("random-ids");
650 if (syncMode
== "nosync")
651 d_asyncFlag
= MDB_NOSYNC
;
652 else if (syncMode
== "nometasync")
653 d_asyncFlag
= MDB_NOMETASYNC
;
654 else if (syncMode
== "mapasync")
655 d_asyncFlag
= MDB_MAPASYNC
;
656 else if (syncMode
.empty() || syncMode
== "sync")
659 throw std::runtime_error("Unknown sync mode " + syncMode
+ " requested for LMDB backend");
661 uint64_t mapSize
= 0;
663 mapSize
= std::stoll(getArg("map-size"));
665 catch (const std::exception
& e
) {
666 throw std::runtime_error(std::string("Unable to parse the 'map-size' LMDB value: ") + e
.what());
669 LMDBLS::s_flag_deleted
= mustDo("flag-deleted");
670 d_handle_dups
= false;
672 if (mustDo("lightning-stream")) {
674 d_handle_dups
= true;
675 LMDBLS::s_flag_deleted
= true;
677 if (atoi(getArg("shards").c_str()) != 1) {
678 throw std::runtime_error(std::string("running with Lightning Stream support requires shards=1"));
685 std::lock_guard
<std::mutex
> l(s_lmdbStartupLock
);
687 auto filename
= getArg("filename");
689 auto currentSchemaVersionAndShards
= getSchemaVersionAndShards(filename
);
690 uint32_t currentSchemaVersion
= currentSchemaVersionAndShards
.first
;
691 // std::cerr<<"current schema version: "<<currentSchemaVersion<<", shards="<<currentSchemaVersionAndShards.second<<std::endl;
693 if (getArgAsNum("schema-version") != SCHEMAVERSION
) {
694 throw std::runtime_error("This version of the lmdbbackend only supports schema version 5. Configuration demands a lower version. Not starting up.");
697 if (currentSchemaVersion
> 0 && currentSchemaVersion
< 3) {
698 throw std::runtime_error("this version of the lmdbbackend can only upgrade from schema v3/v4 to v5. Upgrading from older schemas is not yet supported.");
701 if (currentSchemaVersion
== 0) {
702 // no database is present yet, we can just create them
703 currentSchemaVersion
= 5;
706 if (currentSchemaVersion
== 3 || currentSchemaVersion
== 4) {
707 if (!upgradeToSchemav5(filename
)) {
708 throw std::runtime_error("Failed to perform LMDB schema version upgrade from v4 to v5");
710 currentSchemaVersion
= 5;
713 if (currentSchemaVersion
!= 5) {
714 throw std::runtime_error("Somehow, we are not at schema version 5. Giving up");
717 d_tdomains
= std::make_shared
<tdomains_t
>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR
| d_asyncFlag
, 0600, mapSize
), "domains_v5");
718 d_tmeta
= std::make_shared
<tmeta_t
>(d_tdomains
->getEnv(), "metadata_v5");
719 d_tkdb
= std::make_shared
<tkdb_t
>(d_tdomains
->getEnv(), "keydata_v5");
720 d_ttsig
= std::make_shared
<ttsig_t
>(d_tdomains
->getEnv(), "tsig_v5");
722 auto pdnsdbi
= d_tdomains
->getEnv()->openDB("pdns", MDB_CREATE
);
726 auto txn
= d_tdomains
->getEnv()->getRWTransaction();
729 if (!txn
->get(pdnsdbi
, "shards", shards
)) {
730 s_shards
= shards
.get
<uint32_t>();
732 if (mustDo("lightning-stream") && s_shards
!= 1) {
733 throw std::runtime_error(std::string("running with Lightning Stream support enabled requires a database with exactly 1 shard"));
736 if (s_shards
!= atoi(getArg("shards").c_str())) {
737 g_log
<< Logger::Warning
<< "Note: configured number of lmdb shards (" << atoi(getArg("shards").c_str()) << ") is different from on-disk (" << s_shards
<< "). Using on-disk shard number" << endl
;
741 s_shards
= atoi(getArg("shards").c_str());
742 txn
->put(pdnsdbi
, "shards", s_shards
);
746 if (txn
->get(pdnsdbi
, "uuid", gotuuid
)) {
747 const auto uuid
= getUniqueID();
748 const string
uuids(uuid
.begin(), uuid
.end());
749 txn
->put(pdnsdbi
, "uuid", uuids
);
752 MDBOutVal _schemaversion
;
753 if (txn
->get(pdnsdbi
, "schemaversion", _schemaversion
)) {
754 // our DB is entirely new, so we need to write the schemaversion
755 txn
->put(pdnsdbi
, "schemaversion", currentSchemaVersion
);
764 d_tdomains
= std::make_shared
<tdomains_t
>(getMDBEnv(getArg("filename").c_str(), MDB_NOSUBDIR
| d_asyncFlag
, 0600, mapSize
), "domains_v5");
765 d_tmeta
= std::make_shared
<tmeta_t
>(d_tdomains
->getEnv(), "metadata_v5");
766 d_tkdb
= std::make_shared
<tkdb_t
>(d_tdomains
->getEnv(), "keydata_v5");
767 d_ttsig
= std::make_shared
<ttsig_t
>(d_tdomains
->getEnv(), "tsig_v5");
769 d_trecords
.resize(s_shards
);
770 d_dolog
= ::arg().mustDo("query-logging");
775 namespace serialization
778 template <class Archive
>
779 void save(Archive
& ar
, const DNSName
& g
, const unsigned int /* version */)
785 ar
& g
.toDNSStringLC();
789 template <class Archive
>
790 void load(Archive
& ar
, DNSName
& g
, const unsigned int /* version */)
798 g
= DNSName(tmp
.c_str(), tmp
.size(), 0, false);
802 template <class Archive
>
803 void save(Archive
& ar
, const QType
& g
, const unsigned int /* version */)
808 template <class Archive
>
809 void load(Archive
& ar
, QType
& g
, const unsigned int /* version */)
816 template <class Archive
>
817 void save(Archive
& ar
, const DomainInfo
& g
, const unsigned int /* version */)
824 ar
& g
.notified_serial
;
830 template <class Archive
>
831 void load(Archive
& ar
, DomainInfo
& g
, const unsigned int version
)
838 ar
& g
.notified_serial
;
850 template <class Archive
>
851 void serialize(Archive
& ar
, LMDBBackend::DomainMeta
& g
, const unsigned int /* version */)
853 ar
& g
.domain
& g
.key
& g
.value
;
856 template <class Archive
>
857 void save(Archive
& ar
, const LMDBBackend::KeyDataDB
& g
, const unsigned int /* version */)
859 ar
& g
.domain
& g
.content
& g
.flags
& g
.active
& g
.published
;
862 template <class Archive
>
863 void load(Archive
& ar
, LMDBBackend::KeyDataDB
& g
, const unsigned int version
)
865 ar
& g
.domain
& g
.content
& g
.flags
& g
.active
;
874 template <class Archive
>
875 void serialize(Archive
& ar
, TSIGKey
& g
, const unsigned int /* version */)
878 ar
& g
.algorithm
; // this is the ordername
882 } // namespace serialization
885 BOOST_SERIALIZATION_SPLIT_FREE(DNSName
);
886 BOOST_SERIALIZATION_SPLIT_FREE(QType
);
887 BOOST_SERIALIZATION_SPLIT_FREE(LMDBBackend::KeyDataDB
);
888 BOOST_SERIALIZATION_SPLIT_FREE(DomainInfo
);
889 BOOST_IS_BITWISE_SERIALIZABLE(ComboAddress
);
892 std::string
serToString(const LMDBBackend::LMDBResourceRecord
& lrr
)
895 uint16_t len
= lrr
.content
.length();
896 ret
.reserve(2 + len
+ 7);
898 ret
.assign((const char*)&len
, 2);
900 ret
.append((const char*)&lrr
.ttl
, 4);
901 ret
.append(1, (char)lrr
.auth
);
902 ret
.append(1, (char)lrr
.disabled
);
903 ret
.append(1, (char)lrr
.ordername
);
908 std::string
serToString(const vector
<LMDBBackend::LMDBResourceRecord
>& lrrs
)
911 for (const auto& lrr
: lrrs
) {
912 ret
+= serToString(lrr
);
917 static inline size_t serOneRRFromString(const string_view
& str
, LMDBBackend::LMDBResourceRecord
& lrr
)
920 memcpy(&len
, &str
[0], 2);
921 lrr
.content
.assign(&str
[2], len
); // len bytes
922 memcpy(&lrr
.ttl
, &str
[2] + len
, 4);
923 lrr
.auth
= str
[2 + len
+ 4];
924 lrr
.disabled
= str
[2 + len
+ 4 + 1];
925 lrr
.ordername
= str
[2 + len
+ 4 + 2];
926 lrr
.wildcardname
.clear();
932 void serFromString(const string_view
& str
, LMDBBackend::LMDBResourceRecord
& lrr
)
934 serOneRRFromString(str
, lrr
);
938 void serFromString(const string_view
& str
, vector
<LMDBBackend::LMDBResourceRecord
>& lrrs
)
941 while (str_copy
.size() >= 9) { // minimum length for a record is 10
942 LMDBBackend::LMDBResourceRecord lrr
;
943 auto rrLength
= serOneRRFromString(str_copy
, lrr
);
944 lrrs
.emplace_back(lrr
);
945 str_copy
.remove_prefix(rrLength
);
949 static std::string
serializeContent(uint16_t qtype
, const DNSName
& domain
, const std::string
& content
)
951 auto drc
= DNSRecordContent::make(qtype
, QClass::IN
, content
);
952 return drc
->serialize(domain
, false);
955 static std::shared_ptr
<DNSRecordContent
> deserializeContentZR(uint16_t qtype
, const DNSName
& qname
, const std::string
& content
)
957 if (qtype
== QType::A
&& content
.size() == 4) {
958 return std::make_shared
<ARecordContent
>(*((uint32_t*)content
.c_str()));
960 return DNSRecordContent::deserialize(qname
, qtype
, content
);
963 /* design. If you ask a question without a zone id, we lookup the best
964 zone id for you, and answer from that. This is different than other backends, but I can't see why it would not work.
966 The index we use is "zoneid,canonical relative name". This index is also used
969 Note - domain_id, name and type are ONLY present on the index!
972 #if BOOST_VERSION >= 106100
973 #define StringView string_view
975 #define StringView string
978 void LMDBBackend::deleteDomainRecords(RecordsRWTransaction
& txn
, uint32_t domain_id
, uint16_t qtype
)
980 compoundOrdername co
;
981 string match
= co(domain_id
);
983 auto cursor
= txn
.txn
->getCursor(txn
.db
->dbi
);
985 // cout<<"Match: "<<makeHexDump(match);
986 if (!cursor
.lower_bound(match
, key
, val
)) {
987 while (key
.getNoStripHeader
<StringView
>().rfind(match
, 0) == 0) {
988 if (qtype
== QType::ANY
|| co
.getQType(key
.getNoStripHeader
<StringView
>()) == qtype
)
990 if (cursor
.next(key
, val
))
996 /* Here's the complicated story. Other backends have just one transaction, which is either
999 You can't call feedRecord without a transaction started with startTransaction.
1001 However, other functions can be called after startTransaction() or without startTransaction()
1002 (like updateDNSSECOrderNameAndAuth)
1008 bool LMDBBackend::startTransaction(const DNSName
& domain
, int domain_id
)
1010 // cout <<"startTransaction("<<domain<<", "<<domain_id<<")"<<endl;
1011 int real_id
= domain_id
;
1013 auto rotxn
= d_tdomains
->getROTransaction();
1015 real_id
= rotxn
.get
<0>(domain
, di
);
1016 // cout<<"real_id = "<<real_id << endl;
1021 throw DBException("Attempt to start a transaction while one was open already");
1023 d_rwtxn
= getRecordsRWTransaction(real_id
);
1025 d_transactiondomain
= domain
;
1026 d_transactiondomainid
= real_id
;
1027 if (domain_id
>= 0) {
1028 deleteDomainRecords(*d_rwtxn
, domain_id
);
1034 bool LMDBBackend::commitTransaction()
1036 // cout<<"Commit transaction" <<endl;
1038 throw DBException("Attempt to commit a transaction while there isn't one open");
1041 d_rwtxn
->txn
->commit();
1046 bool LMDBBackend::abortTransaction()
1048 // cout<<"Abort transaction"<<endl;
1050 throw DBException("Attempt to abort a transaction while there isn't one open");
1053 d_rwtxn
->txn
->abort();
1059 // d_rwtxn must be set here
1060 bool LMDBBackend::feedRecord(const DNSResourceRecord
& r
, const DNSName
& ordername
, bool ordernameIsNSEC3
)
1062 LMDBResourceRecord
lrr(r
);
1063 lrr
.qname
.makeUsRelative(d_transactiondomain
);
1064 lrr
.content
= serializeContent(lrr
.qtype
.getCode(), r
.qname
, lrr
.content
);
1066 compoundOrdername co
;
1067 string matchName
= co(lrr
.domain_id
, lrr
.qname
, lrr
.qtype
.getCode());
1071 if (!d_rwtxn
->txn
->get(d_rwtxn
->db
->dbi
, matchName
, _rrs
)) {
1072 rrs
= _rrs
.get
<string
>();
1075 rrs
+= serToString(lrr
);
1077 d_rwtxn
->txn
->put(d_rwtxn
->db
->dbi
, matchName
, rrs
);
1079 if (ordernameIsNSEC3
&& !ordername
.empty()) {
1081 if (d_rwtxn
->txn
->get(d_rwtxn
->db
->dbi
, co(lrr
.domain_id
, lrr
.qname
, QType::NSEC3
), val
)) {
1083 lrr
.content
= lrr
.qname
.toDNSStringLC();
1085 string ser
= serToString(lrr
);
1086 d_rwtxn
->txn
->put(d_rwtxn
->db
->dbi
, co(lrr
.domain_id
, ordername
, QType::NSEC3
), ser
);
1089 lrr
.content
= ordername
.toDNSString();
1090 ser
= serToString(lrr
);
1091 d_rwtxn
->txn
->put(d_rwtxn
->db
->dbi
, co(lrr
.domain_id
, lrr
.qname
, QType::NSEC3
), ser
);
1097 bool LMDBBackend::feedEnts(int domain_id
, map
<DNSName
, bool>& nonterm
)
1099 LMDBResourceRecord lrr
;
1101 compoundOrdername co
;
1102 for (const auto& nt
: nonterm
) {
1103 lrr
.qname
= nt
.first
.makeRelative(d_transactiondomain
);
1104 lrr
.auth
= nt
.second
;
1105 lrr
.ordername
= true;
1107 std::string ser
= serToString(lrr
);
1108 d_rwtxn
->txn
->put(d_rwtxn
->db
->dbi
, co(domain_id
, lrr
.qname
, QType::ENT
), ser
);
1113 bool LMDBBackend::feedEnts3(int domain_id
, const DNSName
& domain
, map
<DNSName
, bool>& nonterm
, const NSEC3PARAMRecordContent
& ns3prc
, bool narrow
)
1117 LMDBResourceRecord lrr
;
1118 compoundOrdername co
;
1119 for (const auto& nt
: nonterm
) {
1120 lrr
.qname
= nt
.first
.makeRelative(domain
);
1122 lrr
.auth
= nt
.second
;
1123 lrr
.ordername
= nt
.second
;
1124 ser
= serToString(lrr
);
1125 d_rwtxn
->txn
->put(d_rwtxn
->db
->dbi
, co(domain_id
, lrr
.qname
, QType::ENT
), ser
);
1127 if (!narrow
&& lrr
.auth
) {
1128 lrr
.content
= lrr
.qname
.toDNSString();
1130 lrr
.ordername
= false;
1131 ser
= serToString(lrr
);
1133 ordername
= DNSName(toBase32Hex(hashQNameWithSalt(ns3prc
, nt
.first
)));
1134 d_rwtxn
->txn
->put(d_rwtxn
->db
->dbi
, co(domain_id
, ordername
, QType::NSEC3
), ser
);
1137 lrr
.content
= ordername
.toDNSString();
1138 ser
= serToString(lrr
);
1139 d_rwtxn
->txn
->put(d_rwtxn
->db
->dbi
, co(domain_id
, lrr
.qname
, QType::NSEC3
), ser
);
1145 // might be called within a transaction, might also be called alone
1146 bool LMDBBackend::replaceRRSet(uint32_t domain_id
, const DNSName
& qname
, const QType
& qt
, const vector
<DNSResourceRecord
>& rrset
)
1148 // zonk qname/qtype within domain_id (go through qname, check domain_id && qtype)
1149 shared_ptr
<RecordsRWTransaction
> txn
;
1150 bool needCommit
= false;
1151 if (d_rwtxn
&& d_transactiondomainid
== domain_id
) {
1153 // cout<<"Reusing open transaction"<<endl;
1156 // cout<<"Making a new RW txn for replace rrset"<<endl;
1157 txn
= getRecordsRWTransaction(domain_id
);
1162 if (!d_tdomains
->getROTransaction().get(domain_id
, di
)) {
1166 compoundOrdername co
;
1167 auto cursor
= txn
->txn
->getCursor(txn
->db
->dbi
);
1169 string match
= co(domain_id
, qname
.makeRelative(di
.zone
), qt
.getCode());
1170 if (!cursor
.find(match
, key
, val
)) {
1174 if (!rrset
.empty()) {
1175 vector
<LMDBResourceRecord
> adjustedRRSet
;
1176 for (const auto& rr
: rrset
) {
1177 LMDBResourceRecord
lrr(rr
);
1178 lrr
.content
= serializeContent(lrr
.qtype
.getCode(), lrr
.qname
, lrr
.content
);
1179 lrr
.qname
.makeUsRelative(di
.zone
);
1181 adjustedRRSet
.emplace_back(lrr
);
1183 txn
->txn
->put(txn
->db
->dbi
, match
, serToString(adjustedRRSet
));
1192 bool LMDBBackend::replaceComments([[maybe_unused
]] const uint32_t domain_id
, [[maybe_unused
]] const DNSName
& qname
, [[maybe_unused
]] const QType
& qt
, const vector
<Comment
>& comments
)
1194 // if the vector is empty, good, that's what we do here (LMDB does not store comments)
1195 // if it's not, report failure
1196 return comments
.empty();
1199 // tempting to templatize these two functions but the pain is not worth it
1200 std::shared_ptr
<LMDBBackend::RecordsRWTransaction
> LMDBBackend::getRecordsRWTransaction(uint32_t id
)
1202 auto& shard
= d_trecords
[id
% s_shards
];
1204 shard
.env
= getMDBEnv((getArg("filename") + "-" + std::to_string(id
% s_shards
)).c_str(),
1205 MDB_NOSUBDIR
| d_asyncFlag
, 0600);
1206 shard
.dbi
= shard
.env
->openDB("records_v5", MDB_CREATE
);
1208 auto ret
= std::make_shared
<RecordsRWTransaction
>(shard
.env
->getRWTransaction());
1209 ret
->db
= std::make_shared
<RecordsDB
>(shard
);
1214 std::shared_ptr
<LMDBBackend::RecordsROTransaction
> LMDBBackend::getRecordsROTransaction(uint32_t id
, std::shared_ptr
<LMDBBackend::RecordsRWTransaction
> rwtxn
)
1216 auto& shard
= d_trecords
[id
% s_shards
];
1219 throw DBException("attempting to start nested transaction without open parent env");
1221 shard
.env
= getMDBEnv((getArg("filename") + "-" + std::to_string(id
% s_shards
)).c_str(),
1222 MDB_NOSUBDIR
| d_asyncFlag
, 0600);
1223 shard
.dbi
= shard
.env
->openDB("records_v5", MDB_CREATE
);
1227 auto ret
= std::make_shared
<RecordsROTransaction
>(rwtxn
->txn
->getROTransaction());
1228 ret
->db
= std::make_shared
<RecordsDB
>(shard
);
1232 auto ret
= std::make_shared
<RecordsROTransaction
>(shard
.env
->getROTransaction());
1233 ret
->db
= std::make_shared
<RecordsDB
>(shard
);
1239 // FIXME reinstate soon
1240 bool LMDBBackend::upgradeToSchemav3()
1242 g_log
<< Logger::Warning
<< "Upgrading LMDB schema" << endl
;
1244 for (auto i
= 0; i
< s_shards
; i
++) {
1245 string filename
= getArg("filename") + "-" + std::to_string(i
);
1246 if (rename(filename
.c_str(), (filename
+ "-old").c_str()) < 0) {
1247 if (errno
== ENOENT
) {
1248 // apparently this shard doesn't exist yet, moving on
1251 unixDie("Rename failed during LMDB upgrade");
1254 LMDBBackend::RecordsDB oldShard
, newShard
;
1256 oldShard
.env
= getMDBEnv((filename
+ "-old").c_str(),
1257 MDB_NOSUBDIR
| d_asyncFlag
, 0600);
1258 oldShard
.dbi
= oldShard
.env
->openDB("records", MDB_CREATE
| MDB_DUPSORT
);
1259 auto txn
= oldShard
.env
->getROTransaction();
1260 auto cursor
= txn
->getROCursor(oldShard
.dbi
);
1262 newShard
.env
= getMDBEnv((filename
).c_str(),
1263 MDB_NOSUBDIR
| d_asyncFlag
, 0600);
1264 newShard
.dbi
= newShard
.env
->openDB("records", MDB_CREATE
);
1265 auto newTxn
= newShard
.env
->getRWTransaction();
1268 if (cursor
.first(key
, val
) != 0) {
1274 string_view currentKey
;
1277 auto newKey
= key
.getNoStripHeader
<string_view
>();
1278 if (currentKey
.compare(newKey
) != 0) {
1279 if (value
.size() > 0) {
1280 newTxn
->put(newShard
.dbi
, currentKey
, value
);
1282 currentKey
= newKey
;
1285 value
+= val
.get
<string
>();
1286 if (cursor
.next(key
, val
) != 0) {
1287 if (value
.size() > 0) {
1288 newTxn
->put(newShard
.dbi
, currentKey
, value
);
1303 bool LMDBBackend::deleteDomain(const DNSName
& domain
)
1306 throw DBException(std::string(__PRETTY_FUNCTION__
) + " called without a transaction");
1309 int transactionDomainId
= d_transactiondomainid
;
1310 DNSName transactionDomain
= d_transactiondomain
;
1316 if (!d_handle_dups
) {
1318 auto txn
= d_tdomains
->getROTransaction();
1321 idvec
.push_back(txn
.get
<0>(domain
, di
));
1324 // this transaction used to be RO.
1325 // it is now RW to narrow a race window between PowerDNS and Lightning Stream
1326 // FIXME: turn the entire delete, including this ID scan, into one RW transaction
1327 // when doing that, first do a short RO check to see if we actually have anything to delete
1328 auto txn
= d_tdomains
->getRWTransaction();
1330 txn
.get_multi
<0>(domain
, idvec
);
1333 for (auto id
: idvec
) {
1335 startTransaction(domain
, id
);
1337 { // Remove metadata
1338 auto txn
= d_tmeta
->getRWTransaction();
1341 txn
.get_multi
<0>(domain
, ids
);
1343 for (auto& _id
: ids
) {
1350 { // Remove cryptokeys
1351 auto txn
= d_tkdb
->getRWTransaction();
1353 txn
.get_multi
<0>(domain
, ids
);
1355 for (auto _id
: ids
) {
1363 commitTransaction();
1366 auto txn
= d_tdomains
->getRWTransaction();
1371 startTransaction(transactionDomain
, transactionDomainId
);
1376 bool LMDBBackend::list(const DNSName
& target
, int /* id */, bool include_disabled
)
1378 d_includedisabled
= include_disabled
;
1382 auto dtxn
= d_tdomains
->getROTransaction();
1383 if ((di
.id
= dtxn
.get
<0>(target
, di
))) {
1384 // cerr << "Found domain " << target << " on domain_id " << di.id << ", list requested " << id << endl;
1387 // cerr << "Did not find " << target << endl;
1392 d_rotxn
= getRecordsROTransaction(di
.id
, d_rwtxn
);
1393 d_getcursor
= std::make_shared
<MDBROCursor
>(d_rotxn
->txn
->getCursor(d_rotxn
->db
->dbi
));
1395 compoundOrdername co
;
1396 d_matchkey
= co(di
.id
);
1399 auto a
= d_getcursor
->lower_bound(d_matchkey
, key
, val
);
1400 auto b0
= key
.getNoStripHeader
<StringView
>();
1401 auto b
= b0
.rfind(d_matchkey
, 0);
1403 d_getcursor
.reset();
1406 d_lookupdomain
= target
;
1408 // Make sure we start with fresh data
1409 d_currentrrset
.clear();
1410 d_currentrrsetpos
= 0;
1415 void LMDBBackend::lookup(const QType
& type
, const DNSName
& qdomain
, int zoneId
, DNSPacket
* /* p */)
1418 g_log
<< Logger::Warning
<< "Got lookup for " << qdomain
<< "|" << type
.toString() << " in zone " << zoneId
<< endl
;
1422 d_includedisabled
= false;
1424 DNSName
hunt(qdomain
);
1427 auto rotxn
= d_tdomains
->getROTransaction();
1430 zoneId
= rotxn
.get
<0>(hunt
, di
);
1431 } while (!zoneId
&& type
!= QType::SOA
&& hunt
.chopOff());
1433 // cout << "Did not find zone for "<< qdomain<<endl;
1434 d_getcursor
.reset();
1439 if (!d_tdomains
->getROTransaction().get(zoneId
, di
)) {
1440 // cout<<"Could not find a zone with id "<<zoneId<<endl;
1441 d_getcursor
.reset();
1447 DNSName relqname
= qdomain
.makeRelative(hunt
);
1448 if (relqname
.empty()) {
1451 // cout<<"get will look for "<<relqname<< " in zone "<<hunt<<" with id "<<zoneId<<" and type "<<type.toString()<<endl;
1452 d_rotxn
= getRecordsROTransaction(zoneId
, d_rwtxn
);
1454 compoundOrdername co
;
1455 d_getcursor
= std::make_shared
<MDBROCursor
>(d_rotxn
->txn
->getCursor(d_rotxn
->db
->dbi
));
1457 if (type
.getCode() == QType::ANY
) {
1458 d_matchkey
= co(zoneId
, relqname
);
1461 d_matchkey
= co(zoneId
, relqname
, type
.getCode());
1464 if (d_getcursor
->lower_bound(d_matchkey
, key
, val
) || key
.getNoStripHeader
<StringView
>().rfind(d_matchkey
, 0) != 0) {
1465 d_getcursor
.reset();
1467 g_log
<< Logger::Warning
<< "Query " << ((long)(void*)this) << ": " << d_dtime
.udiffNoReset() << " us to execute (found nothing)" << endl
;
1473 g_log
<< Logger::Warning
<< "Query " << ((long)(void*)this) << ": " << d_dtime
.udiffNoReset() << " us to execute" << endl
;
1476 d_lookupdomain
= hunt
;
1478 // Make sure we start with fresh data
1479 d_currentrrset
.clear();
1480 d_currentrrsetpos
= 0;
1483 bool LMDBBackend::get(DNSZoneRecord
& zr
)
1486 // std::cerr<<"d_getcursor="<<d_getcursor<<std::endl;
1494 if (d_currentrrset
.empty()) {
1495 d_getcursor
->current(d_currentKey
, d_currentVal
);
1497 key
= d_currentKey
.getNoStripHeader
<string_view
>();
1498 zr
.dr
.d_type
= compoundOrdername::getQType(key
).getCode();
1500 if (zr
.dr
.d_type
== QType::NSEC3
) {
1501 // Hit a magic NSEC3 skipping
1502 if (d_getcursor
->next(d_currentKey
, d_currentVal
) || d_currentKey
.getNoStripHeader
<StringView
>().rfind(d_matchkey
, 0) != 0) {
1503 // cerr<<"resetting d_getcursor 1"<<endl;
1504 d_getcursor
.reset();
1509 serFromString(d_currentVal
.get
<string_view
>(), d_currentrrset
);
1510 d_currentrrsetpos
= 0;
1513 key
= d_currentKey
.getNoStripHeader
<string_view
>();
1516 const auto& lrr
= d_currentrrset
.at(d_currentrrsetpos
++);
1518 zr
.disabled
= lrr
.disabled
;
1519 if (!zr
.disabled
|| d_includedisabled
) {
1520 zr
.dr
.d_name
= compoundOrdername::getQName(key
) + d_lookupdomain
;
1521 zr
.domain_id
= compoundOrdername::getDomainID(key
);
1522 zr
.dr
.d_type
= compoundOrdername::getQType(key
).getCode();
1523 zr
.dr
.d_ttl
= lrr
.ttl
;
1524 zr
.dr
.setContent(deserializeContentZR(zr
.dr
.d_type
, zr
.dr
.d_name
, lrr
.content
));
1528 if (d_currentrrsetpos
>= d_currentrrset
.size()) {
1529 d_currentrrset
.clear(); // will invalidate lrr
1530 if (d_getcursor
->next(d_currentKey
, d_currentVal
) || d_currentKey
.getNoStripHeader
<StringView
>().rfind(d_matchkey
, 0) != 0) {
1531 // cerr<<"resetting d_getcursor 2"<<endl;
1532 d_getcursor
.reset();
1536 if (zr
.disabled
&& !d_includedisabled
) {
1540 catch (const std::exception
& e
) {
1541 throw PDNSException(e
.what());
1550 bool LMDBBackend::get(DNSResourceRecord
& rr
)
1557 rr
.qname
= zr
.dr
.d_name
;
1558 rr
.ttl
= zr
.dr
.d_ttl
;
1559 rr
.qtype
= zr
.dr
.d_type
;
1560 rr
.content
= zr
.dr
.getContent()->getZoneRepresentation(true);
1561 rr
.domain_id
= zr
.domain_id
;
1563 rr
.disabled
= zr
.disabled
;
1568 bool LMDBBackend::getSerial(DomainInfo
& di
)
1570 auto txn
= getRecordsROTransaction(di
.id
);
1571 compoundOrdername co
;
1573 if (!txn
->txn
->get(txn
->db
->dbi
, co(di
.id
, g_rootdnsname
, QType::SOA
), val
)) {
1574 LMDBResourceRecord lrr
;
1575 serFromString(val
.get
<string_view
>(), lrr
);
1576 if (lrr
.content
.size() >= 5 * sizeof(uint32_t)) {
1578 // a SOA has five 32 bit fields, the first of which is the serial
1579 // there are two variable length names before the serial, so we calculate from the back
1580 memcpy(&serial
, &lrr
.content
[lrr
.content
.size() - (5 * sizeof(uint32_t))], sizeof(serial
));
1581 di
.serial
= ntohl(serial
);
1583 return !lrr
.disabled
;
1588 bool LMDBBackend::getDomainInfo(const DNSName
& domain
, DomainInfo
& di
, bool getserial
)
1591 auto txn
= d_tdomains
->getROTransaction();
1592 // auto range = txn.prefix_range<0>(domain);
1594 // bool found = false;
1596 // for (auto& iter = range.first ; iter != range.second; ++iter) {
1598 // di.id = iter.getID();
1599 // di.backend = this;
1605 if (!(di
.id
= txn
.get
<0>(domain
, di
))) {
1619 int LMDBBackend::genChangeDomain(const DNSName
& domain
, std::function
<void(DomainInfo
&)> func
)
1621 auto txn
= d_tdomains
->getRWTransaction();
1625 auto id
= txn
.get
<0>(domain
, di
);
1633 int LMDBBackend::genChangeDomain(uint32_t id
, std::function
<void(DomainInfo
&)> func
)
1637 auto txn
= d_tdomains
->getRWTransaction();
1639 if (!txn
.get(id
, di
))
1650 bool LMDBBackend::setKind(const DNSName
& domain
, const DomainInfo::DomainKind kind
)
1652 return genChangeDomain(domain
, [kind
](DomainInfo
& di
) {
1657 bool LMDBBackend::setAccount(const DNSName
& domain
, const std::string
& account
)
1659 return genChangeDomain(domain
, [account
](DomainInfo
& di
) {
1660 di
.account
= account
;
1664 bool LMDBBackend::setPrimaries(const DNSName
& domain
, const vector
<ComboAddress
>& primaries
)
1666 return genChangeDomain(domain
, [&primaries
](DomainInfo
& di
) {
1667 di
.primaries
= primaries
;
1671 bool LMDBBackend::createDomain(const DNSName
& domain
, const DomainInfo::DomainKind kind
, const vector
<ComboAddress
>& primaries
, const string
& account
)
1676 auto txn
= d_tdomains
->getRWTransaction();
1677 if (txn
.get
<0>(domain
, di
)) {
1678 throw DBException("Domain '" + domain
.toLogString() + "' exists already");
1683 di
.primaries
= primaries
;
1684 di
.account
= account
;
1686 txn
.put(di
, 0, d_random_ids
);
1693 void LMDBBackend::getAllDomainsFiltered(vector
<DomainInfo
>* domains
, const std::function
<bool(DomainInfo
&)>& allow
)
1695 auto txn
= d_tdomains
->getROTransaction();
1696 if (d_handle_dups
) {
1697 map
<DNSName
, DomainInfo
> zonemap
;
1700 for (auto iter
= txn
.begin(); iter
!= txn
.end(); ++iter
) {
1701 DomainInfo di
= *iter
;
1702 di
.id
= iter
.getID();
1705 if (!zonemap
.emplace(di
.zone
, di
).second
) {
1706 dups
.insert(di
.zone
);
1710 for (const auto& zone
: dups
) {
1713 // this get grabs the oldest item if there are duplicates
1714 di
.id
= txn
.get
<0>(zone
, di
);
1717 // .get actually found nothing for us
1722 zonemap
[di
.zone
] = di
;
1725 for (auto& [k
, v
] : zonemap
) {
1727 domains
->push_back(std::move(v
));
1732 for (auto iter
= txn
.begin(); iter
!= txn
.end(); ++iter
) {
1733 DomainInfo di
= *iter
;
1734 di
.id
= iter
.getID();
1738 domains
->push_back(di
);
1744 void LMDBBackend::getAllDomains(vector
<DomainInfo
>* domains
, bool /* doSerial */, bool include_disabled
)
1748 getAllDomainsFiltered(domains
, [this, include_disabled
](DomainInfo
& di
) {
1749 if (!getSerial(di
) && !include_disabled
) {
1757 void LMDBBackend::getUnfreshSecondaryInfos(vector
<DomainInfo
>* domains
)
1760 time_t now
= time(0);
1761 LMDBResourceRecord lrr
;
1764 getAllDomainsFiltered(domains
, [this, &lrr
, &st
, &now
, &serial
](DomainInfo
& di
) {
1765 if (!di
.isSecondaryType()) {
1769 auto txn2
= getRecordsROTransaction(di
.id
);
1770 compoundOrdername co
;
1772 if (!txn2
->txn
->get(txn2
->db
->dbi
, co(di
.id
, g_rootdnsname
, QType::SOA
), val
)) {
1773 serFromString(val
.get
<string_view
>(), lrr
);
1774 memcpy(&st
, &lrr
.content
[lrr
.content
.size() - sizeof(soatimes
)], sizeof(soatimes
));
1775 if ((time_t)(di
.last_check
+ ntohl(st
.refresh
)) > now
) { // still fresh
1778 serial
= ntohl(st
.serial
);
1788 void LMDBBackend::setStale(uint32_t domain_id
)
1790 genChangeDomain(domain_id
, [](DomainInfo
& di
) {
1795 void LMDBBackend::setFresh(uint32_t domain_id
)
1797 genChangeDomain(domain_id
, [](DomainInfo
& di
) {
1798 di
.last_check
= time(nullptr);
1802 void LMDBBackend::getUpdatedPrimaries(vector
<DomainInfo
>& updatedDomains
, std::unordered_set
<DNSName
>& catalogs
, CatalogHashMap
& catalogHashes
)
1806 getAllDomainsFiltered(&(updatedDomains
), [this, &catalogs
, &catalogHashes
, &ci
](DomainInfo
& di
) {
1807 if (!di
.isPrimaryType()) {
1811 if (di
.kind
== DomainInfo::Producer
) {
1812 catalogs
.insert(di
.zone
);
1813 catalogHashes
[di
.zone
].process("\0");
1814 return false; // Producer fresness check is performed elsewhere
1817 if (!di
.catalog
.empty()) {
1818 ci
.fromJson(di
.options
, CatalogInfo::CatalogType::Producer
);
1819 ci
.updateHash(catalogHashes
, di
);
1822 if (getSerial(di
) && di
.serial
!= di
.notified_serial
) {
1831 void LMDBBackend::setNotified(uint32_t domain_id
, uint32_t serial
)
1833 genChangeDomain(domain_id
, [serial
](DomainInfo
& di
) {
1834 di
.notified_serial
= serial
;
1838 class getCatalogMembersReturnFalseException
: std::runtime_error
1841 getCatalogMembersReturnFalseException() :
1842 std::runtime_error("getCatalogMembers should return false") {}
1845 bool LMDBBackend::getCatalogMembers(const DNSName
& catalog
, vector
<CatalogInfo
>& members
, CatalogInfo::CatalogType type
)
1847 vector
<DomainInfo
> scratch
;
1850 getAllDomainsFiltered(&scratch
, [&catalog
, &members
, &type
](DomainInfo
& di
) {
1851 if ((type
== CatalogInfo::CatalogType::Producer
&& di
.kind
!= DomainInfo::Primary
) || (type
== CatalogInfo::CatalogType::Consumer
&& di
.kind
!= DomainInfo::Secondary
) || di
.catalog
!= catalog
) {
1857 ci
.d_zone
= di
.zone
;
1858 ci
.d_primaries
= di
.primaries
;
1860 ci
.fromJson(di
.options
, type
);
1862 catch (const std::runtime_error
& e
) {
1863 g_log
<< Logger::Warning
<< __PRETTY_FUNCTION__
<< " options '" << di
.options
<< "' for zone '" << di
.zone
<< "' is no valid JSON: " << e
.what() << endl
;
1865 throw getCatalogMembersReturnFalseException();
1867 members
.emplace_back(ci
);
1872 catch (const getCatalogMembersReturnFalseException
& e
) {
1878 bool LMDBBackend::setOptions(const DNSName
& domain
, const std::string
& options
)
1880 return genChangeDomain(domain
, [options
](DomainInfo
& di
) {
1881 di
.options
= options
;
1885 bool LMDBBackend::setCatalog(const DNSName
& domain
, const DNSName
& catalog
)
1887 return genChangeDomain(domain
, [catalog
](DomainInfo
& di
) {
1888 di
.catalog
= catalog
;
1892 bool LMDBBackend::getAllDomainMetadata(const DNSName
& name
, std::map
<std::string
, std::vector
<std::string
>>& meta
)
1895 auto txn
= d_tmeta
->getROTransaction();
1897 txn
.get_multi
<0>(name
, ids
);
1900 // cerr<<"getAllDomainMetadata start"<<endl;
1901 for (auto id
: ids
) {
1902 if (txn
.get(id
, dm
)) {
1903 meta
[dm
.key
].push_back(dm
.value
);
1909 bool LMDBBackend::setDomainMetadata(const DNSName
& name
, const std::string
& kind
, const std::vector
<std::string
>& meta
)
1911 auto txn
= d_tmeta
->getRWTransaction();
1914 txn
.get_multi
<0>(name
, ids
);
1917 for (auto id
: ids
) {
1918 if (txn
.get(id
, dmeta
)) {
1919 if (dmeta
.key
== kind
) {
1920 // cerr<<"delete"<<endl;
1926 for (const auto& m
: meta
) {
1927 DomainMeta dm
{name
, kind
, m
};
1928 txn
.put(dm
, 0, d_random_ids
);
1934 bool LMDBBackend::getDomainKeys(const DNSName
& name
, std::vector
<KeyData
>& keys
)
1936 auto txn
= d_tkdb
->getROTransaction();
1938 txn
.get_multi
<0>(name
, ids
);
1942 for (auto id
: ids
) {
1943 if (txn
.get(id
, key
)) {
1944 KeyData kd
{key
.content
, id
, key
.flags
, key
.active
, key
.published
};
1952 bool LMDBBackend::removeDomainKey(const DNSName
& name
, unsigned int id
)
1954 auto txn
= d_tkdb
->getRWTransaction();
1956 if (txn
.get(id
, kdb
)) {
1957 if (kdb
.domain
== name
) {
1963 // cout << "??? wanted to remove domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
1967 bool LMDBBackend::addDomainKey(const DNSName
& name
, const KeyData
& key
, int64_t& id
)
1969 auto txn
= d_tkdb
->getRWTransaction();
1970 KeyDataDB kdb
{name
, key
.content
, key
.flags
, key
.active
, key
.published
};
1971 id
= txn
.put(kdb
, 0, d_random_ids
);
1977 bool LMDBBackend::activateDomainKey(const DNSName
& name
, unsigned int id
)
1979 auto txn
= d_tkdb
->getRWTransaction();
1981 if (txn
.get(id
, kdb
)) {
1982 if (kdb
.domain
== name
) {
1983 txn
.modify(id
, [](KeyDataDB
& kdbarg
) {
1984 kdbarg
.active
= true;
1991 // cout << "??? wanted to activate domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
1995 bool LMDBBackend::deactivateDomainKey(const DNSName
& name
, unsigned int id
)
1997 auto txn
= d_tkdb
->getRWTransaction();
1999 if (txn
.get(id
, kdb
)) {
2000 if (kdb
.domain
== name
) {
2001 txn
.modify(id
, [](KeyDataDB
& kdbarg
) {
2002 kdbarg
.active
= false;
2008 // cout << "??? wanted to deactivate domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
2012 bool LMDBBackend::publishDomainKey(const DNSName
& name
, unsigned int id
)
2014 auto txn
= d_tkdb
->getRWTransaction();
2016 if (txn
.get(id
, kdb
)) {
2017 if (kdb
.domain
== name
) {
2018 txn
.modify(id
, [](KeyDataDB
& kdbarg
) {
2019 kdbarg
.published
= true;
2026 // cout << "??? wanted to hide domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
2030 bool LMDBBackend::unpublishDomainKey(const DNSName
& name
, unsigned int id
)
2032 auto txn
= d_tkdb
->getRWTransaction();
2034 if (txn
.get(id
, kdb
)) {
2035 if (kdb
.domain
== name
) {
2036 txn
.modify(id
, [](KeyDataDB
& kdbarg
) {
2037 kdbarg
.published
= false;
2043 // cout << "??? wanted to unhide domain key for domain "<<name<<" with id "<<id<<", could not find it"<<endl;
2047 bool LMDBBackend::getBeforeAndAfterNamesAbsolute(uint32_t id
, const DNSName
& qname
, DNSName
& unhashed
, DNSName
& before
, DNSName
& after
)
2049 // cout << __PRETTY_FUNCTION__<< ": "<<id <<", "<<qname << " " << unhashed<<endl;
2052 if (!d_tdomains
->getROTransaction().get(id
, di
)) {
2053 // domain does not exist, tough luck
2056 // cout <<"Zone: "<<di.zone<<endl;
2058 compoundOrdername co
;
2059 auto txn
= getRecordsROTransaction(id
);
2061 auto cursor
= txn
->txn
->getCursor(txn
->db
->dbi
);
2064 LMDBResourceRecord lrr
;
2066 string matchkey
= co(id
, qname
, QType::NSEC3
);
2067 if (cursor
.lower_bound(matchkey
, key
, val
)) {
2068 // this is beyond the end of the database
2069 // cout << "Beyond end of database!" << endl;
2070 cursor
.last(key
, val
);
2073 if (co
.getDomainID(key
.getNoStripHeader
<StringView
>()) != id
) {
2074 //cout<<"Last record also not part of this zone!"<<endl;
2075 // this implies something is wrong in the database, nothing we can do
2079 if (co
.getQType(key
.getNoStripHeader
<StringView
>()) == QType::NSEC3
) {
2080 serFromString(val
.get
<StringView
>(), lrr
);
2081 if (!lrr
.ttl
) // the kind of NSEC3 we need
2084 if (cursor
.prev(key
, val
)) {
2085 // hit beginning of database, again means something is wrong with it
2089 before
= co
.getQName(key
.getNoStripHeader
<StringView
>());
2090 unhashed
= DNSName(lrr
.content
.c_str(), lrr
.content
.size(), 0, false) + di
.zone
;
2092 // now to find after .. at the beginning of the zone
2093 if (cursor
.lower_bound(co(id
), key
, val
)) {
2094 // cout<<"hit end of zone find when we shouldn't"<<endl;
2098 if (co
.getQType(key
.getNoStripHeader
<StringView
>()) == QType::NSEC3
) {
2099 serFromString(val
.get
<StringView
>(), lrr
);
2104 if (cursor
.next(key
, val
) || co
.getDomainID(key
.getNoStripHeader
<StringView
>()) != id
) {
2105 // cout<<"hit end of zone or database when we shouldn't"<<endl;
2109 after
= co
.getQName(key
.getNoStripHeader
<StringView
>());
2110 // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2114 // cout<<"Ended up at "<<co.getQName(key.get<StringView>()) <<endl;
2116 before
= co
.getQName(key
.getNoStripHeader
<StringView
>());
2117 if (before
== qname
) {
2118 // cout << "Ended up on exact right node" << endl;
2119 before
= co
.getQName(key
.getNoStripHeader
<StringView
>());
2120 // unhashed should be correct now, maybe check?
2121 if (cursor
.next(key
, val
)) {
2122 // xxx should find first hash now
2124 if (cursor
.lower_bound(co(id
), key
, val
)) {
2125 // cout<<"hit end of zone find when we shouldn't for id "<<id<< __LINE__<<endl;
2129 if (co
.getQType(key
.getNoStripHeader
<StringView
>()) == QType::NSEC3
) {
2130 serFromString(val
.get
<StringView
>(), lrr
);
2135 if (cursor
.next(key
, val
) || co
.getDomainID(key
.getNoStripHeader
<StringView
>()) != id
) {
2136 // cout<<"hit end of zone or database when we shouldn't" << __LINE__<<endl;
2140 after
= co
.getQName(key
.getNoStripHeader
<StringView
>());
2141 // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2146 // cout <<"Going backwards to find 'before'"<<endl;
2149 if (co
.getQName(key
.getNoStripHeader
<StringView
>()).canonCompare(qname
) && co
.getQType(key
.getNoStripHeader
<StringView
>()) == QType::NSEC3
) {
2150 // cout<<"Potentially stopping traverse at "<< co.getQName(key.get<StringView>()) <<", " << (co.getQName(key.get<StringView>()).canonCompare(qname))<<endl;
2151 // cout<<"qname = "<<qname<<endl;
2152 // cout<<"here = "<<co.getQName(key.get<StringView>())<<endl;
2153 serFromString(val
.get
<StringView
>(), lrr
);
2158 if (cursor
.prev(key
, val
) || co
.getDomainID(key
.getNoStripHeader
<StringView
>()) != id
) {
2159 // cout <<"XXX Hit *beginning* of zone or database"<<endl;
2160 // this can happen, must deal with it
2161 // should now find the last hash of the zone
2163 if (cursor
.lower_bound(co(id
+ 1), key
, val
)) {
2164 // cout << "Could not find the next higher zone, going to the end of the database then"<<endl;
2165 cursor
.last(key
, val
);
2168 cursor
.prev(key
, val
);
2171 if (co
.getDomainID(key
.getNoStripHeader
<StringView
>()) != id
) {
2172 //cout<<"Last record also not part of this zone!"<<endl;
2173 // this implies something is wrong in the database, nothing we can do
2177 if (co
.getQType(key
.getNoStripHeader
<StringView
>()) == QType::NSEC3
) {
2178 serFromString(val
.get
<StringView
>(), lrr
);
2179 if (!lrr
.ttl
) // the kind of NSEC3 we need
2182 if (cursor
.prev(key
, val
)) {
2183 // hit beginning of database, again means something is wrong with it
2187 before
= co
.getQName(key
.getNoStripHeader
<StringView
>());
2188 unhashed
= DNSName(lrr
.content
.c_str(), lrr
.content
.size(), 0, false) + di
.zone
;
2189 // cout <<"Should still find 'after'!"<<endl;
2190 // for 'after', we need to find the first hash of this zone
2192 if (cursor
.lower_bound(co(id
), key
, val
)) {
2193 // cout<<"hit end of zone find when we shouldn't"<<endl;
2194 // means database is wrong, nothing we can do
2198 if (co
.getQType(key
.getNoStripHeader
<StringView
>()) == QType::NSEC3
) {
2199 serFromString(val
.get
<StringView
>(), lrr
);
2204 if (cursor
.next(key
, val
)) {
2205 // means database is wrong, nothing we can do
2206 // cout<<"hit end of zone when we shouldn't 2"<<endl;
2210 after
= co
.getQName(key
.getNoStripHeader
<StringView
>());
2212 // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2217 before
= co
.getQName(key
.getNoStripHeader
<StringView
>());
2218 unhashed
= DNSName(lrr
.content
.c_str(), lrr
.content
.size(), 0, false) + di
.zone
;
2219 // cout<<"Went backwards, found "<<before<<endl;
2220 // return us to starting point
2222 cursor
.next(key
, val
);
2224 // cout<<"Now going forward"<<endl;
2225 for (int count
= 0;; ++count
) {
2226 if ((count
&& cursor
.next(key
, val
)) || co
.getDomainID(key
.getNoStripHeader
<StringView
>()) != id
) {
2227 // cout <<"Hit end of database or zone, finding first hash then in zone "<<id<<endl;
2228 if (cursor
.lower_bound(co(id
), key
, val
)) {
2229 // cout<<"hit end of zone find when we shouldn't"<<endl;
2230 // means database is wrong, nothing we can do
2234 if (co
.getQType(key
.getNoStripHeader
<StringView
>()) == QType::NSEC3
) {
2235 serFromString(val
.get
<StringView
>(), lrr
);
2240 if (cursor
.next(key
, val
)) {
2241 // means database is wrong, nothing we can do
2242 // cout<<"hit end of zone when we shouldn't 2"<<endl;
2245 // cout << "Next.. "<<endl;
2247 after
= co
.getQName(key
.getNoStripHeader
<StringView
>());
2249 // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2253 // cout<<"After "<<co.getQName(key.get<StringView>()) <<endl;
2254 if (co
.getQType(key
.getNoStripHeader
<StringView
>()) == QType::NSEC3
) {
2255 serFromString(val
.get
<StringView
>(), lrr
);
2261 after
= co
.getQName(key
.getNoStripHeader
<StringView
>());
2262 // cout<<"returning: before="<<before<<", after="<<after<<", unhashed: "<<unhashed<<endl;
2266 bool LMDBBackend::getBeforeAndAfterNames(uint32_t id
, const DNSName
& zonenameU
, const DNSName
& qname
, DNSName
& before
, DNSName
& after
)
2268 DNSName zonename
= zonenameU
.makeLowerCase();
2269 // cout << __PRETTY_FUNCTION__<< ": "<<id <<", "<<zonename << ", '"<<qname<<"'"<<endl;
2271 auto txn
= getRecordsROTransaction(id
);
2272 compoundOrdername co
;
2273 DNSName qname2
= qname
.makeRelative(zonename
);
2274 string matchkey
= co(id
, qname2
);
2275 auto cursor
= txn
->txn
->getCursor(txn
->db
->dbi
);
2277 // cout<<"Lower_bound for "<<qname2<<endl;
2278 if (cursor
.lower_bound(matchkey
, key
, val
)) {
2279 // cout << "Hit end of database, bummer"<<endl;
2280 cursor
.last(key
, val
);
2281 if (co
.getDomainID(key
.getNoStripHeader
<string_view
>()) == id
) {
2282 before
= co
.getQName(key
.getNoStripHeader
<string_view
>()) + zonename
;
2286 // cout << "We were at end of database, but this zone is not there?!"<<endl;
2289 // cout<<"Cursor is at "<<co.getQName(key.get<string_view>()) <<", in zone id "<<co.getDomainID(key.get<string_view>())<< endl;
2291 if (co
.getQType(key
.getNoStripHeader
<string_view
>()).getCode() && co
.getDomainID(key
.getNoStripHeader
<string_view
>()) == id
&& co
.getQName(key
.getNoStripHeader
<string_view
>()) == qname2
) { // don't match ENTs
2292 // cout << "Had an exact match!"<<endl;
2293 before
= qname2
+ zonename
;
2296 rc
= cursor
.next(key
, val
);
2300 if (co
.getDomainID(key
.getNoStripHeader
<string_view
>()) == id
&& key
.getNoStripHeader
<StringView
>().rfind(matchkey
, 0) == 0)
2302 LMDBResourceRecord lrr
;
2303 serFromString(val
.get
<StringView
>(), lrr
);
2304 if (co
.getQType(key
.getNoStripHeader
<string_view
>()).getCode() && (lrr
.auth
|| co
.getQType(key
.getNoStripHeader
<string_view
>()).getCode() == QType::NS
))
2307 if (rc
|| co
.getDomainID(key
.getNoStripHeader
<string_view
>()) != id
) {
2308 // cout << "We hit the end of the zone or database. 'after' is apex" << endl;
2312 after
= co
.getQName(key
.getNoStripHeader
<string_view
>()) + zonename
;
2316 if (co
.getDomainID(key
.getNoStripHeader
<string_view
>()) != id
) {
2317 // cout << "Ended up in next zone, 'after' is zonename" <<endl;
2319 // cout << "Now hunting for previous" << endl;
2322 rc
= cursor
.prev(key
, val
);
2324 // cout<<"Reversed into zone, but got not found from lmdb" <<endl;
2328 if (co
.getDomainID(key
.getNoStripHeader
<string_view
>()) != id
) {
2329 // cout<<"Reversed into zone, but found wrong zone id " << co.getDomainID(key.getNoStripHeader<string_view>()) << " != "<<id<<endl;
2330 // "this can't happen"
2333 LMDBResourceRecord lrr
;
2334 serFromString(val
.get
<StringView
>(), lrr
);
2335 if (co
.getQType(key
.getNoStripHeader
<string_view
>()).getCode() && (lrr
.auth
|| co
.getQType(key
.getNoStripHeader
<string_view
>()).getCode() == QType::NS
))
2339 before
= co
.getQName(key
.getNoStripHeader
<string_view
>()) + zonename
;
2340 // cout<<"Found: "<< before<<endl;
2344 // cout <<"We ended up after "<<qname<<", on "<<co.getQName(key.getNoStripHeader<string_view>())<<endl;
2348 LMDBResourceRecord lrr
;
2349 serFromString(val
.get
<StringView
>(), lrr
);
2350 if (co
.getQType(key
.getNoStripHeader
<string_view
>()).getCode() && (lrr
.auth
|| co
.getQType(key
.getNoStripHeader
<string_view
>()).getCode() == QType::NS
)) {
2351 after
= co
.getQName(key
.getNoStripHeader
<string_view
>()) + zonename
;
2352 // cout <<"Found auth ("<<lrr.auth<<") or an NS record "<<after<<", type: "<<co.getQType(key.getNoStripHeader<string_view>()).toString()<<", ttl = "<<lrr.ttl<<endl;
2353 // cout << makeHexDump(val.get<string>()) << endl;
2356 // cout <<" oops, " << co.getQName(key.getNoStripHeader<string_view>()) << " was not auth "<<lrr.auth<< " type=" << lrr.qtype.toString()<<" or NS, so need to skip ahead a bit more" << endl;
2357 int rc
= cursor
.next(key
, val
);
2360 if (rc
|| co
.getDomainID(key
.getNoStripHeader
<string_view
>()) != id
) {
2361 // cout << " oops, hit end of database or zone. This means after is apex" <<endl;
2366 // go back to where we were
2368 cursor
.prev(key
, val
);
2371 int rc
= cursor
.prev(key
, val
);
2372 if (rc
|| co
.getDomainID(key
.getNoStripHeader
<string_view
>()) != id
) {
2373 // XX I don't think this case can happen
2374 // cout << "We hit the beginning of the zone or database.. now what" << endl;
2377 before
= co
.getQName(key
.getNoStripHeader
<string_view
>()) + zonename
;
2378 LMDBResourceRecord lrr
;
2379 serFromString(val
.get
<string_view
>(), lrr
);
2380 // cout<<"And before to "<<before<<", auth = "<<rr.auth<<endl;
2381 if (co
.getQType(key
.getNoStripHeader
<string_view
>()).getCode() && (lrr
.auth
|| co
.getQType(key
.getNoStripHeader
<string_view
>()) == QType::NS
))
2383 // cout << "Oops, that was wrong, go back one more"<<endl;
2389 bool LMDBBackend::updateDNSSECOrderNameAndAuth(uint32_t domain_id
, const DNSName
& qname
, const DNSName
& ordername
, bool auth
, const uint16_t qtype
)
2391 // cout << __PRETTY_FUNCTION__<< ": "<< domain_id <<", '"<<qname <<"', '"<<ordername<<"', "<<auth<< ", " << qtype << endl;
2392 shared_ptr
<RecordsRWTransaction
> txn
;
2393 bool needCommit
= false;
2394 if (d_rwtxn
&& d_transactiondomainid
== domain_id
) {
2396 // cout<<"Reusing open transaction"<<endl;
2399 // cout<<"Making a new RW txn for " << __PRETTY_FUNCTION__ <<endl;
2400 txn
= getRecordsRWTransaction(domain_id
);
2405 if (!d_tdomains
->getROTransaction().get(domain_id
, di
)) {
2406 // cout<<"Could not find domain_id "<<domain_id <<endl;
2410 DNSName rel
= qname
.makeRelative(di
.zone
);
2412 compoundOrdername co
;
2413 string matchkey
= co(domain_id
, rel
);
2415 auto cursor
= txn
->txn
->getCursor(txn
->db
->dbi
);
2417 if (cursor
.lower_bound(matchkey
, key
, val
)) {
2418 // cout << "Could not find anything"<<endl;
2422 bool hasOrderName
= !ordername
.empty();
2423 bool needNSEC3
= hasOrderName
;
2425 for (; key
.getNoStripHeader
<StringView
>().rfind(matchkey
, 0) == 0;) {
2426 vector
<LMDBResourceRecord
> lrrs
;
2428 if (co
.getQType(key
.getNoStripHeader
<StringView
>()) != QType::NSEC3
) {
2429 serFromString(val
.get
<StringView
>(), lrrs
);
2430 bool changed
= false;
2431 vector
<LMDBResourceRecord
> newRRs
;
2432 for (auto& lrr
: lrrs
) {
2433 lrr
.qtype
= co
.getQType(key
.getNoStripHeader
<StringView
>());
2434 if (!needNSEC3
&& qtype
!= QType::ANY
) {
2435 needNSEC3
= (lrr
.ordername
&& QType(qtype
) != lrr
.qtype
);
2438 if ((qtype
== QType::ANY
|| QType(qtype
) == lrr
.qtype
) && (lrr
.ordername
!= hasOrderName
|| lrr
.auth
!= auth
)) {
2440 lrr
.ordername
= hasOrderName
;
2443 newRRs
.push_back(std::move(lrr
));
2446 cursor
.put(key
, serToString(newRRs
));
2450 if (cursor
.next(key
, val
))
2455 LMDBResourceRecord lrr
;
2456 matchkey
= co(domain_id
, rel
, QType::NSEC3
);
2457 // cerr<<"here qname="<<qname<<" ordername="<<ordername<<" qtype="<<qtype<<" matchkey="<<makeHexDump(matchkey)<<endl;
2459 if (!(txngetrc
= txn
->txn
->get(txn
->db
->dbi
, matchkey
, val
))) {
2460 serFromString(val
.get
<string_view
>(), lrr
);
2463 if (hasOrderName
&& lrr
.content
!= ordername
.toDNSStringLC()) {
2471 txn
->txn
->del(txn
->db
->dbi
, co(domain_id
, DNSName(lrr
.content
.c_str(), lrr
.content
.size(), 0, false), QType::NSEC3
));
2472 txn
->txn
->del(txn
->db
->dbi
, matchkey
);
2479 if (hasOrderName
&& del
) {
2480 matchkey
= co(domain_id
, rel
, QType::NSEC3
);
2484 lrr
.content
= rel
.toDNSStringLC();
2486 string str
= serToString(lrr
);
2487 txn
->txn
->put(txn
->db
->dbi
, co(domain_id
, ordername
, QType::NSEC3
), str
);
2489 lrr
.content
= ordername
.toDNSStringLC();
2490 str
= serToString(lrr
);
2491 txn
->txn
->put(txn
->db
->dbi
, matchkey
, str
); // 2
2499 bool LMDBBackend::updateEmptyNonTerminals(uint32_t domain_id
, set
<DNSName
>& insert
, set
<DNSName
>& erase
, bool remove
)
2501 // cout << __PRETTY_FUNCTION__<< ": "<< domain_id << ", insert.size() "<<insert.size()<<", "<<erase.size()<<", " <<remove<<endl;
2503 bool needCommit
= false;
2504 shared_ptr
<RecordsRWTransaction
> txn
;
2505 if (d_rwtxn
&& d_transactiondomainid
== domain_id
) {
2507 // cout<<"Reusing open transaction"<<endl;
2510 // cout<<"Making a new RW txn for delete domain"<<endl;
2511 txn
= getRecordsRWTransaction(domain_id
);
2515 // if remove is set, all ENTs should be removed & nothing else should be done
2517 deleteDomainRecords(*txn
, domain_id
, 0);
2521 auto rotxn
= d_tdomains
->getROTransaction();
2522 if (!rotxn
.get(domain_id
, di
)) {
2523 // cout <<"No such domain with id "<<domain_id<<endl;
2526 compoundOrdername co
;
2527 for (const auto& n
: insert
) {
2528 LMDBResourceRecord lrr
;
2529 lrr
.qname
= n
.makeRelative(di
.zone
);
2533 std::string ser
= serToString(lrr
);
2535 txn
->txn
->put(txn
->db
->dbi
, co(domain_id
, lrr
.qname
, 0), ser
);
2537 // cout <<" +"<<n<<endl;
2539 for (auto n
: erase
) {
2540 // cout <<" -"<<n<<endl;
2541 n
.makeUsRelative(di
.zone
);
2542 txn
->txn
->del(txn
->db
->dbi
, co(domain_id
, n
, 0));
2551 bool LMDBBackend::getTSIGKey(const DNSName
& name
, DNSName
& algorithm
, string
& content
)
2553 auto txn
= d_ttsig
->getROTransaction();
2555 txn
.get_multi
<0>(name
, ids
);
2558 for (auto id
: ids
) {
2559 if (txn
.get(id
, key
)) {
2560 if (algorithm
.empty() || algorithm
== DNSName(key
.algorithm
)) {
2561 algorithm
= DNSName(key
.algorithm
);
2570 // this deletes an old key if it has the same algorithm
2571 bool LMDBBackend::setTSIGKey(const DNSName
& name
, const DNSName
& algorithm
, const string
& content
)
2573 auto txn
= d_ttsig
->getRWTransaction();
2576 txn
.get_multi
<0>(name
, ids
);
2579 for (auto id
: ids
) {
2580 if (txn
.get(id
, key
)) {
2581 if (key
.algorithm
== algorithm
) {
2589 tk
.algorithm
= algorithm
;
2592 txn
.put(tk
, 0, d_random_ids
);
2597 bool LMDBBackend::deleteTSIGKey(const DNSName
& name
)
2599 auto txn
= d_ttsig
->getRWTransaction();
2602 txn
.get_multi
<0>(name
, ids
);
2606 for (auto id
: ids
) {
2607 if (txn
.get(id
, key
)) {
2614 bool LMDBBackend::getTSIGKeys(std::vector
<struct TSIGKey
>& keys
)
2616 auto txn
= d_ttsig
->getROTransaction();
2619 for (auto iter
= txn
.begin(); iter
!= txn
.end(); ++iter
) {
2620 keys
.push_back(*iter
);
2625 string
LMDBBackend::directBackendCmd(const string
& query
)
2627 ostringstream ret
, usage
;
2629 usage
<< "info show some information about the database" << endl
;
2630 usage
<< "index check domains check zone<>ID indexes" << endl
;
2631 usage
<< "index refresh domains <ID> refresh index for zone with this ID" << endl
;
2632 usage
<< "index refresh-all domains refresh index for all zones with disconnected indexes" << endl
;
2633 vector
<string
> argv
;
2634 stringtok(argv
, query
);
2640 string
& cmd
= argv
[0];
2642 if (cmd
== "help") {
2646 if (cmd
== "info") {
2647 ret
<< "shards: " << s_shards
<< endl
;
2648 ret
<< "schemaversion: " << SCHEMAVERSION
<< endl
;
2653 if (cmd
== "index") {
2654 if (argv
.size() < 2) {
2655 return "need an index subcommand\n";
2658 string
& subcmd
= argv
[1];
2660 if (subcmd
== "check" || subcmd
== "refresh-all") {
2661 bool refresh
= false;
2663 if (subcmd
== "refresh-all") {
2667 if (argv
.size() < 3) {
2668 return "need an index name\n";
2671 if (argv
[2] != "domains") {
2672 return "can only check the domains index\n";
2675 vector
<uint32_t> refreshQueue
;
2678 auto txn
= d_tdomains
->getROTransaction();
2680 for (auto iter
= txn
.begin(); iter
!= txn
.end(); ++iter
) {
2681 DomainInfo di
= *iter
;
2683 auto id
= iter
.getID();
2686 txn
.get_multi
<0>(di
.zone
, ids
);
2688 if (ids
.size() != 1) {
2689 ret
<< "ID->zone index has " << id
<< "->" << di
.zone
<< ", ";
2692 ret
<< "zone->ID index has no entry for " << di
.zone
<< endl
;
2694 refreshQueue
.push_back(id
);
2697 ret
<< " suggested remedy: index refresh domains " << id
<< endl
;
2702 ret
<< "zone->ID index has multiple entries for " << di
.zone
<< ": ";
2703 for (auto id_
: ids
) {
2713 for (const auto& id
: refreshQueue
) {
2714 if (genChangeDomain(id
, [](DomainInfo
& /* di */) {})) {
2715 ret
<< "refreshed " << id
<< endl
;
2718 ret
<< "failed to refresh " << id
<< endl
;
2724 if (subcmd
== "refresh") {
2725 // index refresh domains 12345
2726 if (argv
.size() < 4) {
2727 return "usage: index refresh domains <ID>\n";
2730 if (argv
[2] != "domains") {
2731 return "can only refresh in the domains index\n";
2737 id
= pdns::checked_stoi
<uint32_t>(argv
[3]);
2739 catch (const std::out_of_range
& e
) {
2740 return "ID out of range\n";
2743 if (genChangeDomain(id
, [](DomainInfo
& /* di */) {})) {
2744 ret
<< "refreshed" << endl
;
2747 ret
<< "failed" << endl
;
2753 return "unknown lmdbbackend command\n";
2756 class LMDBFactory
: public BackendFactory
2760 BackendFactory("lmdb") {}
2761 void declareArguments(const string
& suffix
= "") override
2763 declare(suffix
, "filename", "Filename for lmdb", "./pdns.lmdb");
2764 declare(suffix
, "sync-mode", "Synchronisation mode: nosync, nometasync, mapasync, sync", "mapasync");
2765 // there just is no room for more on 32 bit
2766 declare(suffix
, "shards", "Records database will be split into this number of shards", (sizeof(void*) == 4) ? "2" : "64");
2767 declare(suffix
, "schema-version", "Maximum allowed schema version to run on this DB. If a lower version is found, auto update is performed", std::to_string(SCHEMAVERSION
));
2768 declare(suffix
, "random-ids", "Numeric IDs inside the database are generated randomly instead of sequentially", "no");
2769 declare(suffix
, "map-size", "LMDB map size in megabytes", (sizeof(void*) == 4) ? "100" : "16000");
2770 declare(suffix
, "flag-deleted", "Flag entries on deletion instead of deleting them", "no");
2771 declare(suffix
, "lightning-stream", "Run in Lightning Stream compatible mode", "no");
2773 DNSBackend
* make(const string
& suffix
= "") override
2775 return new LMDBBackend(suffix
);
2786 BackendMakers().report(new LMDBFactory
);
2787 g_log
<< Logger::Info
<< "[lmdbbackend] This is the lmdb backend version " VERSION
2788 #ifndef REPRODUCIBLE
2789 << " (" __DATE__
" " __TIME__
")"
2791 << " reporting" << endl
;
2795 static LMDBLoader randomLoader
;