From: Pieter Lexis Date: Thu, 14 Nov 2019 11:37:20 +0000 (+0100) Subject: LMDB: Update lmdb-safe and make the backend compile X-Git-Tag: auth-4.3.0-alpha1~38^2~10 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9c1e549181acb646957f3f9d464cb80a69ef4b07;p=thirdparty%2Fpdns.git LMDB: Update lmdb-safe and make the backend compile This updates lmdb-safe to https://github.com/ahupowerdns/lmdb-safe/pull/6 at 7ce9a821412480c699ce73e85d8bbafa2a9535e5 --- diff --git a/ext/lmdb-safe/lmdb-safe.cc b/ext/lmdb-safe/lmdb-safe.cc index 094303f253..7397cd3d44 100644 --- a/ext/lmdb-safe/lmdb-safe.cc +++ b/ext/lmdb-safe/lmdb-safe.cc @@ -26,10 +26,8 @@ MDBDbi::MDBDbi(MDB_env* env, MDB_txn* txn, const string_view dbname, int flags) MDBEnv::MDBEnv(const char* fname, int flags, int mode) { - mdb_env_create(&d_env); - uint64_t mapsizeMB = (sizeof(long)==4) ? 100 : 16000; - // on 32 bit platforms, there is just no room for more - if(mdb_env_set_mapsize(d_env, mapsizeMB * 1048576)) + mdb_env_create(&d_env); + if(mdb_env_set_mapsize(d_env, 16ULL*4096*244140ULL)) // 4GB throw std::runtime_error("setting map size"); /* Various other options may also need to be set before opening the handle, e.g. mdb_env_set_mapsize(), mdb_env_set_maxreaders(), mdb_env_set_maxdbs(), @@ -141,53 +139,107 @@ MDBDbi MDBEnv::openDB(const string_view dbname, int flags) if(!(envflags & MDB_RDONLY)) { auto rwt = getRWTransaction(); - MDBDbi ret = rwt.openDB(dbname, flags); - rwt.commit(); + MDBDbi ret = rwt->openDB(dbname, flags); + rwt->commit(); return ret; } MDBDbi ret; { auto rwt = getROTransaction(); - ret = rwt.openDB(dbname, flags); + ret = rwt->openDB(dbname, flags); } return ret; } -MDBRWTransaction::MDBRWTransaction(MDBEnv* parent, int flags) : d_parent(parent) +MDBRWTransactionImpl::MDBRWTransactionImpl(MDBEnv *parent, MDB_txn *txn): + MDBROTransactionImpl(parent, txn) + +{ + +} + +MDB_txn *MDBRWTransactionImpl::openRWTransaction(MDBEnv *env, MDB_txn *parent, int flags) { - if(d_parent->getROTX() || d_parent->getRWTX()) + MDB_txn *result; + if(env->getROTX() || env->getRWTX()) throw std::runtime_error("Duplicate RW transaction"); for(int tries =0 ; tries < 3; ++tries) { // it might happen twice, who knows - if(int rc=mdb_txn_begin(d_parent->d_env, 0, flags, &d_txn)) { + if(int rc=mdb_txn_begin(env->d_env, parent, flags, &result)) { if(rc == MDB_MAP_RESIZED && tries < 2) { // "If the mapsize is increased by another process (..) mdb_txn_begin() will return MDB_MAP_RESIZED. // call mdb_env_set_mapsize with a size of zero to adopt the new size." - mdb_env_set_mapsize(d_parent->d_env, 0); + mdb_env_set_mapsize(env->d_env, 0); continue; } throw std::runtime_error("Unable to start RW transaction: "+std::string(mdb_strerror(rc))); } break; } - d_parent->incRWTX(); + env->incRWTX(); + return result; +} + +MDBRWTransactionImpl::MDBRWTransactionImpl(MDBEnv* parent, int flags): + MDBRWTransactionImpl(parent, openRWTransaction(parent, nullptr, flags)) +{ +} + +MDBRWTransactionImpl::~MDBRWTransactionImpl() +{ + abort(); +} + +void MDBRWTransactionImpl::commit() +{ + closeRORWCursors(); + if (!d_txn) { + return; + } + + if(int rc = mdb_txn_commit(d_txn)) { + throw std::runtime_error("committing: " + std::string(mdb_strerror(rc))); + } + environment().decRWTX(); + d_txn = nullptr; +} + +void MDBRWTransactionImpl::abort() +{ + closeRORWCursors(); + if (!d_txn) { + return; + } + + mdb_txn_abort(d_txn); + // prevent the RO destructor from cleaning up the transaction itself + environment().decRWTX(); + d_txn = nullptr; +} + +MDBROTransactionImpl::MDBROTransactionImpl(MDBEnv *parent, MDB_txn *txn): + d_parent(parent), + d_cursors(), + d_txn(txn) +{ + } -MDBROTransaction::MDBROTransaction(MDBEnv* parent, int flags) : d_parent(parent) +MDB_txn *MDBROTransactionImpl::openROTransaction(MDBEnv *env, MDB_txn *parent, int flags) { - if(d_parent->getRWTX()) + if(env->getRWTX()) throw std::runtime_error("Duplicate RO transaction"); /* A transaction and its cursors must only be used by a single thread, and a thread may only have a single transaction at a time. If MDB_NOTLS is in use, this does not apply to read-only transactions. */ - + MDB_txn *result = nullptr; for(int tries =0 ; tries < 3; ++tries) { // it might happen twice, who knows - if(int rc=mdb_txn_begin(d_parent->d_env, 0, MDB_RDONLY | flags, &d_txn)) { + if(int rc=mdb_txn_begin(env->d_env, parent, MDB_RDONLY | flags, &result)) { if(rc == MDB_MAP_RESIZED && tries < 2) { // "If the mapsize is increased by another process (..) mdb_txn_begin() will return MDB_MAP_RESIZED. // call mdb_env_set_mapsize with a size of zero to adopt the new size." - mdb_env_set_mapsize(d_parent->d_env, 0); + mdb_env_set_mapsize(env->d_env, 0); continue; } @@ -195,43 +247,125 @@ MDBROTransaction::MDBROTransaction(MDBEnv* parent, int flags) : d_parent(parent) } break; } - d_parent->incROTX(); + env->incROTX(); + + return result; } +void MDBROTransactionImpl::closeROCursors() +{ + // we need to move the vector away to ensure that the cursors don’t mess with our iteration. + std::vector buf; + std::swap(d_cursors, buf); + for (auto &cursor: buf) { + cursor->close(); + } +} +MDBROTransactionImpl::MDBROTransactionImpl(MDBEnv *parent, int flags): + MDBROTransactionImpl(parent, openROTransaction(parent, nullptr, flags)) +{ -void MDBRWTransaction::clear(MDB_dbi dbi) +} + +MDBROTransactionImpl::~MDBROTransactionImpl() +{ + // this is safe because C++ will not call overrides of virtual methods in destructors. + commit(); +} + +void MDBROTransactionImpl::abort() +{ + closeROCursors(); + // if d_txn is non-nullptr here, either the transaction object was invalidated earlier (e.g. by moving from it), or it is an RW transaction which has already cleaned up the d_txn pointer (with an abort). + if (d_txn) { + d_parent->decROTX(); + mdb_txn_abort(d_txn); // this appears to work better than abort for r/o database opening + d_txn = nullptr; + } +} + +void MDBROTransactionImpl::commit() +{ + closeROCursors(); + // if d_txn is non-nullptr here, either the transaction object was invalidated earlier (e.g. by moving from it), or it is an RW transaction which has already cleaned up the d_txn pointer (with an abort). + if (d_txn) { + d_parent->decROTX(); + mdb_txn_commit(d_txn); // this appears to work better than abort for r/o database opening + d_txn = nullptr; + } +} + + + +void MDBRWTransactionImpl::clear(MDB_dbi dbi) { if(int rc = mdb_drop(d_txn, dbi, 0)) { throw runtime_error("Error clearing database: " + MDBError(rc)); } } -MDBRWCursor MDBRWTransaction::getCursor(const MDBDbi& dbi) +MDBRWCursor MDBRWTransactionImpl::getRWCursor(const MDBDbi& dbi) { - return MDBRWCursor(this, dbi); + MDB_cursor *cursor; + int rc= mdb_cursor_open(d_txn, dbi, &cursor); + if(rc) { + throw std::runtime_error("Error creating RO cursor: "+std::string(mdb_strerror(rc))); + } + return MDBRWCursor(d_rw_cursors, cursor); +} + +MDBRWCursor MDBRWTransactionImpl::getCursor(const MDBDbi &dbi) +{ + return getRWCursor(dbi); +} + +MDBRWTransaction MDBRWTransactionImpl::getRWTransaction() +{ + MDB_txn *txn; + if (int rc = mdb_txn_begin(environment(), *this, 0, &txn)) { + throw std::runtime_error(std::string("failed to start child transaction: ")+mdb_strerror(rc)); + } + // we need to increase the counter here because commit/abort on the child transaction will decrease it + environment().incRWTX(); + return MDBRWTransaction(new MDBRWTransactionImpl(&environment(), txn)); +} + +MDBROTransaction MDBRWTransactionImpl::getROTransaction() +{ + return std::move(getRWTransaction()); } MDBROTransaction MDBEnv::getROTransaction() { - return MDBROTransaction(this); + return MDBROTransaction(new MDBROTransactionImpl(this)); } MDBRWTransaction MDBEnv::getRWTransaction() { - return MDBRWTransaction(this); + return MDBRWTransaction(new MDBRWTransactionImpl(this)); } -void MDBRWTransaction::closeCursors() +void MDBRWTransactionImpl::closeRWCursors() { - for(auto& c : d_cursors) - c->close(); - d_cursors.clear(); + decltype(d_rw_cursors) buf; + std::swap(d_rw_cursors, buf); + for (auto &cursor: buf) { + cursor->close(); + } } -MDBROCursor MDBROTransaction::getCursor(const MDBDbi& dbi) +MDBROCursor MDBROTransactionImpl::getCursor(const MDBDbi& dbi) { - return MDBROCursor(this, dbi); + return getROCursor(dbi); } - +MDBROCursor MDBROTransactionImpl::getROCursor(const MDBDbi &dbi) +{ + MDB_cursor *cursor; + int rc= mdb_cursor_open(d_txn, dbi, &cursor); + if(rc) { + throw std::runtime_error("Error creating RO cursor: "+std::string(mdb_strerror(rc))); + } + return MDBROCursor(d_cursors, cursor); +} diff --git a/ext/lmdb-safe/lmdb-safe.hh b/ext/lmdb-safe/lmdb-safe.hh index 66aae7e503..5185807f31 100644 --- a/ext/lmdb-safe/lmdb-safe.hh +++ b/ext/lmdb-safe/lmdb-safe.hh @@ -9,11 +9,13 @@ #include #include #include +#include +#include // apple compiler somehow has string_view even in c++11! #if __cplusplus < 201703L && !defined(__APPLE__) #include -#if BOOST_VERSION >= 106100 +#if BOOST_VERSION > 105400 #include using boost::string_view; #else @@ -43,8 +45,9 @@ The error strategy. Anything that "should never happen" turns into an exception. class MDBDbi { public: - MDBDbi(): d_dbi(-1) + MDBDbi() { + d_dbi = -1; } explicit MDBDbi(MDB_env* env, MDB_txn* txn, string_view dbname, int flags); @@ -56,8 +59,11 @@ public: MDB_dbi d_dbi; }; -class MDBRWTransaction; -class MDBROTransaction; +class MDBRWTransactionImpl; +class MDBROTransactionImpl; + +using MDBROTransaction = std::unique_ptr; +using MDBRWTransaction = std::unique_ptr; class MDBEnv { @@ -133,6 +139,16 @@ struct MDBOutVal memcpy(&ret, d_mdbval.mv_data, sizeof(T)); return ret; } + + template + const T* get_struct_ptr() const + { + if(d_mdbval.mv_size != sizeof(T)) + throw std::runtime_error("MDB data has wrong length for type"); + + return reinterpret_cast(d_mdbval.mv_data); + } + MDB_val d_mdbval; }; @@ -150,8 +166,9 @@ template<> inline string_view MDBOutVal::get() const class MDBInVal { public: - MDBInVal(const MDBOutVal& rhs): d_mdbval(rhs.d_mdbval) + MDBInVal(const MDBOutVal& rhs) { + d_mdbval = rhs.d_mdbval; } template d_cursors; + +protected: + MDB_txn* d_txn; + + void closeROCursors(); + public: - explicit MDBROTransaction(MDBEnv* parent, int flags=0); + explicit MDBROTransactionImpl(MDBEnv* parent, int flags=0); - MDBROTransaction(MDBROTransaction&& rhs) - { - d_parent = rhs.d_parent; - d_txn = rhs.d_txn; - rhs.d_parent = 0; - rhs.d_txn = 0; - } + MDBROTransactionImpl(const MDBROTransactionImpl& src) = delete; + MDBROTransactionImpl &operator=(const MDBROTransactionImpl& src) = delete; - void reset() - { - // this does not free cursors - mdb_txn_reset(d_txn); - d_parent->decROTX(); - } + // The move constructor/operator cannot be made safe due to Object Slicing with MDBRWTransaction. + MDBROTransactionImpl(MDBROTransactionImpl&& rhs) = delete; + MDBROTransactionImpl &operator=(MDBROTransactionImpl &&rhs) = delete; - void renew() - { - if(d_parent->getROTX()) - throw std::runtime_error("Duplicate RO transaction"); - if(int rc = mdb_txn_renew(d_txn)) - throw std::runtime_error("Renewing RO transaction: "+std::string(mdb_strerror(rc))); - d_parent->incROTX(); - } - + virtual ~MDBROTransactionImpl(); + + virtual void abort(); + virtual void commit(); int get(MDB_dbi dbi, const MDBInVal& key, MDBOutVal& val) { @@ -268,22 +286,21 @@ public: } MDBROCursor getCursor(const MDBDbi&); + MDBROCursor getROCursor(const MDBDbi&); - ~MDBROTransaction() + operator MDB_txn*() { - if(d_txn) { - d_parent->decROTX(); - mdb_txn_commit(d_txn); // this appears to work better than abort for r/o database opening - } + return d_txn; } - operator MDB_txn*&() - { + inline operator bool() const { return d_txn; } - - MDBEnv* d_parent; - MDB_txn* d_txn; + + inline MDBEnv &environment() + { + return *d_parent; + } }; /* @@ -292,12 +309,75 @@ public: "If the parent transaction commits, the cursor must not be used again." */ -template +template class MDBGenCursor { +private: + std::vector *d_registry; + MDB_cursor* d_cursor; + +public: + MDBGenCursor(): + d_registry(nullptr), + d_cursor(nullptr) + { + + } + + MDBGenCursor(std::vector ®istry, MDB_cursor *cursor): + d_registry(®istry), + d_cursor(cursor) + { + registry.emplace_back(static_cast(this)); + } + +private: + void move_from(MDBGenCursor *src) + { + if (!d_registry) { + return; + } + + auto iter = std::find(d_registry->begin(), + d_registry->end(), + src); + if (iter != d_registry->end()) { + *iter = static_cast(this); + } else { + d_registry->emplace_back(static_cast(this)); + } + } + +public: + MDBGenCursor(const MDBGenCursor &src) = delete; + + MDBGenCursor(MDBGenCursor &&src) noexcept: + d_registry(src.d_registry), + d_cursor(src.d_cursor) + { + move_from(&src); + src.d_registry = nullptr; + src.d_cursor = nullptr; + } + + MDBGenCursor &operator=(const MDBGenCursor &src) = delete; + + MDBGenCursor &operator=(MDBGenCursor &&src) noexcept + { + d_registry = src.d_registry; + d_cursor = src.d_cursor; + move_from(&src); + src.d_registry = nullptr; + src.d_cursor = nullptr; + return *this; + } + + ~MDBGenCursor() + { + close(); + } + public: - MDBGenCursor(Transaction *t) : d_parent(t) - {} int get(MDBOutVal& key, MDBOutVal& data, MDB_cursor_op op) { int rc = mdb_cursor_get(d_cursor, &key.d_mdbval, &data.d_mdbval, op); @@ -365,103 +445,78 @@ public: return currentlast(key, data, MDB_FIRST); } - operator MDB_cursor*&() + operator MDB_cursor*() { return d_cursor; } - MDB_cursor* d_cursor{nullptr}; - Transaction* d_parent; -}; - -class MDBROCursor : public MDBGenCursor -{ -public: - MDBROCursor(MDBROTransaction* parent, const MDB_dbi& dbi) : MDBGenCursor(parent) + operator bool() const { - int rc= mdb_cursor_open(d_parent->d_txn, dbi, &d_cursor); - if(rc) { - throw std::runtime_error("Error creating RO cursor: "+std::string(mdb_strerror(rc))); - } - } - MDBROCursor(MDBROCursor&& rhs) : MDBGenCursor(rhs.d_parent) - { - d_cursor = rhs.d_cursor; - rhs.d_cursor = nullptr; + return d_cursor; } void close() { + if (d_registry) { + auto iter = std::find(d_registry->begin(), + d_registry->end(), + static_cast(this)); + if (iter != d_registry->end()) { + d_registry->erase(iter); + } + d_registry = nullptr; + } if (d_cursor) { mdb_cursor_close(d_cursor); d_cursor = nullptr; } } - - ~MDBROCursor() - { - if(d_cursor) - mdb_cursor_close(d_cursor); - } +}; + +class MDBROCursor : public MDBGenCursor +{ +public: + MDBROCursor() = default; + using MDBGenCursor::MDBGenCursor; + MDBROCursor(const MDBROCursor &src) = delete; + MDBROCursor(MDBROCursor &&src) = default; + MDBROCursor &operator=(const MDBROCursor &src) = delete; + MDBROCursor &operator=(MDBROCursor &&src) = default; + ~MDBROCursor() = default; }; class MDBRWCursor; -class MDBRWTransaction +class MDBRWTransactionImpl: public MDBROTransactionImpl { -public: - explicit MDBRWTransaction(MDBEnv* parent, int flags=0); +protected: + MDBRWTransactionImpl(MDBEnv* parent, MDB_txn* txn); - MDBRWTransaction(MDBRWTransaction&& rhs) - { - d_parent = rhs.d_parent; - d_txn = rhs.d_txn; - rhs.d_parent = 0; - rhs.d_txn = 0; - } +private: + static MDB_txn *openRWTransaction(MDBEnv* env, MDB_txn *parent, int flags); - MDBRWTransaction& operator=(MDBRWTransaction&& rhs) - { - if(d_txn) - abort(); +private: + std::vector d_rw_cursors; - d_parent = rhs.d_parent; - d_txn = rhs.d_txn; - rhs.d_parent = 0; - rhs.d_txn = 0; - - return *this; + void closeRWCursors(); + inline void closeRORWCursors() { + closeROCursors(); + closeRWCursors(); } - ~MDBRWTransaction() - { - if(d_txn) { - d_parent->decRWTX(); - closeCursors(); - mdb_txn_abort(d_txn); // XXX check response? - } - } - void closeCursors(); - - void commit() - { - closeCursors(); - if(int rc = mdb_txn_commit(d_txn)) { - throw std::runtime_error("committing: " + std::string(mdb_strerror(rc))); - } - d_parent->decRWTX(); +public: + explicit MDBRWTransactionImpl(MDBEnv* parent, int flags=0); - d_txn=0; - } + MDBRWTransactionImpl(const MDBRWTransactionImpl& rhs) = delete; + MDBRWTransactionImpl(MDBRWTransactionImpl&& rhs) = delete; + MDBRWTransactionImpl &operator=(const MDBRWTransactionImpl& rhs) = delete; + MDBRWTransactionImpl &operator=(MDBRWTransactionImpl&& rhs) = delete; - void abort() - { - closeCursors(); - mdb_txn_abort(d_txn); // XXX check error? - d_txn = 0; - d_parent->decRWTX(); - } + ~MDBRWTransactionImpl() override; + + void commit() override; + void abort() override; void clear(MDB_dbi dbi); @@ -519,79 +574,35 @@ public: MDBDbi openDB(string_view dbname, int flags) { - return MDBDbi(d_parent->d_env, d_txn, dbname, flags); + return MDBDbi(environment().d_env, d_txn, dbname, flags); } + MDBRWCursor getRWCursor(const MDBDbi&); MDBRWCursor getCursor(const MDBDbi&); - void reportCursor(MDBRWCursor* child) - { - d_cursors.insert(child); - } - void unreportCursor(MDBRWCursor* child) - { - d_cursors.erase(child); - } - - void reportCursorMove(MDBRWCursor* from, MDBRWCursor* to) - { - d_cursors.erase(from); - d_cursors.insert(to); - } - - operator MDB_txn*&() - { - return d_txn; - } - - - - std::set d_cursors; - MDBEnv* d_parent; - MDB_txn* d_txn; + MDBRWTransaction getRWTransaction(); + MDBROTransaction getROTransaction(); }; /* "A cursor in a write-transaction can be closed before its transaction ends, and will otherwise be closed when its transaction ends" This is a problem for us since it may means we are closing the cursor twice, which is bad */ -class MDBRWCursor : public MDBGenCursor +class MDBRWCursor : public MDBGenCursor { public: - MDBRWCursor(MDBRWTransaction* parent, const MDB_dbi& dbi) : MDBGenCursor(parent) - { - int rc= mdb_cursor_open(d_parent->d_txn, dbi, &d_cursor); - if(rc) { - throw std::runtime_error("Error creating RW cursor: "+std::string(mdb_strerror(rc))); - } - d_parent->reportCursor(this); - } - MDBRWCursor(MDBRWCursor&& rhs) : MDBGenCursor(rhs.d_parent) - { - d_cursor = rhs.d_cursor; - rhs.d_cursor = nullptr; - d_parent->reportCursorMove(&rhs, this); - } - - void close() - { - if(d_cursor) - mdb_cursor_close(d_cursor); - d_cursor = nullptr; - } - - ~MDBRWCursor() - { - if(d_cursor) - mdb_cursor_close(d_cursor); - d_parent->unreportCursor(this); - } - + MDBRWCursor() = default; + using MDBGenCursor::MDBGenCursor; + MDBRWCursor(const MDBRWCursor &src) = delete; + MDBRWCursor(MDBRWCursor &&src) = default; + MDBRWCursor &operator=(const MDBRWCursor &src) = delete; + MDBRWCursor &operator=(MDBRWCursor &&src) = default; + ~MDBRWCursor() = default; void put(const MDBOutVal& key, const MDBInVal& data) { - int rc = mdb_cursor_put(d_cursor, - const_cast(&key.d_mdbval), - const_cast(&data.d_mdbval), MDB_CURRENT); + int rc = mdb_cursor_put(*this, + const_cast(&key.d_mdbval), + const_cast(&data.d_mdbval), MDB_CURRENT); if(rc) throw std::runtime_error("mdb_cursor_put: " + std::string(mdb_strerror(rc))); } @@ -600,14 +611,15 @@ public: int put(const MDBOutVal& key, const MDBOutVal& data, int flags=0) { // XXX check errors - return mdb_cursor_put(d_cursor, + return mdb_cursor_put(*this, const_cast(&key.d_mdbval), const_cast(&data.d_mdbval), flags); } int del(int flags=0) { - return mdb_cursor_del(d_cursor, flags); + return mdb_cursor_del(*this, flags); } + }; diff --git a/ext/lmdb-safe/lmdb-typed.cc b/ext/lmdb-safe/lmdb-typed.cc index 95206e3aac..b4ecd38746 100644 --- a/ext/lmdb-safe/lmdb-typed.cc +++ b/ext/lmdb-safe/lmdb-typed.cc @@ -2,7 +2,7 @@ unsigned int MDBGetMaxID(MDBRWTransaction& txn, MDBDbi& dbi) { - auto cursor = txn.getCursor(dbi); + auto cursor = txn->getRWCursor(dbi); MDBOutVal maxidval, maxcontent; unsigned int maxid{0}; if(!cursor.get(maxidval, maxcontent, MDB_LAST)) { diff --git a/ext/lmdb-safe/lmdb-typed.hh b/ext/lmdb-safe/lmdb-typed.hh index e6da3dd7d2..b573841ba0 100644 --- a/ext/lmdb-safe/lmdb-typed.hh +++ b/ext/lmdb-safe/lmdb-typed.hh @@ -101,12 +101,12 @@ struct LMDBIndexOps explicit LMDBIndexOps(Parent* parent) : d_parent(parent){} void put(MDBRWTransaction& txn, const Class& t, uint32_t id, int flags=0) { - txn.put(d_idx, keyConv(d_parent->getMember(t)), id, flags); + txn->put(d_idx, keyConv(d_parent->getMember(t)), id, flags); } void del(MDBRWTransaction& txn, const Class& t, uint32_t id) { - if(int rc = txn.del(d_idx, keyConv(d_parent->getMember(t)), id)) { + if(int rc = txn->del(d_idx, keyConv(d_parent->getMember(t)), id)) { throw std::runtime_error("Error deleting from index: " + std::string(mdb_strerror(rc))); } } @@ -205,7 +205,7 @@ public: uint32_t size() { MDB_stat stat; - mdb_stat(*d_parent.d_txn, d_parent.d_parent->d_main, &stat); + mdb_stat(**d_parent.d_txn, d_parent.d_parent->d_main, &stat); return stat.ms_entries; } @@ -214,7 +214,7 @@ public: uint32_t size() { MDB_stat stat; - mdb_stat(*d_parent.d_txn, std::get(d_parent.d_parent->d_tuple).d_idx, &stat); + mdb_stat(**d_parent.d_txn, std::get(d_parent.d_parent->d_tuple).d_idx, &stat); return stat.ms_entries; } @@ -222,7 +222,7 @@ public: bool get(uint32_t id, T& t) { MDBOutVal data; - if(d_parent.d_txn->get(d_parent.d_parent->d_main, id, data)) + if((*d_parent.d_txn)->get(d_parent.d_parent->d_main, id, data)) return false; serFromString(data.get(), t); @@ -234,7 +234,7 @@ public: uint32_t get(const typename std::tuple_element::type::type& key, T& out) { MDBOutVal id; - if(!d_parent.d_txn->get(std::get(d_parent.d_parent->d_tuple).d_idx, keyConv(key), id)) { + if(!(*d_parent.d_txn)->get(std::get(d_parent.d_parent->d_tuple).d_idx, keyConv(key), id)) { if(get(id.get(), out)) return id.get(); } @@ -245,7 +245,7 @@ public: template uint32_t cardinality() { - auto cursor = d_parent.d_txn->getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); + auto cursor = (*d_parent.d_txn)->getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); bool first = true; MDBOutVal key, data; uint32_t count = 0; @@ -256,7 +256,7 @@ public: return count; } - //! End iterator type + //! End iderator type struct eiter_t {}; @@ -284,7 +284,7 @@ public: } if(d_on_index) { - if(d_parent->d_txn->get(d_parent->d_parent->d_main, d_id, d_data)) + if((*d_parent->d_txn)->get(d_parent->d_parent->d_main, d_id, d_data)) throw std::runtime_error("Missing id in constructor"); serFromString(d_data.get(), d_t); } @@ -309,7 +309,7 @@ public: } if(d_on_index) { - if(d_parent->d_txn->get(d_parent->d_parent->d_main, d_id, d_data)) + if((*d_parent->d_txn)->get(d_parent->d_parent->d_main, d_id, d_data)) throw std::runtime_error("Missing id in constructor"); serFromString(d_data.get(), d_t); } @@ -362,7 +362,7 @@ public: } else { if(d_on_index) { - if(d_parent->d_txn->get(d_parent->d_parent->d_main, d_id, data)) + if((*d_parent->d_txn)->get(d_parent->d_parent->d_main, d_id, data)) throw std::runtime_error("Missing id field"); if(filter && !filter(data)) goto next; @@ -419,7 +419,7 @@ public: template iter_t genbegin(MDB_cursor_op op) { - typename Parent::cursor_t cursor = d_parent.d_txn->getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); + typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); MDBOutVal out, id; @@ -445,7 +445,7 @@ public: iter_t begin() { - typename Parent::cursor_t cursor = d_parent.d_txn->getCursor(d_parent.d_parent->d_main); + typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(d_parent.d_parent->d_main); MDBOutVal out, id; @@ -466,7 +466,7 @@ public: template iter_t genfind(const typename std::tuple_element::type::type& key, MDB_cursor_op op) { - typename Parent::cursor_t cursor = d_parent.d_txn->getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); + typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); std::string keystr = keyConv(key); MDBInVal in(keystr); @@ -498,7 +498,7 @@ public: template std::pair equal_range(const typename std::tuple_element::type::type& key) { - typename Parent::cursor_t cursor = d_parent.d_txn->getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); + typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); std::string keyString=keyConv(key); MDBInVal in(keyString); @@ -517,7 +517,7 @@ public: template std::pair prefix_range(const typename std::tuple_element::type::type& key) { - typename Parent::cursor_t cursor = d_parent.d_txn->getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); + typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get(d_parent.d_parent->d_tuple).d_idx); std::string keyString=keyConv(key); MDBInVal in(keyString); @@ -595,7 +595,7 @@ public: id = MDBGetMaxID(*d_txn, d_parent->d_main) + 1; flags = MDB_APPEND; } - d_txn->put(d_parent->d_main, id, serToString(t), flags); + (*d_txn)->put(d_parent->d_main, id, serToString(t), flags); #define insertMacro(N) std::get(d_parent->d_tuple).put(*d_txn, t, id); insertMacro(0); @@ -626,14 +626,14 @@ public: if(!this->get(id, t)) return; - d_txn->del(d_parent->d_main, id); + (*d_txn)->del(d_parent->d_main, id); clearIndex(id, t); } //! clear database & indexes (by hand!) void clear() { - auto cursor = d_txn->getCursor(d_parent->d_main); + auto cursor = (*d_txn)->getRWCursor(d_parent->d_main); bool first = true; MDBOutVal key, data; while(!cursor.get(key, data, first ? MDB_FIRST : MDB_NEXT)) { @@ -648,13 +648,13 @@ public: //! commit this transaction void commit() { - d_txn->commit(); + (*d_txn)->commit(); } //! abort this transaction void abort() { - d_txn->abort(); + (*d_txn)->abort(); } typedef MDBRWCursor cursor_t; diff --git a/modules/lmdbbackend/lmdbbackend.cc b/modules/lmdbbackend/lmdbbackend.cc index 92499916db..938a3f397d 100644 --- a/modules/lmdbbackend/lmdbbackend.cc +++ b/modules/lmdbbackend/lmdbbackend.cc @@ -73,17 +73,17 @@ LMDBBackend::LMDBBackend(const std::string& suffix) auto pdnsdbi = d_tdomains->getEnv()->openDB("pdns", MDB_CREATE); auto txn = d_tdomains->getEnv()->getRWTransaction(); MDBOutVal _schemaversion; - if(!txn.get(pdnsdbi, "schemaversion", _schemaversion)) { + if(!txn->get(pdnsdbi, "schemaversion", _schemaversion)) { auto schemaversion = _schemaversion.get(); if (schemaversion != SCHEMAVERSION) { throw std::runtime_error("Expected LMDB schema version "+std::to_string(SCHEMAVERSION)+" but got "+std::to_string(schemaversion)); } } else { - txn.put(pdnsdbi, "schemaversion", SCHEMAVERSION); + txn->put(pdnsdbi, "schemaversion", SCHEMAVERSION); } MDBOutVal shards; - if(!txn.get(pdnsdbi, "shards", shards)) { + if(!txn->get(pdnsdbi, "shards", shards)) { d_shards = shards.get(); if(d_shards != atoi(getArg("shards").c_str())) { @@ -92,9 +92,9 @@ LMDBBackend::LMDBBackend(const std::string& suffix) } else { d_shards = atoi(getArg("shards").c_str()); - txn.put(pdnsdbi, "shards", d_shards); + txn->put(pdnsdbi, "shards", d_shards); } - txn.commit(); + txn->commit(); d_trecords.resize(d_shards); d_dolog = ::arg().mustDo("query-logging"); } @@ -247,7 +247,7 @@ void LMDBBackend::deleteDomainRecords(RecordsRWTransaction& txn, uint32_t domain compoundOrdername co; string match = co(domain_id); - auto cursor = txn.txn.getCursor(txn.db->dbi); + auto cursor = txn.txn->getCursor(txn.db->dbi); MDBOutVal key, val; // cout<<"Match: "<txn.commit(); + d_rwtxn->txn->commit(); d_rwtxn.reset(); return true; } @@ -308,7 +308,7 @@ bool LMDBBackend::commitTransaction() bool LMDBBackend::abortTransaction() { // cout<<"Abort transaction"<txn.abort(); + d_rwtxn->txn->abort(); d_rwtxn.reset(); return true; @@ -323,21 +323,21 @@ bool LMDBBackend::feedRecord(const DNSResourceRecord &r, const DNSName &ordernam rr.disabled = false; compoundOrdername co; - d_rwtxn->txn.put(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, rr.qtype.getCode()), serToString(rr)); + d_rwtxn->txn->put(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, rr.qtype.getCode()), serToString(rr)); if(ordernameIsNSEC3 && !ordername.empty()) { MDBOutVal val; - if(d_rwtxn->txn.get(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, QType::NSEC3), val)) { + if(d_rwtxn->txn->get(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, QType::NSEC3), val)) { rr.ttl = 0; rr.content=rr.qname.toDNSStringLC(); rr.auth = 0; string ser = serToString(rr); - d_rwtxn->txn.put(d_rwtxn->db->dbi, co(r.domain_id, ordername, QType::NSEC3), ser); + d_rwtxn->txn->put(d_rwtxn->db->dbi, co(r.domain_id, ordername, QType::NSEC3), ser); rr.ttl = 1; rr.content = ordername.toDNSString(); ser = serToString(rr); - d_rwtxn->txn.put(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, QType::NSEC3), ser); + d_rwtxn->txn->put(d_rwtxn->db->dbi, co(r.domain_id, rr.qname, QType::NSEC3), ser); } } return true; @@ -354,7 +354,7 @@ bool LMDBBackend::feedEnts(int domain_id, map& nonterm) rr.disabled = true; std::string ser = serToString(rr); - d_rwtxn->txn.put(d_rwtxn->db->dbi, co(domain_id, rr.qname, 0), ser); + d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, rr.qname, 0), ser); } return true; } @@ -371,7 +371,7 @@ bool LMDBBackend::feedEnts3(int domain_id, const DNSName &domain, maptxn.put(d_rwtxn->db->dbi, co(domain_id, rr.qname, 0), ser); + d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, rr.qname, 0), ser); if(!narrow && rr.auth) { rr.content = rr.qname.toDNSString(); @@ -380,12 +380,12 @@ bool LMDBBackend::feedEnts3(int domain_id, const DNSName &domain, maptxn.put(d_rwtxn->db->dbi, co(domain_id, ordername, QType::NSEC3), ser); + d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, ordername, QType::NSEC3), ser); rr.ttl = 1; rr.content = ordername.toDNSString(); ser = serToString(rr); - d_rwtxn->txn.put(d_rwtxn->db->dbi, co(domain_id, rr.qname, QType::NSEC3), ser); + d_rwtxn->txn->put(d_rwtxn->db->dbi, co(domain_id, rr.qname, QType::NSEC3), ser); } } return true; @@ -414,7 +414,7 @@ bool LMDBBackend::replaceRRSet(uint32_t domain_id, const DNSName& qname, const Q } compoundOrdername co; - auto cursor = txn->txn.getCursor(txn->db->dbi); + auto cursor = txn->txn->getCursor(txn->db->dbi); MDBOutVal key, val; string match =co(domain_id, qname.makeRelative(di.zone), qt.getCode()); if(!cursor.find(match, key, val)) { @@ -426,11 +426,11 @@ bool LMDBBackend::replaceRRSet(uint32_t domain_id, const DNSName& qname, const Q for(auto rr : rrset) { rr.content = serializeContent(rr.qtype.getCode(), rr.qname, rr.content); rr.qname.makeUsRelative(di.zone); - txn->txn.put(txn->db->dbi, match, serToString(rr)); + txn->txn->put(txn->db->dbi, match, serToString(rr)); } if(needCommit) - txn->txn.commit(); + txn->txn->commit(); return true; } @@ -491,7 +491,7 @@ bool LMDBBackend::deleteDomain(const DNSName &domain) compoundOrdername co; string match=co(id); - auto cursor = txn->txn.getCursor(txn->db->dbi); + auto cursor = txn->txn->getCursor(txn->db->dbi); MDBOutVal key, val; if(!cursor.find(match, key, val)) { do { @@ -500,7 +500,7 @@ bool LMDBBackend::deleteDomain(const DNSName &domain) } if(needCommit) - txn->txn.commit(); + txn->txn->commit(); doms.commit(); @@ -525,7 +525,7 @@ bool LMDBBackend::list(const DNSName &target, int id, bool include_disabled) d_rotxn = getRecordsROTransaction(di.id); compoundOrdername co; d_matchkey = co(di.id); - d_getcursor = std::make_shared(d_rotxn->txn.getCursor(d_rotxn->db->dbi)); + d_getcursor = std::make_shared(d_rotxn->txn->getCursor(d_rotxn->db->dbi)); MDBOutVal key, val; d_inlist = true; @@ -574,7 +574,7 @@ void LMDBBackend::lookup(const QType &type, const DNSName &qdomain, int zoneId, d_rotxn = getRecordsROTransaction(zoneId); compoundOrdername co; - d_getcursor = std::make_shared(d_rotxn->txn.getCursor(d_rotxn->db->dbi)); + d_getcursor = std::make_shared(d_rotxn->txn->getCursor(d_rotxn->db->dbi)); MDBOutVal key, val; if(type.getCode() == QType::ANY) { d_matchkey = co(zoneId,relqname); @@ -867,7 +867,7 @@ void LMDBBackend::getAllDomains(vector *domains, bool include_disabl di.id = iter.getID(); auto txn2 = getRecordsROTransaction(iter.getID()); - if(!txn2->txn.get(txn2->db->dbi, co(di.id, g_rootdnsname, QType::SOA), val)) { + if(!txn2->txn->get(txn2->db->dbi, co(di.id, g_rootdnsname, QType::SOA), val)) { DNSResourceRecord rr; serFromString(val.get(), rr); @@ -897,7 +897,7 @@ void LMDBBackend::getUnfreshSlaveInfos(vector* domains) compoundOrdername co; MDBOutVal val; uint32_t serial = 0; - if(!txn2->txn.get(txn2->db->dbi, co(iter.getID(), g_rootdnsname, QType::SOA), val)) { + if(!txn2->txn->get(txn2->db->dbi, co(iter.getID(), g_rootdnsname, QType::SOA), val)) { DNSResourceRecord rr; serFromString(val.get(), rr); struct soatimes st; @@ -1043,7 +1043,7 @@ bool LMDBBackend::getBeforeAndAfterNamesAbsolute(uint32_t id, const DNSName& qna compoundOrdername co; auto txn = getRecordsROTransaction(id); - auto cursor = txn->txn.getCursor(txn->db->dbi); + auto cursor = txn->txn->getCursor(txn->db->dbi); MDBOutVal key, val; DNSResourceRecord rr; @@ -1258,7 +1258,7 @@ bool LMDBBackend::getBeforeAndAfterNames(uint32_t id, const DNSName& zonenameU, compoundOrdername co; DNSName qname2 = qname.makeRelative(zonename); string matchkey=co(id,qname2); - auto cursor = txn->txn.getCursor(txn->db->dbi); + auto cursor = txn->txn->getCursor(txn->db->dbi); MDBOutVal key, val; // cout<<"Lower_bound for "<txn.getCursor(txn->db->dbi); + auto cursor = txn->txn->getCursor(txn->db->dbi); MDBOutVal key, val; if(cursor.lower_bound(matchkey, key, val)) { // cout << "Could not find anything"<txn.get(txn->db->dbi, matchkey, val)) { + if(!txn->txn->get(txn->db->dbi, matchkey, val)) { serFromString(val.get(), rr); if(needNSEC3) { @@ -1445,8 +1445,8 @@ bool LMDBBackend::updateDNSSECOrderNameAndAuth(uint32_t domain_id, const DNSName del = true; } if(del) { - txn->txn.del(txn->db->dbi, co(domain_id, DNSName(rr.content.c_str(), rr.content.size(), 0, false), QType::NSEC3)); - txn->txn.del(txn->db->dbi, matchkey); + txn->txn->del(txn->db->dbi, co(domain_id, DNSName(rr.content.c_str(), rr.content.size(), 0, false), QType::NSEC3)); + txn->txn->del(txn->db->dbi, matchkey); } } else { del = true; @@ -1460,15 +1460,15 @@ bool LMDBBackend::updateDNSSECOrderNameAndAuth(uint32_t domain_id, const DNSName rr.content=rel.toDNSStringLC(); string str = serToString(rr); - txn->txn.put(txn->db->dbi, co(domain_id,ordername,QType::NSEC3), str); + txn->txn->put(txn->db->dbi, co(domain_id,ordername,QType::NSEC3), str); rr.ttl = 1; rr.content = ordername.toDNSStringLC(); str = serToString(rr); - txn->txn.put(txn->db->dbi, matchkey, str); // 2 + txn->txn->put(txn->db->dbi, matchkey, str); // 2 } if(needCommit) - txn->txn.commit(); + txn->txn->commit(); return false; } @@ -1509,7 +1509,7 @@ bool LMDBBackend::updateEmptyNonTerminals(uint32_t domain_id, set& inse std::string ser = serToString(rr); - txn->txn.put(txn->db->dbi, co(domain_id, rr.qname, 0), ser); + txn->txn->put(txn->db->dbi, co(domain_id, rr.qname, 0), ser); DNSResourceRecord rr2; serFromString(ser, rr2); @@ -1519,11 +1519,11 @@ bool LMDBBackend::updateEmptyNonTerminals(uint32_t domain_id, set& inse for(auto n : erase) { // cout <<" -"<txn.del(txn->db->dbi, co(domain_id, n, 0)); + txn->txn->del(txn->db->dbi, co(domain_id, n, 0)); } } if(needCommit) - txn->txn.commit(); + txn->txn->commit(); return false; }