3 #include "lmdb-safe.hh"
4 #include <boost/archive/binary_oarchive.hpp>
5 #include <boost/archive/binary_iarchive.hpp>
6 #include <boost/serialization/vector.hpp>
7 #include <boost/serialization/string.hpp>
8 #include <boost/serialization/utility.hpp>
9 #include <boost/iostreams/stream.hpp>
10 #include <boost/iostreams/stream_buffer.hpp>
11 #include <boost/iostreams/device/back_inserter.hpp>
20 Everything should go into a namespace
21 What is an error? What is an exception?
22 could id=0 be magic? ('no such id')
24 Is boost the best serializer?
26 Perhaps use the separate index concept from multi_index
27 perhaps get eiter to be of same type so for(auto& a : x) works
28 make it more value "like" with unique_ptr
32 /** Return the highest ID used in a database. Returns 0 for an empty DB.
33 This makes us start everything at ID=1, which might make it possible to
36 unsigned int MDBGetMaxID(MDBRWTransaction& txn, MDBDbi& dbi);
38 /** Return a randomly generated ID that is unique and not zero.
39 May throw if the database is very full.
41 unsigned int MDBGetRandomID(MDBRWTransaction& txn, MDBDbi& dbi);
44 /** This is our serialization interface.
45 You can define your own serToString for your type if you know better
48 std::string serToString(const T& t)
50 std::string serial_str;
51 boost::iostreams::back_insert_device<std::string> inserter(serial_str);
52 boost::iostreams::stream<boost::iostreams::back_insert_device<std::string> > s(inserter);
53 boost::archive::binary_oarchive oa(s, boost::archive::no_header | boost::archive::no_codecvt);
60 void serFromString(const string_view& str, T& ret)
64 boost::iostreams::array_source source(&str[0], str.size());
65 boost::iostreams::stream<boost::iostreams::array_source> stream(source);
66 boost::archive::binary_iarchive in_archive(stream, boost::archive::no_header|boost::archive::no_codecvt);
70 std::istringstream istr{str};
71 boost::archive::binary_iarchive oi(istr,boost::archive::no_header|boost::archive::no_codecvt );
77 template <class T, class Enable>
78 inline std::string keyConv(const T& t);
80 template <class T, typename std::enable_if<std::is_arithmetic<T>::value,T>::type* = nullptr>
81 inline std::string keyConv(const T& t)
83 return std::string((char*)&t, sizeof(t));
86 // this is how to override specific types.. it is ugly
87 template<class T, typename std::enable_if<std::is_same<T, std::string>::value,T>::type* = nullptr>
88 inline std::string keyConv(const T& t)
94 /** This is a struct that implements index operations, but
95 only the operations that are broadcast to all indexes.
96 Specifically, to deal with databases with less than the maximum
97 number of interfaces, this only includes calls that should be
98 ignored for empty indexes.
100 this only needs methods that must happen for all indexes at once
101 so specifically, not size<t> or get<t>, people ask for those themselves, and
102 should no do that on indexes that don't exist */
104 template<class Class,typename Type, typename Parent>
107 explicit LMDBIndexOps(Parent* parent) : d_parent(parent){}
108 void put(MDBRWTransaction& txn, const Class& t, uint32_t id, int flags=0)
110 txn->put(d_idx, keyConv(d_parent->getMember(t)), id, flags);
113 void del(MDBRWTransaction& txn, const Class& t, uint32_t id)
115 if(int rc = txn->del(d_idx, keyConv(d_parent->getMember(t)), id)) {
116 throw std::runtime_error("Error deleting from index: " + std::string(mdb_strerror(rc)));
120 void openDB(std::shared_ptr<MDBEnv>& env, string_view str, int flags)
122 d_idx = env->openDB(str, flags);
128 /** This is an index on a field in a struct, it derives from the LMDBIndexOps */
130 template<class Class,typename Type,Type Class::*PtrToMember>
131 struct index_on : LMDBIndexOps<Class, Type, index_on<Class, Type, PtrToMember>>
133 index_on() : LMDBIndexOps<Class, Type, index_on<Class, Type, PtrToMember>>(this)
135 static Type getMember(const Class& c)
137 return c.*PtrToMember;
143 /** This is a calculated index */
144 template<class Class, typename Type, class Func>
145 struct index_on_function : LMDBIndexOps<Class, Type, index_on_function<Class, Type, Func> >
147 index_on_function() : LMDBIndexOps<Class, Type, index_on_function<Class, Type, Func> >(this)
149 static Type getMember(const Class& c)
158 /** nop index, so we can fill our N indexes, even if you don't use them all */
161 template<typename Class>
162 void put(MDBRWTransaction& txn, const Class& t, uint32_t id, int flags=0)
164 template<typename Class>
165 void del(MDBRWTransaction& txn, const Class& t, uint32_t id)
168 void openDB(std::shared_ptr<MDBEnv>& env, string_view str, int flags)
172 typedef uint32_t type; // dummy
176 /** The main class. Templatized only on the indexes and typename right now */
177 template<typename T, class I1=nullindex_t, class I2=nullindex_t, class I3 = nullindex_t, class I4 = nullindex_t>
181 TypedDBI(std::shared_ptr<MDBEnv> env, string_view name)
182 : d_env(env), d_name(name)
184 d_main = d_env->openDB(name, MDB_CREATE | MDB_INTEGERKEY);
186 // now you might be tempted to go all MPL on this so we can get rid of the
187 // ugly macro. I'm not very receptive to that idea since it will make things
189 #define openMacro(N) std::get<N>(d_tuple).openDB(d_env, std::string(name)+"_"#N, MDB_CREATE | MDB_DUPFIXED | MDB_DUPSORT);
198 // we get a lot of our smarts from this tuple, it enables get<0> etc
199 typedef std::tuple<I1, I2, I3, I4> tuple_t;
202 // We support readonly and rw transactions. Here we put the Readonly operations
203 // which get sourced by both kinds of transactions
204 template<class Parent>
205 struct ReadonlyOperations
207 ReadonlyOperations(Parent& parent) : d_parent(parent)
210 //! Number of entries in main database
214 mdb_stat(**d_parent.d_txn, d_parent.d_parent->d_main, &stat);
215 return stat.ms_entries;
218 //! Number of entries in the various indexes - should be the same
223 mdb_stat(**d_parent.d_txn, std::get<N>(d_parent.d_parent->d_tuple).d_idx, &stat);
224 return stat.ms_entries;
227 //! Get item with id, from main table directly
228 bool get(uint32_t id, T& t)
231 if((*d_parent.d_txn)->get(d_parent.d_parent->d_main, id, data))
234 serFromString(data.get<std::string>(), t);
238 //! Get item through index N, then via the main database
240 uint32_t get(const typename std::tuple_element<N, tuple_t>::type::type& key, T& out)
243 if(!(*d_parent.d_txn)->get(std::get<N>(d_parent.d_parent->d_tuple).d_idx, keyConv(key), id)) {
244 if(get(id.get<uint32_t>(), out))
245 return id.get<uint32_t>();
250 //! Cardinality of index N
252 uint32_t cardinality()
254 auto cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
258 while(!cursor.get(key, data, first ? MDB_FIRST : MDB_NEXT_NODUP)) {
265 //! End iterator type
269 // can be on main, or on an index
270 // when on main, return data directly
271 // when on index, indirect
272 // we can be limited to one key, or iterate over entire database
273 // iter requires you to put the cursor in the right place first!
276 explicit iter_t(Parent* parent, typename Parent::cursor_t&& cursor, bool on_index, bool one_key, bool end=false) :
278 d_cursor(std::move(cursor)),
279 d_on_index(on_index), // is this an iterator on main database or on index?
280 d_one_key(one_key), // should we stop at end of key? (equal range)
287 if(d_cursor.get(d_key, d_id, MDB_GET_CURRENT)) {
293 if((*d_parent->d_txn)->get(d_parent->d_parent->d_main, d_id, d_data))
294 throw std::runtime_error("Missing id in constructor");
295 serFromString(d_data.get<std::string>(), d_t);
298 serFromString(d_id.get<std::string>(), d_t);
301 explicit iter_t(Parent* parent, typename Parent::cursor_t&& cursor, const std::string& prefix) :
303 d_cursor(std::move(cursor)),
304 d_on_index(true), // is this an iterator on main database or on index?
312 if(d_cursor.get(d_key, d_id, MDB_GET_CURRENT)) {
318 if((*d_parent->d_txn)->get(d_parent->d_parent->d_main, d_id, d_data))
319 throw std::runtime_error("Missing id in constructor");
320 serFromString(d_data.get<std::string>(), d_t);
323 serFromString(d_id.get<std::string>(), d_t);
327 std::function<bool(const MDBOutVal&)> filter;
333 bool operator!=(const eiter_t& rhs) const
338 bool operator==(const eiter_t& rhs) const
348 const T* operator->()
353 // implements generic ++ or --
354 iter_t& genoperator(MDB_cursor_op dupop, MDB_cursor_op op)
359 rc = d_cursor.get(d_key, d_id, d_one_key ? dupop : op);
360 if(rc == MDB_NOTFOUND) {
364 throw std::runtime_error("in genoperator, " + std::string(mdb_strerror(rc)));
366 else if(!d_prefix.empty() && d_key.get<std::string>().rfind(d_prefix, 0)!=0) {
371 if((*d_parent->d_txn)->get(d_parent->d_parent->d_main, d_id, data))
372 throw std::runtime_error("Missing id field");
373 if(filter && !filter(data))
376 serFromString(data.get<std::string>(), d_t);
379 if(filter && !filter(data))
382 serFromString(d_id.get<std::string>(), d_t);
390 return genoperator(MDB_NEXT_DUP, MDB_NEXT);
394 return genoperator(MDB_PREV_DUP, MDB_PREV);
397 // get ID this iterator points to
401 return d_id.get<uint32_t>();
403 return d_key.get<uint32_t>();
406 const MDBOutVal& getKey()
412 // transaction we are part of
414 typename Parent::cursor_t d_cursor;
416 // gcc complains if I don't zero-init these, which is worrying XXX
417 MDBOutVal d_key{{0,0}}, d_data{{0,0}}, d_id{{0,0}};
420 std::string d_prefix;
426 iter_t genbegin(MDB_cursor_op op)
428 typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
432 if(cursor.get(out, id, op)) {
433 // on_index, one_key, end
434 return iter_t{&d_parent, std::move(cursor), true, false, true};
437 return iter_t{&d_parent, std::move(cursor), true, false};
443 return genbegin<N>(MDB_FIRST);
449 return genbegin<N>(MDB_LAST);
454 typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(d_parent.d_parent->d_main);
458 if(cursor.get(out, id, MDB_FIRST)) {
459 // on_index, one_key, end
460 return iter_t{&d_parent, std::move(cursor), false, false, true};
463 return iter_t{&d_parent, std::move(cursor), false, false};
471 // basis for find, lower_bound
473 iter_t genfind(const typename std::tuple_element<N, tuple_t>::type::type& key, MDB_cursor_op op)
475 typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
477 std::string keystr = keyConv(key);
480 out.d_mdbval = in.d_mdbval;
482 if(cursor.get(out, id, op)) {
483 // on_index, one_key, end
484 return iter_t{&d_parent, std::move(cursor), true, false, true};
487 return iter_t{&d_parent, std::move(cursor), true, false};
491 iter_t find(const typename std::tuple_element<N, tuple_t>::type::type& key)
493 return genfind<N>(key, MDB_SET);
497 iter_t lower_bound(const typename std::tuple_element<N, tuple_t>::type::type& key)
499 return genfind<N>(key, MDB_SET_RANGE);
503 //! equal range - could possibly be expressed through genfind
505 std::pair<iter_t,eiter_t> equal_range(const typename std::tuple_element<N, tuple_t>::type::type& key)
507 typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
509 std::string keyString=keyConv(key);
510 MDBInVal in(keyString);
512 out.d_mdbval = in.d_mdbval;
514 if(cursor.get(out, id, MDB_SET)) {
515 // on_index, one_key, end
516 return {iter_t{&d_parent, std::move(cursor), true, true, true}, eiter_t()};
519 return {iter_t{&d_parent, std::move(cursor), true, true}, eiter_t()};
522 //! equal range - could possibly be expressed through genfind
524 std::pair<iter_t,eiter_t> prefix_range(const typename std::tuple_element<N, tuple_t>::type::type& key)
526 typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
528 std::string keyString=keyConv(key);
529 MDBInVal in(keyString);
531 out.d_mdbval = in.d_mdbval;
533 if(cursor.get(out, id, MDB_SET_RANGE)) {
534 // on_index, one_key, end
535 return {iter_t{&d_parent, std::move(cursor), true, true, true}, eiter_t()};
538 return {iter_t(&d_parent, std::move(cursor), keyString), eiter_t()};
545 class ROTransaction : public ReadonlyOperations<ROTransaction>
548 explicit ROTransaction(TypedDBI* parent) : ReadonlyOperations<ROTransaction>(*this), d_parent(parent), d_txn(std::make_shared<MDBROTransaction>(d_parent->d_env->getROTransaction()))
552 explicit ROTransaction(TypedDBI* parent, std::shared_ptr<MDBROTransaction> txn) : ReadonlyOperations<ROTransaction>(*this), d_parent(parent), d_txn(txn)
557 ROTransaction(ROTransaction&& rhs) :
558 ReadonlyOperations<ROTransaction>(*this), d_parent(rhs.d_parent),d_txn(std::move(rhs.d_txn))
564 std::shared_ptr<MDBROTransaction> getTransactionHandle()
569 typedef MDBROCursor cursor_t;
572 std::shared_ptr<MDBROTransaction> d_txn;
576 class RWTransaction : public ReadonlyOperations<RWTransaction>
579 explicit RWTransaction(TypedDBI* parent) : ReadonlyOperations<RWTransaction>(*this), d_parent(parent)
581 d_txn = std::make_shared<MDBRWTransaction>(d_parent->d_env->getRWTransaction());
584 explicit RWTransaction(TypedDBI* parent, std::shared_ptr<MDBRWTransaction> txn) : ReadonlyOperations<RWTransaction>(*this), d_parent(parent), d_txn(txn)
589 RWTransaction(RWTransaction&& rhs) :
590 ReadonlyOperations<RWTransaction>(*this),
591 d_parent(rhs.d_parent), d_txn(std::move(rhs.d_txn))
596 // insert something, with possibly a specific id
597 uint32_t put(const T& t, uint32_t id, bool random_ids=false)
602 id = MDBGetRandomID(*d_txn, d_parent->d_main);
605 id = MDBGetMaxID(*d_txn, d_parent->d_main) + 1;
609 (*d_txn)->put(d_parent->d_main, id, serToString(t), flags);
611 #define insertMacro(N) std::get<N>(d_parent->d_tuple).put(*d_txn, t, id);
621 // modify an item 'in place', plus update indexes
622 void modify(uint32_t id, std::function<void(T&)> func)
625 if(!this->get(id, t))
626 throw std::runtime_error("Could not modify id "+std::to_string(id));
629 del(id); // this is the lazy way. We could test for changed index fields
633 //! delete an item, and from indexes
634 void del(uint32_t id)
637 if(!this->get(id, t))
640 (*d_txn)->del(d_parent->d_main, id);
644 //! clear database & indexes (by hand!)
647 auto cursor = (*d_txn)->getRWCursor(d_parent->d_main);
650 while(!cursor.get(key, data, first ? MDB_FIRST : MDB_NEXT)) {
653 serFromString(data.get<std::string>(), t);
654 clearIndex(key.get<uint32_t>(), t);
659 //! commit this transaction
665 //! abort this transaction
671 typedef MDBRWCursor cursor_t;
673 std::shared_ptr<MDBRWTransaction> getTransactionHandle()
680 // clear this ID from all indexes
681 void clearIndex(uint32_t id, const T& t)
683 #define clearMacro(N) std::get<N>(d_parent->d_tuple).del(*d_txn, t, id);
693 std::shared_ptr<MDBRWTransaction> d_txn;
696 //! Get an RW transaction
697 RWTransaction getRWTransaction()
699 return RWTransaction(this);
702 //! Get an RO transaction
703 ROTransaction getROTransaction()
705 return ROTransaction(this);
708 //! Get an RW transaction
709 RWTransaction getRWTransaction(std::shared_ptr<MDBRWTransaction> txn)
711 return RWTransaction(this, txn);
714 //! Get an RO transaction
715 ROTransaction getROTransaction(std::shared_ptr<MDBROTransaction> txn)
717 return ROTransaction(this, txn);
720 std::shared_ptr<MDBEnv> getEnv()
726 std::shared_ptr<MDBEnv> d_env;