]> git.ipfire.org Git - thirdparty/pdns.git/blob - ext/lmdb-safe/lmdb-typed.hh
updated KSK and ZSK Rollover procedures, small fixes in Algorithm Rollover procedure
[thirdparty/pdns.git] / ext / lmdb-safe / lmdb-typed.hh
1 #pragma once
2 #include <iostream>
3 #include "lmdb-safe.hh"
4 #include <boost/archive/binary_oarchive.hpp>
5 #include <boost/archive/binary_iarchive.hpp>
6 #include <boost/serialization/vector.hpp>
7 #include <boost/serialization/string.hpp>
8 #include <boost/serialization/utility.hpp>
9 #include <boost/iostreams/stream.hpp>
10 #include <boost/iostreams/stream_buffer.hpp>
11 #include <boost/iostreams/device/back_inserter.hpp>
12 #include <sstream>
13 // using std::cout;
14 // using std::endl;
15
16
17 /*
18 Open issues:
19
20 Everything should go into a namespace
21 What is an error? What is an exception?
22 could id=0 be magic? ('no such id')
23 yes
24 Is boost the best serializer?
25 good default
26 Perhaps use the separate index concept from multi_index
27 perhaps get eiter to be of same type so for(auto& a : x) works
28 make it more value "like" with unique_ptr
29 */
30
31
32 /** Return the highest ID used in a database. Returns 0 for an empty DB.
33 This makes us start everything at ID=1, which might make it possible to
34 treat id 0 as special
35 */
36 unsigned int MDBGetMaxID(MDBRWTransaction& txn, MDBDbi& dbi);
37
38 /** Return a randomly generated ID that is unique and not zero.
39 May throw if the database is very full.
40 */
41 unsigned int MDBGetRandomID(MDBRWTransaction& txn, MDBDbi& dbi);
42
43
44 /** This is our serialization interface.
45 You can define your own serToString for your type if you know better
46 */
47 template<typename T>
48 std::string serToString(const T& t)
49 {
50 std::string serial_str;
51 boost::iostreams::back_insert_device<std::string> inserter(serial_str);
52 boost::iostreams::stream<boost::iostreams::back_insert_device<std::string> > s(inserter);
53 boost::archive::binary_oarchive oa(s, boost::archive::no_header | boost::archive::no_codecvt);
54
55 oa << t;
56 return serial_str;
57 }
58
59 template<typename T>
60 void serFromString(const string_view& str, T& ret)
61 {
62 ret = T();
63
64 boost::iostreams::array_source source(&str[0], str.size());
65 boost::iostreams::stream<boost::iostreams::array_source> stream(source);
66 boost::archive::binary_iarchive in_archive(stream, boost::archive::no_header|boost::archive::no_codecvt);
67 in_archive >> ret;
68
69 /*
70 std::istringstream istr{str};
71 boost::archive::binary_iarchive oi(istr,boost::archive::no_header|boost::archive::no_codecvt );
72 oi >> ret;
73 */
74 }
75
76
77 template <class T, class Enable>
78 inline std::string keyConv(const T& t);
79
80 template <class T, typename std::enable_if<std::is_arithmetic<T>::value,T>::type* = nullptr>
81 inline std::string keyConv(const T& t)
82 {
83 return std::string((char*)&t, sizeof(t));
84 }
85
86 // this is how to override specific types.. it is ugly
87 template<class T, typename std::enable_if<std::is_same<T, std::string>::value,T>::type* = nullptr>
88 inline std::string keyConv(const T& t)
89 {
90 return t;
91 }
92
93
94 /** This is a struct that implements index operations, but
95 only the operations that are broadcast to all indexes.
96 Specifically, to deal with databases with less than the maximum
97 number of interfaces, this only includes calls that should be
98 ignored for empty indexes.
99
100 this only needs methods that must happen for all indexes at once
101 so specifically, not size<t> or get<t>, people ask for those themselves, and
102 should no do that on indexes that don't exist */
103
104 template<class Class,typename Type, typename Parent>
105 struct LMDBIndexOps
106 {
107 explicit LMDBIndexOps(Parent* parent) : d_parent(parent){}
108 void put(MDBRWTransaction& txn, const Class& t, uint32_t id, int flags=0)
109 {
110 txn->put(d_idx, keyConv(d_parent->getMember(t)), id, flags);
111 }
112
113 void del(MDBRWTransaction& txn, const Class& t, uint32_t id)
114 {
115 if(int rc = txn->del(d_idx, keyConv(d_parent->getMember(t)), id)) {
116 throw std::runtime_error("Error deleting from index: " + std::string(mdb_strerror(rc)));
117 }
118 }
119
120 void openDB(std::shared_ptr<MDBEnv>& env, string_view str, int flags)
121 {
122 d_idx = env->openDB(str, flags);
123 }
124 MDBDbi d_idx;
125 Parent* d_parent;
126 };
127
128 /** This is an index on a field in a struct, it derives from the LMDBIndexOps */
129
130 template<class Class,typename Type,Type Class::*PtrToMember>
131 struct index_on : LMDBIndexOps<Class, Type, index_on<Class, Type, PtrToMember>>
132 {
133 index_on() : LMDBIndexOps<Class, Type, index_on<Class, Type, PtrToMember>>(this)
134 {}
135 static Type getMember(const Class& c)
136 {
137 return c.*PtrToMember;
138 }
139
140 typedef Type type;
141 };
142
143 /** This is a calculated index */
144 template<class Class, typename Type, class Func>
145 struct index_on_function : LMDBIndexOps<Class, Type, index_on_function<Class, Type, Func> >
146 {
147 index_on_function() : LMDBIndexOps<Class, Type, index_on_function<Class, Type, Func> >(this)
148 {}
149 static Type getMember(const Class& c)
150 {
151 Func f;
152 return f(c);
153 }
154
155 typedef Type type;
156 };
157
158 /** nop index, so we can fill our N indexes, even if you don't use them all */
159 struct nullindex_t
160 {
161 template<typename Class>
162 void put(MDBRWTransaction& txn, const Class& t, uint32_t id, int flags=0)
163 {}
164 template<typename Class>
165 void del(MDBRWTransaction& txn, const Class& t, uint32_t id)
166 {}
167
168 void openDB(std::shared_ptr<MDBEnv>& env, string_view str, int flags)
169 {
170
171 }
172 typedef uint32_t type; // dummy
173 };
174
175
176 /** The main class. Templatized only on the indexes and typename right now */
177 template<typename T, class I1=nullindex_t, class I2=nullindex_t, class I3 = nullindex_t, class I4 = nullindex_t>
178 class TypedDBI
179 {
180 public:
181 TypedDBI(std::shared_ptr<MDBEnv> env, string_view name)
182 : d_env(env), d_name(name)
183 {
184 d_main = d_env->openDB(name, MDB_CREATE | MDB_INTEGERKEY);
185
186 // now you might be tempted to go all MPL on this so we can get rid of the
187 // ugly macro. I'm not very receptive to that idea since it will make things
188 // EVEN uglier.
189 #define openMacro(N) std::get<N>(d_tuple).openDB(d_env, std::string(name)+"_"#N, MDB_CREATE | MDB_DUPFIXED | MDB_DUPSORT);
190 openMacro(0);
191 openMacro(1);
192 openMacro(2);
193 openMacro(3);
194 #undef openMacro
195 }
196
197
198 // we get a lot of our smarts from this tuple, it enables get<0> etc
199 typedef std::tuple<I1, I2, I3, I4> tuple_t;
200 tuple_t d_tuple;
201
202 // We support readonly and rw transactions. Here we put the Readonly operations
203 // which get sourced by both kinds of transactions
204 template<class Parent>
205 struct ReadonlyOperations
206 {
207 ReadonlyOperations(Parent& parent) : d_parent(parent)
208 {}
209
210 //! Number of entries in main database
211 uint32_t size()
212 {
213 MDB_stat stat;
214 mdb_stat(**d_parent.d_txn, d_parent.d_parent->d_main, &stat);
215 return stat.ms_entries;
216 }
217
218 //! Number of entries in the various indexes - should be the same
219 template<int N>
220 uint32_t size()
221 {
222 MDB_stat stat;
223 mdb_stat(**d_parent.d_txn, std::get<N>(d_parent.d_parent->d_tuple).d_idx, &stat);
224 return stat.ms_entries;
225 }
226
227 //! Get item with id, from main table directly
228 bool get(uint32_t id, T& t)
229 {
230 MDBOutVal data;
231 if((*d_parent.d_txn)->get(d_parent.d_parent->d_main, id, data))
232 return false;
233
234 serFromString(data.get<std::string>(), t);
235 return true;
236 }
237
238 //! Get item through index N, then via the main database
239 template<int N>
240 uint32_t get(const typename std::tuple_element<N, tuple_t>::type::type& key, T& out)
241 {
242 MDBOutVal id;
243 if(!(*d_parent.d_txn)->get(std::get<N>(d_parent.d_parent->d_tuple).d_idx, keyConv(key), id)) {
244 if(get(id.get<uint32_t>(), out))
245 return id.get<uint32_t>();
246 }
247 return 0;
248 }
249
250 //! Cardinality of index N
251 template<int N>
252 uint32_t cardinality()
253 {
254 auto cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
255 bool first = true;
256 MDBOutVal key, data;
257 uint32_t count = 0;
258 while(!cursor.get(key, data, first ? MDB_FIRST : MDB_NEXT_NODUP)) {
259 ++count;
260 first=false;
261 }
262 return count;
263 }
264
265 //! End iterator type
266 struct eiter_t
267 {};
268
269 // can be on main, or on an index
270 // when on main, return data directly
271 // when on index, indirect
272 // we can be limited to one key, or iterate over entire database
273 // iter requires you to put the cursor in the right place first!
274 struct iter_t
275 {
276 explicit iter_t(Parent* parent, typename Parent::cursor_t&& cursor, bool on_index, bool one_key, bool end=false) :
277 d_parent(parent),
278 d_cursor(std::move(cursor)),
279 d_on_index(on_index), // is this an iterator on main database or on index?
280 d_one_key(one_key), // should we stop at end of key? (equal range)
281 d_end(end)
282 {
283 if(d_end)
284 return;
285 d_prefix.clear();
286
287 if(d_cursor.get(d_key, d_id, MDB_GET_CURRENT)) {
288 d_end = true;
289 return;
290 }
291
292 if(d_on_index) {
293 if((*d_parent->d_txn)->get(d_parent->d_parent->d_main, d_id, d_data))
294 throw std::runtime_error("Missing id in constructor");
295 serFromString(d_data.get<std::string>(), d_t);
296 }
297 else
298 serFromString(d_id.get<std::string>(), d_t);
299 }
300
301 explicit iter_t(Parent* parent, typename Parent::cursor_t&& cursor, const std::string& prefix) :
302 d_parent(parent),
303 d_cursor(std::move(cursor)),
304 d_on_index(true), // is this an iterator on main database or on index?
305 d_one_key(false),
306 d_prefix(prefix),
307 d_end(false)
308 {
309 if(d_end)
310 return;
311
312 if(d_cursor.get(d_key, d_id, MDB_GET_CURRENT)) {
313 d_end = true;
314 return;
315 }
316
317 if(d_on_index) {
318 if((*d_parent->d_txn)->get(d_parent->d_parent->d_main, d_id, d_data))
319 throw std::runtime_error("Missing id in constructor");
320 serFromString(d_data.get<std::string>(), d_t);
321 }
322 else
323 serFromString(d_id.get<std::string>(), d_t);
324 }
325
326
327 std::function<bool(const MDBOutVal&)> filter;
328 void del()
329 {
330 d_cursor.del();
331 }
332
333 bool operator!=(const eiter_t& rhs) const
334 {
335 return !d_end;
336 }
337
338 bool operator==(const eiter_t& rhs) const
339 {
340 return d_end;
341 }
342
343 const T& operator*()
344 {
345 return d_t;
346 }
347
348 const T* operator->()
349 {
350 return &d_t;
351 }
352
353 // implements generic ++ or --
354 iter_t& genoperator(MDB_cursor_op dupop, MDB_cursor_op op)
355 {
356 MDBOutVal data;
357 int rc;
358 next:;
359 rc = d_cursor.get(d_key, d_id, d_one_key ? dupop : op);
360 if(rc == MDB_NOTFOUND) {
361 d_end = true;
362 }
363 else if(rc) {
364 throw std::runtime_error("in genoperator, " + std::string(mdb_strerror(rc)));
365 }
366 else if(!d_prefix.empty() && d_key.get<std::string>().rfind(d_prefix, 0)!=0) {
367 d_end = true;
368 }
369 else {
370 if(d_on_index) {
371 if((*d_parent->d_txn)->get(d_parent->d_parent->d_main, d_id, data))
372 throw std::runtime_error("Missing id field");
373 if(filter && !filter(data))
374 goto next;
375
376 serFromString(data.get<std::string>(), d_t);
377 }
378 else {
379 if(filter && !filter(data))
380 goto next;
381
382 serFromString(d_id.get<std::string>(), d_t);
383 }
384 }
385 return *this;
386 }
387
388 iter_t& operator++()
389 {
390 return genoperator(MDB_NEXT_DUP, MDB_NEXT);
391 }
392 iter_t& operator--()
393 {
394 return genoperator(MDB_PREV_DUP, MDB_PREV);
395 }
396
397 // get ID this iterator points to
398 uint32_t getID()
399 {
400 if(d_on_index)
401 return d_id.get<uint32_t>();
402 else
403 return d_key.get<uint32_t>();
404 }
405
406 const MDBOutVal& getKey()
407 {
408 return d_key;
409 }
410
411
412 // transaction we are part of
413 Parent* d_parent;
414 typename Parent::cursor_t d_cursor;
415
416 // gcc complains if I don't zero-init these, which is worrying XXX
417 MDBOutVal d_key{{0,0}}, d_data{{0,0}}, d_id{{0,0}};
418 bool d_on_index;
419 bool d_one_key;
420 std::string d_prefix;
421 bool d_end{false};
422 T d_t;
423 };
424
425 template<int N>
426 iter_t genbegin(MDB_cursor_op op)
427 {
428 typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
429
430 MDBOutVal out, id;
431
432 if(cursor.get(out, id, op)) {
433 // on_index, one_key, end
434 return iter_t{&d_parent, std::move(cursor), true, false, true};
435 }
436
437 return iter_t{&d_parent, std::move(cursor), true, false};
438 };
439
440 template<int N>
441 iter_t begin()
442 {
443 return genbegin<N>(MDB_FIRST);
444 }
445
446 template<int N>
447 iter_t rbegin()
448 {
449 return genbegin<N>(MDB_LAST);
450 }
451
452 iter_t begin()
453 {
454 typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(d_parent.d_parent->d_main);
455
456 MDBOutVal out, id;
457
458 if(cursor.get(out, id, MDB_FIRST)) {
459 // on_index, one_key, end
460 return iter_t{&d_parent, std::move(cursor), false, false, true};
461 }
462
463 return iter_t{&d_parent, std::move(cursor), false, false};
464 };
465
466 eiter_t end()
467 {
468 return eiter_t();
469 }
470
471 // basis for find, lower_bound
472 template<int N>
473 iter_t genfind(const typename std::tuple_element<N, tuple_t>::type::type& key, MDB_cursor_op op)
474 {
475 typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
476
477 std::string keystr = keyConv(key);
478 MDBInVal in(keystr);
479 MDBOutVal out, id;
480 out.d_mdbval = in.d_mdbval;
481
482 if(cursor.get(out, id, op)) {
483 // on_index, one_key, end
484 return iter_t{&d_parent, std::move(cursor), true, false, true};
485 }
486
487 return iter_t{&d_parent, std::move(cursor), true, false};
488 };
489
490 template<int N>
491 iter_t find(const typename std::tuple_element<N, tuple_t>::type::type& key)
492 {
493 return genfind<N>(key, MDB_SET);
494 }
495
496 template<int N>
497 iter_t lower_bound(const typename std::tuple_element<N, tuple_t>::type::type& key)
498 {
499 return genfind<N>(key, MDB_SET_RANGE);
500 }
501
502
503 //! equal range - could possibly be expressed through genfind
504 template<int N>
505 std::pair<iter_t,eiter_t> equal_range(const typename std::tuple_element<N, tuple_t>::type::type& key)
506 {
507 typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
508
509 std::string keyString=keyConv(key);
510 MDBInVal in(keyString);
511 MDBOutVal out, id;
512 out.d_mdbval = in.d_mdbval;
513
514 if(cursor.get(out, id, MDB_SET)) {
515 // on_index, one_key, end
516 return {iter_t{&d_parent, std::move(cursor), true, true, true}, eiter_t()};
517 }
518
519 return {iter_t{&d_parent, std::move(cursor), true, true}, eiter_t()};
520 };
521
522 //! equal range - could possibly be expressed through genfind
523 template<int N>
524 std::pair<iter_t,eiter_t> prefix_range(const typename std::tuple_element<N, tuple_t>::type::type& key)
525 {
526 typename Parent::cursor_t cursor = (*d_parent.d_txn)->getCursor(std::get<N>(d_parent.d_parent->d_tuple).d_idx);
527
528 std::string keyString=keyConv(key);
529 MDBInVal in(keyString);
530 MDBOutVal out, id;
531 out.d_mdbval = in.d_mdbval;
532
533 if(cursor.get(out, id, MDB_SET_RANGE)) {
534 // on_index, one_key, end
535 return {iter_t{&d_parent, std::move(cursor), true, true, true}, eiter_t()};
536 }
537
538 return {iter_t(&d_parent, std::move(cursor), keyString), eiter_t()};
539 };
540
541
542 Parent& d_parent;
543 };
544
545 class ROTransaction : public ReadonlyOperations<ROTransaction>
546 {
547 public:
548 explicit ROTransaction(TypedDBI* parent) : ReadonlyOperations<ROTransaction>(*this), d_parent(parent), d_txn(std::make_shared<MDBROTransaction>(d_parent->d_env->getROTransaction()))
549 {
550 }
551
552 explicit ROTransaction(TypedDBI* parent, std::shared_ptr<MDBROTransaction> txn) : ReadonlyOperations<ROTransaction>(*this), d_parent(parent), d_txn(txn)
553 {
554 }
555
556
557 ROTransaction(ROTransaction&& rhs) :
558 ReadonlyOperations<ROTransaction>(*this), d_parent(rhs.d_parent),d_txn(std::move(rhs.d_txn))
559
560 {
561 rhs.d_parent = 0;
562 }
563
564 std::shared_ptr<MDBROTransaction> getTransactionHandle()
565 {
566 return d_txn;
567 }
568
569 typedef MDBROCursor cursor_t;
570
571 TypedDBI* d_parent;
572 std::shared_ptr<MDBROTransaction> d_txn;
573 };
574
575
576 class RWTransaction : public ReadonlyOperations<RWTransaction>
577 {
578 public:
579 explicit RWTransaction(TypedDBI* parent) : ReadonlyOperations<RWTransaction>(*this), d_parent(parent)
580 {
581 d_txn = std::make_shared<MDBRWTransaction>(d_parent->d_env->getRWTransaction());
582 }
583
584 explicit RWTransaction(TypedDBI* parent, std::shared_ptr<MDBRWTransaction> txn) : ReadonlyOperations<RWTransaction>(*this), d_parent(parent), d_txn(txn)
585 {
586 }
587
588
589 RWTransaction(RWTransaction&& rhs) :
590 ReadonlyOperations<RWTransaction>(*this),
591 d_parent(rhs.d_parent), d_txn(std::move(rhs.d_txn))
592 {
593 rhs.d_parent = 0;
594 }
595
596 // insert something, with possibly a specific id
597 uint32_t put(const T& t, uint32_t id, bool random_ids=false)
598 {
599 int flags = 0;
600 if(!id) {
601 if(random_ids) {
602 id = MDBGetRandomID(*d_txn, d_parent->d_main);
603 }
604 else {
605 id = MDBGetMaxID(*d_txn, d_parent->d_main) + 1;
606 flags = MDB_APPEND;
607 }
608 }
609 (*d_txn)->put(d_parent->d_main, id, serToString(t), flags);
610
611 #define insertMacro(N) std::get<N>(d_parent->d_tuple).put(*d_txn, t, id);
612 insertMacro(0);
613 insertMacro(1);
614 insertMacro(2);
615 insertMacro(3);
616 #undef insertMacro
617
618 return id;
619 }
620
621 // modify an item 'in place', plus update indexes
622 void modify(uint32_t id, std::function<void(T&)> func)
623 {
624 T t;
625 if(!this->get(id, t))
626 throw std::runtime_error("Could not modify id "+std::to_string(id));
627 func(t);
628
629 del(id); // this is the lazy way. We could test for changed index fields
630 put(t, id);
631 }
632
633 //! delete an item, and from indexes
634 void del(uint32_t id)
635 {
636 T t;
637 if(!this->get(id, t))
638 return;
639
640 (*d_txn)->del(d_parent->d_main, id);
641 clearIndex(id, t);
642 }
643
644 //! clear database & indexes (by hand!)
645 void clear()
646 {
647 auto cursor = (*d_txn)->getRWCursor(d_parent->d_main);
648 bool first = true;
649 MDBOutVal key, data;
650 while(!cursor.get(key, data, first ? MDB_FIRST : MDB_NEXT)) {
651 first = false;
652 T t;
653 serFromString(data.get<std::string>(), t);
654 clearIndex(key.get<uint32_t>(), t);
655 cursor.del();
656 }
657 }
658
659 //! commit this transaction
660 void commit()
661 {
662 (*d_txn)->commit();
663 }
664
665 //! abort this transaction
666 void abort()
667 {
668 (*d_txn)->abort();
669 }
670
671 typedef MDBRWCursor cursor_t;
672
673 std::shared_ptr<MDBRWTransaction> getTransactionHandle()
674 {
675 return d_txn;
676 }
677
678
679 private:
680 // clear this ID from all indexes
681 void clearIndex(uint32_t id, const T& t)
682 {
683 #define clearMacro(N) std::get<N>(d_parent->d_tuple).del(*d_txn, t, id);
684 clearMacro(0);
685 clearMacro(1);
686 clearMacro(2);
687 clearMacro(3);
688 #undef clearMacro
689 }
690
691 public:
692 TypedDBI* d_parent;
693 std::shared_ptr<MDBRWTransaction> d_txn;
694 };
695
696 //! Get an RW transaction
697 RWTransaction getRWTransaction()
698 {
699 return RWTransaction(this);
700 }
701
702 //! Get an RO transaction
703 ROTransaction getROTransaction()
704 {
705 return ROTransaction(this);
706 }
707
708 //! Get an RW transaction
709 RWTransaction getRWTransaction(std::shared_ptr<MDBRWTransaction> txn)
710 {
711 return RWTransaction(this, txn);
712 }
713
714 //! Get an RO transaction
715 ROTransaction getROTransaction(std::shared_ptr<MDBROTransaction> txn)
716 {
717 return ROTransaction(this, txn);
718 }
719
720 std::shared_ptr<MDBEnv> getEnv()
721 {
722 return d_env;
723 }
724
725 private:
726 std::shared_ptr<MDBEnv> d_env;
727 MDBDbi d_main;
728 std::string d_name;
729 };
730
731
732
733
734