ext/lmdb-safe/lmdb-safe.hh

   1 #pragma once
   2 #include <lmdb.h>
   3 #include <iostream>
   4 #include <fstream>
   5 #include <set>
   6 #include <map>
   7 #include <thread>
   8 #include <memory>
   9 #include <string>
  10 #include <string.h>
  11 #include <mutex>
  12 #include <vector>
  13 #include <algorithm>
  14
  15 // apple compiler somehow has string_view even in c++11!
  16 #if __cplusplus < 201703L && !defined(__APPLE__)
  17 #include <boost/version.hpp>
  18 #if BOOST_VERSION >= 106100
  19 #include <boost/utility/string_view.hpp>
  20 using boost::string_view;
  21 #else
  22 #include <boost/utility/string_ref.hpp>
  23 using string_view = boost::string_ref;
  24 #endif
  25 #else // C++17
  26 using std::string_view;
  27 #endif
  28
  29
  30 /* open issues:
  31  *
  32  * - missing convenience functions (string_view, string)
  33  */
  34
  35 /*
  36 The error strategy. Anything that "should never happen" turns into an exception. But things like 'duplicate entry' or 'no such key' are for you to deal with.
  37  */
  38
  39 /*
  40   Thread safety: we are as safe as lmdb. You can talk to MDBEnv from as many threads as you want
  41 */
  42
  43 /** MDBDbi is our only 'value type' object, as 1) a dbi is actually an integer
  44     and 2) per LMDB documentation, we never close it. */
  45 class MDBDbi
  46 {
  47 public:
  48   MDBDbi(): d_dbi(-1)
  49   {
  50   }
  51   explicit MDBDbi(MDB_env* env, MDB_txn* txn, string_view dbname, int flags);
  52
  53   operator const MDB_dbi&() const
  54   {
  55     return d_dbi;
  56   }
  57
  58   MDB_dbi d_dbi;
  59 };
  60
  61 class MDBRWTransactionImpl;
  62 class MDBROTransactionImpl;
  63
  64 using MDBROTransaction = std::unique_ptr<MDBROTransactionImpl>;
  65 using MDBRWTransaction = std::unique_ptr<MDBRWTransactionImpl>;
  66
  67 class MDBEnv
  68 {
  69 public:
  70   MDBEnv(const char* fname, int flags, int mode);
  71
  72   ~MDBEnv()
  73   {
  74     //    Only a single thread may call this function. All transactions, databases, and cursors must already be closed before calling this function
  75     mdb_env_close(d_env);
  76     // but, elsewhere, docs say database handles do not need to be closed?
  77   }
  78
  79   MDBDbi openDB(const string_view dbname, int flags);
  80
  81   MDBRWTransaction getRWTransaction();
  82   MDBROTransaction getROTransaction();
  83
  84   operator MDB_env*& ()
  85   {
  86     return d_env;
  87   }
  88   MDB_env* d_env;
  89
  90   int getRWTX();
  91   void incRWTX();
  92   void decRWTX();
  93   int getROTX();
  94   void incROTX();
  95   void decROTX();
  96 private:
  97   std::mutex d_openmut;
  98   std::mutex d_countmutex;
  99   std::map<std::thread::id, int> d_RWtransactionsOut;
 100   std::map<std::thread::id, int> d_ROtransactionsOut;
 101 };
 102
 103 std::shared_ptr<MDBEnv> getMDBEnv(const char* fname, int flags, int mode);
 104
 105
 106
 107 struct MDBOutVal
 108 {
 109   operator MDB_val&()
 110   {
 111     return d_mdbval;
 112   }
 113
 114   template <class T,
 115           typename std::enable_if<std::is_arithmetic<T>::value,
 116                                   T>::type* = nullptr> const
 117   T get()
 118   {
 119     T ret;
 120     if(d_mdbval.mv_size != sizeof(T))
 121       throw std::runtime_error("MDB data has wrong length for type");
 122
 123     memcpy(&ret, d_mdbval.mv_data, sizeof(T));
 124     return ret;
 125   }
 126
 127   template <class T,
 128             typename std::enable_if<std::is_class<T>::value,T>::type* = nullptr>
 129   T get() const;
 130
 131   template<class T>
 132   T get_struct() const
 133   {
 134     T ret;
 135     if(d_mdbval.mv_size != sizeof(T))
 136       throw std::runtime_error("MDB data has wrong length for type");
 137
 138     memcpy(&ret, d_mdbval.mv_data, sizeof(T));
 139     return ret;
 140   }
 141
 142   template<class T>
 143   const T* get_struct_ptr() const
 144   {
 145     if(d_mdbval.mv_size != sizeof(T))
 146       throw std::runtime_error("MDB data has wrong length for type");
 147
 148     return reinterpret_cast<const T*>(d_mdbval.mv_data);
 149   }
 150
 151
 152   MDB_val d_mdbval;
 153 };
 154
 155 template<> inline std::string MDBOutVal::get<std::string>() const
 156 {
 157   return std::string((char*)d_mdbval.mv_data, d_mdbval.mv_size);
 158 }
 159
 160 template<> inline string_view MDBOutVal::get<string_view>() const
 161 {
 162   return string_view((char*)d_mdbval.mv_data, d_mdbval.mv_size);
 163 }
 164
 165 class MDBInVal
 166 {
 167 public:
 168   MDBInVal(const MDBOutVal& rhs): d_mdbval(rhs.d_mdbval)
 169   {
 170   }
 171
 172   template <class T,
 173             typename std::enable_if<std::is_arithmetic<T>::value,
 174                                     T>::type* = nullptr>
 175   MDBInVal(T i)
 176   {
 177     memcpy(&d_memory[0], &i, sizeof(i));
 178     d_mdbval.mv_size = sizeof(T);
 179     d_mdbval.mv_data = d_memory;;
 180   }
 181
 182   MDBInVal(const char* s)
 183   {
 184     d_mdbval.mv_size = strlen(s);
 185     d_mdbval.mv_data = (void*)s;
 186   }
 187
 188   MDBInVal(const string_view& v)
 189   {
 190     d_mdbval.mv_size = v.size();
 191     d_mdbval.mv_data = (void*)&v[0];
 192   }
 193
 194   MDBInVal(const std::string& v)
 195   {
 196     d_mdbval.mv_size = v.size();
 197     d_mdbval.mv_data = (void*)&v[0];
 198   }
 199
 200
 201   template<typename T>
 202   static MDBInVal fromStruct(const T& t)
 203   {
 204     MDBInVal ret;
 205     ret.d_mdbval.mv_size = sizeof(T);
 206     ret.d_mdbval.mv_data = (void*)&t;
 207     return ret;
 208   }
 209
 210   operator MDB_val&()
 211   {
 212     return d_mdbval;
 213   }
 214   MDB_val d_mdbval;
 215 private:
 216   MDBInVal(){}
 217   char d_memory[sizeof(double)];
 218
 219 };
 220
 221
 222
 223
 224 class MDBROCursor;
 225
 226 class MDBROTransactionImpl
 227 {
 228 protected:
 229   MDBROTransactionImpl(MDBEnv *parent, MDB_txn *txn);
 230
 231 private:
 232   static MDB_txn *openROTransaction(MDBEnv *env, MDB_txn *parent, int flags=0);
 233
 234   MDBEnv* d_parent;
 235   std::vector<MDBROCursor*> d_cursors;
 236
 237 protected:
 238   MDB_txn* d_txn;
 239
 240   void closeROCursors();
 241
 242 public:
 243   explicit MDBROTransactionImpl(MDBEnv* parent, int flags=0);
 244
 245   MDBROTransactionImpl(const MDBROTransactionImpl& src) = delete;
 246   MDBROTransactionImpl &operator=(const MDBROTransactionImpl& src) = delete;
 247
 248   // The move constructor/operator cannot be made safe due to Object Slicing with MDBRWTransaction.
 249   MDBROTransactionImpl(MDBROTransactionImpl&& rhs) = delete;
 250   MDBROTransactionImpl &operator=(MDBROTransactionImpl &&rhs) = delete;
 251
 252   virtual ~MDBROTransactionImpl();
 253
 254   virtual void abort();
 255   virtual void commit();
 256
 257   int get(MDB_dbi dbi, const MDBInVal& key, MDBOutVal& val)
 258   {
 259     if(!d_txn)
 260       throw std::runtime_error("Attempt to use a closed RO transaction for get");
 261
 262     int rc = mdb_get(d_txn, dbi, const_cast<MDB_val*>(&key.d_mdbval),
 263                      const_cast<MDB_val*>(&val.d_mdbval));
 264     if(rc && rc != MDB_NOTFOUND)
 265       throw std::runtime_error("getting data: " + std::string(mdb_strerror(rc)));
 266
 267     return rc;
 268   }
 269
 270   int get(MDB_dbi dbi, const MDBInVal& key, string_view& val)
 271   {
 272     MDBOutVal out;
 273     int rc = get(dbi, key, out);
 274     if(!rc)
 275       val = out.get<string_view>();
 276     return rc;
 277   }
 278
 279
 280   // this is something you can do, readonly
 281   MDBDbi openDB(string_view dbname, int flags)
 282   {
 283     return MDBDbi( d_parent->d_env, d_txn, dbname, flags);
 284   }
 285
 286   MDBROCursor getCursor(const MDBDbi&);
 287   MDBROCursor getROCursor(const MDBDbi&);
 288
 289   operator MDB_txn*()
 290   {
 291     return d_txn;
 292   }
 293
 294   inline operator bool() const {
 295     return d_txn;
 296   }
 297
 298   inline MDBEnv &environment()
 299   {
 300     return *d_parent;
 301   }
 302 };
 303
 304 /*
 305    A cursor in a read-only transaction must be closed explicitly, before or after its transaction ends. It can be reused with mdb_cursor_renew() before finally closing it.
 306
 307    "If the parent transaction commits, the cursor must not be used again."
 308 */
 309
 310 template<class Transaction, class T>
 311 class MDBGenCursor
 312 {
 313 private:
 314   std::vector<T*> *d_registry;
 315   MDB_cursor* d_cursor{nullptr};
 316
 317 public:
 318   MDBGenCursor():
 319     d_registry(nullptr),
 320     d_cursor(nullptr)
 321   {
 322
 323   }
 324
 325   MDBGenCursor(std::vector<T*> &registry, MDB_cursor *cursor):
 326     d_registry(&registry),
 327     d_cursor(cursor)
 328   {
 329     registry.emplace_back(static_cast<T*>(this));
 330   }
 331
 332 private:
 333   void move_from(MDBGenCursor *src)
 334   {
 335     if (!d_registry) {
 336       return;
 337     }
 338
 339     auto iter = std::find(d_registry->begin(),
 340                           d_registry->end(),
 341                           src);
 342     if (iter != d_registry->end()) {
 343       *iter = static_cast<T*>(this);
 344     } else {
 345       d_registry->emplace_back(static_cast<T*>(this));
 346     }
 347   }
 348
 349 public:
 350   MDBGenCursor(const MDBGenCursor &src) = delete;
 351
 352   MDBGenCursor(MDBGenCursor &&src) noexcept:
 353     d_registry(src.d_registry),
 354     d_cursor(src.d_cursor)
 355   {
 356     move_from(&src);
 357     src.d_registry = nullptr;
 358     src.d_cursor = nullptr;
 359   }
 360
 361   MDBGenCursor &operator=(const MDBGenCursor &src) = delete;
 362
 363   MDBGenCursor &operator=(MDBGenCursor &&src) noexcept
 364   {
 365     d_registry = src.d_registry;
 366     d_cursor = src.d_cursor;
 367     move_from(&src);
 368     src.d_registry = nullptr;
 369     src.d_cursor = nullptr;
 370     return *this;
 371   }
 372
 373   ~MDBGenCursor()
 374   {
 375     close();
 376   }
 377
 378 public:
 379   int get(MDBOutVal& key, MDBOutVal& data, MDB_cursor_op op)
 380   {
 381     int rc = mdb_cursor_get(d_cursor, &key.d_mdbval, &data.d_mdbval, op);
 382     if(rc && rc != MDB_NOTFOUND)
 383        throw std::runtime_error("Unable to get from cursor: " + std::string(mdb_strerror(rc)));
 384     return rc;
 385   }
 386
 387   int find(const MDBInVal& in, MDBOutVal& key, MDBOutVal& data)
 388   {
 389     key.d_mdbval = in.d_mdbval;
 390     int rc=mdb_cursor_get(d_cursor, const_cast<MDB_val*>(&key.d_mdbval), &data.d_mdbval, MDB_SET);
 391     if(rc && rc != MDB_NOTFOUND)
 392        throw std::runtime_error("Unable to find from cursor: " + std::string(mdb_strerror(rc)));
 393     return rc;
 394   }
 395
 396   int lower_bound(const MDBInVal& in, MDBOutVal& key, MDBOutVal& data)
 397   {
 398     key.d_mdbval = in.d_mdbval;
 399
 400     int rc = mdb_cursor_get(d_cursor, const_cast<MDB_val*>(&key.d_mdbval), &data.d_mdbval, MDB_SET_RANGE);
 401     if(rc && rc != MDB_NOTFOUND)
 402        throw std::runtime_error("Unable to lower_bound from cursor: " + std::string(mdb_strerror(rc)));
 403     return rc;
 404   }
 405
 406
 407   int nextprev(MDBOutVal& key, MDBOutVal& data, MDB_cursor_op op)
 408   {
 409     int rc = mdb_cursor_get(d_cursor, const_cast<MDB_val*>(&key.d_mdbval), &data.d_mdbval, op);
 410     if(rc && rc != MDB_NOTFOUND)
 411        throw std::runtime_error("Unable to prevnext from cursor: " + std::string(mdb_strerror(rc)));
 412     return rc;
 413   }
 414
 415   int next(MDBOutVal& key, MDBOutVal& data)
 416   {
 417     return nextprev(key, data, MDB_NEXT);
 418   }
 419
 420   int prev(MDBOutVal& key, MDBOutVal& data)
 421   {
 422     return nextprev(key, data, MDB_PREV);
 423   }
 424
 425   int currentlast(MDBOutVal& key, MDBOutVal& data, MDB_cursor_op op)
 426   {
 427     int rc = mdb_cursor_get(d_cursor, const_cast<MDB_val*>(&key.d_mdbval), &data.d_mdbval, op);
 428     if(rc && rc != MDB_NOTFOUND)
 429        throw std::runtime_error("Unable to next from cursor: " + std::string(mdb_strerror(rc)));
 430     return rc;
 431   }
 432
 433   int current(MDBOutVal& key, MDBOutVal& data)
 434   {
 435     return currentlast(key, data, MDB_GET_CURRENT);
 436   }
 437   int last(MDBOutVal& key, MDBOutVal& data)
 438   {
 439     return currentlast(key, data, MDB_LAST);
 440   }
 441   int first(MDBOutVal& key, MDBOutVal& data)
 442   {
 443     return currentlast(key, data, MDB_FIRST);
 444   }
 445
 446   operator MDB_cursor*()
 447   {
 448     return d_cursor;
 449   }
 450
 451   operator bool() const
 452   {
 453     return d_cursor;
 454   }
 455
 456   void close()
 457   {
 458     if (d_registry) {
 459       auto iter = std::find(d_registry->begin(),
 460                             d_registry->end(),
 461                             static_cast<T*>(this));
 462       if (iter != d_registry->end()) {
 463         d_registry->erase(iter);
 464       }
 465       d_registry = nullptr;
 466     }
 467     if (d_cursor) {
 468       mdb_cursor_close(d_cursor);
 469       d_cursor = nullptr;
 470     }
 471   }
 472 };
 473
 474 class MDBROCursor : public MDBGenCursor<MDBROTransactionImpl, MDBROCursor>
 475 {
 476 public:
 477   MDBROCursor() = default;
 478   using MDBGenCursor<MDBROTransactionImpl, MDBROCursor>::MDBGenCursor;
 479   MDBROCursor(const MDBROCursor &src) = delete;
 480   MDBROCursor(MDBROCursor &&src) = default;
 481   MDBROCursor &operator=(const MDBROCursor &src) = delete;
 482   MDBROCursor &operator=(MDBROCursor &&src) = default;
 483   ~MDBROCursor() = default;
 484
 485 };
 486
 487 class MDBRWCursor;
 488
 489 class MDBRWTransactionImpl: public MDBROTransactionImpl
 490 {
 491 protected:
 492   MDBRWTransactionImpl(MDBEnv* parent, MDB_txn* txn);
 493
 494 private:
 495   static MDB_txn *openRWTransaction(MDBEnv* env, MDB_txn *parent, int flags);
 496
 497 private:
 498   std::vector<MDBRWCursor*> d_rw_cursors;
 499
 500   void closeRWCursors();
 501   inline void closeRORWCursors() {
 502     closeROCursors();
 503     closeRWCursors();
 504   }
 505
 506 public:
 507   explicit MDBRWTransactionImpl(MDBEnv* parent, int flags=0);
 508
 509   MDBRWTransactionImpl(const MDBRWTransactionImpl& rhs) = delete;
 510   MDBRWTransactionImpl(MDBRWTransactionImpl&& rhs) = delete;
 511   MDBRWTransactionImpl &operator=(const MDBRWTransactionImpl& rhs) = delete;
 512   MDBRWTransactionImpl &operator=(MDBRWTransactionImpl&& rhs) = delete;
 513
 514   ~MDBRWTransactionImpl() override;
 515
 516   void commit() override;
 517   void abort() override;
 518
 519   void clear(MDB_dbi dbi);
 520
 521   void put(MDB_dbi dbi, const MDBInVal& key, const MDBInVal& val, int flags=0)
 522   {
 523     if(!d_txn)
 524       throw std::runtime_error("Attempt to use a closed RW transaction for put");
 525     int rc;
 526     if((rc=mdb_put(d_txn, dbi,
 527                    const_cast<MDB_val*>(&key.d_mdbval),
 528                    const_cast<MDB_val*>(&val.d_mdbval), flags)))
 529       throw std::runtime_error("putting data: " + std::string(mdb_strerror(rc)));
 530   }
 531
 532
 533   int del(MDBDbi& dbi, const MDBInVal& key, const MDBInVal& val)
 534   {
 535     int rc;
 536     rc=mdb_del(d_txn, dbi, (MDB_val*)&key.d_mdbval, (MDB_val*)&val.d_mdbval);
 537     if(rc && rc != MDB_NOTFOUND)
 538       throw std::runtime_error("deleting data: " + std::string(mdb_strerror(rc)));
 539     return rc;
 540   }
 541
 542   int del(MDBDbi& dbi, const MDBInVal& key)
 543   {
 544     int rc;
 545     rc=mdb_del(d_txn, dbi, (MDB_val*)&key.d_mdbval, 0);
 546     if(rc && rc != MDB_NOTFOUND)
 547       throw std::runtime_error("deleting data: " + std::string(mdb_strerror(rc)));
 548     return rc;
 549   }
 550
 551
 552   int get(MDBDbi& dbi, const MDBInVal& key, MDBOutVal& val)
 553   {
 554     if(!d_txn)
 555       throw std::runtime_error("Attempt to use a closed RW transaction for get");
 556
 557     int rc = mdb_get(d_txn, dbi, const_cast<MDB_val*>(&key.d_mdbval),
 558                      const_cast<MDB_val*>(&val.d_mdbval));
 559     if(rc && rc != MDB_NOTFOUND)
 560       throw std::runtime_error("getting data: " + std::string(mdb_strerror(rc)));
 561     return rc;
 562   }
 563
 564   int get(MDBDbi& dbi, const MDBInVal& key, string_view& val)
 565   {
 566     MDBOutVal out;
 567     int rc = get(dbi, key, out);
 568     if(!rc)
 569       val = out.get<string_view>();
 570     return rc;
 571   }
 572
 573   MDBDbi openDB(string_view dbname, int flags)
 574   {
 575     return MDBDbi(environment().d_env, d_txn, dbname, flags);
 576   }
 577
 578   MDBRWCursor getRWCursor(const MDBDbi&);
 579   MDBRWCursor getCursor(const MDBDbi&);
 580
 581   MDBRWTransaction getRWTransaction();
 582   MDBROTransaction getROTransaction();
 583 };
 584
 585 /* "A cursor in a write-transaction can be closed before its transaction ends, and will otherwise be closed when its transaction ends"
 586    This is a problem for us since it may means we are closing the cursor twice, which is bad
 587 */
 588 class MDBRWCursor : public MDBGenCursor<MDBRWTransactionImpl, MDBRWCursor>
 589 {
 590 public:
 591   MDBRWCursor() = default;
 592   using MDBGenCursor<MDBRWTransactionImpl, MDBRWCursor>::MDBGenCursor;
 593   MDBRWCursor(const MDBRWCursor &src) = delete;
 594   MDBRWCursor(MDBRWCursor &&src) = default;
 595   MDBRWCursor &operator=(const MDBRWCursor &src) = delete;
 596   MDBRWCursor &operator=(MDBRWCursor &&src) = default;
 597   ~MDBRWCursor() = default;
 598
 599   void put(const MDBOutVal& key, const MDBInVal& data)
 600   {
 601     int rc = mdb_cursor_put(*this,
 602                             const_cast<MDB_val*>(&key.d_mdbval),
 603                             const_cast<MDB_val*>(&data.d_mdbval), MDB_CURRENT);
 604     if(rc)
 605       throw std::runtime_error("mdb_cursor_put: " + std::string(mdb_strerror(rc)));
 606   }
 607
 608
 609   int put(const MDBOutVal& key, const MDBOutVal& data, int flags=0)
 610   {
 611     // XXX check errors
 612     return mdb_cursor_put(*this,
 613                           const_cast<MDB_val*>(&key.d_mdbval),
 614                           const_cast<MDB_val*>(&data.d_mdbval), flags);
 615   }
 616
 617   int del(int flags=0)
 618   {
 619     return mdb_cursor_del(*this, flags);
 620   }
 621
 622 };
 623