ext/lmdb-safe/lmdb-safe.hh

   1 #pragma once
   2 #include <lmdb.h>
   3 #include <iostream>
   4 #include <fstream>
   5 #include <set>
   6 #include <map>
   7 #include <thread>
   8 #include <memory>
   9 #include <string>
  10 #include <string.h>
  11 #include <mutex>
  12 #include <vector>
  13 #include <algorithm>
  14
  15 // apple compiler somehow has string_view even in c++11!
  16 #if __cplusplus < 201703L && !defined(__APPLE__)
  17 #include <boost/version.hpp>
  18 #if BOOST_VERSION >= 106100
  19 #include <boost/utility/string_view.hpp>
  20 using boost::string_view;
  21 #else
  22 #include <boost/utility/string_ref.hpp>
  23 using string_view = boost::string_ref;
  24 #endif
  25 #else // C++17
  26 using std::string_view;
  27 #endif
  28
  29
  30 /* open issues:
  31  *
  32  * - missing convenience functions (string_view, string)
  33  */
  34
  35 /*
  36 The error strategy. Anything that "should never happen" turns into an exception. But things like 'duplicate entry' or 'no such key' are for you to deal with.
  37  */
  38
  39 /*
  40   Thread safety: we are as safe as lmdb. You can talk to MDBEnv from as many threads as you want
  41 */
  42
  43 /** MDBDbi is our only 'value type' object, as 1) a dbi is actually an integer
  44     and 2) per LMDB documentation, we never close it. */
  45 class MDBDbi
  46 {
  47 public:
  48   MDBDbi()
  49   {
  50     d_dbi = -1;
  51   }
  52   explicit MDBDbi(MDB_env* env, MDB_txn* txn, string_view dbname, int flags);
  53
  54   operator const MDB_dbi&() const
  55   {
  56     return d_dbi;
  57   }
  58
  59   MDB_dbi d_dbi;
  60 };
  61
  62 class MDBRWTransactionImpl;
  63 class MDBROTransactionImpl;
  64
  65 using MDBROTransaction = std::unique_ptr<MDBROTransactionImpl>;
  66 using MDBRWTransaction = std::unique_ptr<MDBRWTransactionImpl>;
  67
  68 class MDBEnv
  69 {
  70 public:
  71   MDBEnv(const char* fname, int flags, int mode);
  72
  73   ~MDBEnv()
  74   {
  75     //    Only a single thread may call this function. All transactions, databases, and cursors must already be closed before calling this function
  76     mdb_env_close(d_env);
  77     // but, elsewhere, docs say database handles do not need to be closed?
  78   }
  79
  80   MDBDbi openDB(const string_view dbname, int flags);
  81
  82   MDBRWTransaction getRWTransaction();
  83   MDBROTransaction getROTransaction();
  84
  85   operator MDB_env*& ()
  86   {
  87     return d_env;
  88   }
  89   MDB_env* d_env;
  90
  91   int getRWTX();
  92   void incRWTX();
  93   void decRWTX();
  94   int getROTX();
  95   void incROTX();
  96   void decROTX();
  97 private:
  98   std::mutex d_openmut;
  99   std::mutex d_countmutex;
 100   std::map<std::thread::id, int> d_RWtransactionsOut;
 101   std::map<std::thread::id, int> d_ROtransactionsOut;
 102 };
 103
 104 std::shared_ptr<MDBEnv> getMDBEnv(const char* fname, int flags, int mode);
 105
 106
 107
 108 struct MDBOutVal
 109 {
 110   operator MDB_val&()
 111   {
 112     return d_mdbval;
 113   }
 114
 115   template <class T,
 116           typename std::enable_if<std::is_arithmetic<T>::value,
 117                                   T>::type* = nullptr> const
 118   T get()
 119   {
 120     T ret;
 121     if(d_mdbval.mv_size != sizeof(T))
 122       throw std::runtime_error("MDB data has wrong length for type");
 123
 124     memcpy(&ret, d_mdbval.mv_data, sizeof(T));
 125     return ret;
 126   }
 127
 128   template <class T,
 129             typename std::enable_if<std::is_class<T>::value,T>::type* = nullptr>
 130   T get() const;
 131
 132   template<class T>
 133   T get_struct() const
 134   {
 135     T ret;
 136     if(d_mdbval.mv_size != sizeof(T))
 137       throw std::runtime_error("MDB data has wrong length for type");
 138
 139     memcpy(&ret, d_mdbval.mv_data, sizeof(T));
 140     return ret;
 141   }
 142
 143   template<class T>
 144   const T* get_struct_ptr() const
 145   {
 146     if(d_mdbval.mv_size != sizeof(T))
 147       throw std::runtime_error("MDB data has wrong length for type");
 148
 149     return reinterpret_cast<const T*>(d_mdbval.mv_data);
 150   }
 151
 152
 153   MDB_val d_mdbval;
 154 };
 155
 156 template<> inline std::string MDBOutVal::get<std::string>() const
 157 {
 158   return std::string((char*)d_mdbval.mv_data, d_mdbval.mv_size);
 159 }
 160
 161 template<> inline string_view MDBOutVal::get<string_view>() const
 162 {
 163   return string_view((char*)d_mdbval.mv_data, d_mdbval.mv_size);
 164 }
 165
 166 class MDBInVal
 167 {
 168 public:
 169   MDBInVal(const MDBOutVal& rhs)
 170   {
 171     d_mdbval = rhs.d_mdbval;
 172   }
 173
 174   template <class T,
 175             typename std::enable_if<std::is_arithmetic<T>::value,
 176                                     T>::type* = nullptr>
 177   MDBInVal(T i)
 178   {
 179     memcpy(&d_memory[0], &i, sizeof(i));
 180     d_mdbval.mv_size = sizeof(T);
 181     d_mdbval.mv_data = d_memory;;
 182   }
 183
 184   MDBInVal(const char* s)
 185   {
 186     d_mdbval.mv_size = strlen(s);
 187     d_mdbval.mv_data = (void*)s;
 188   }
 189
 190   MDBInVal(const string_view& v)
 191   {
 192     d_mdbval.mv_size = v.size();
 193     d_mdbval.mv_data = (void*)&v[0];
 194   }
 195
 196   MDBInVal(const std::string& v)
 197   {
 198     d_mdbval.mv_size = v.size();
 199     d_mdbval.mv_data = (void*)&v[0];
 200   }
 201
 202
 203   template<typename T>
 204   static MDBInVal fromStruct(const T& t)
 205   {
 206     MDBInVal ret;
 207     ret.d_mdbval.mv_size = sizeof(T);
 208     ret.d_mdbval.mv_data = (void*)&t;
 209     return ret;
 210   }
 211
 212   operator MDB_val&()
 213   {
 214     return d_mdbval;
 215   }
 216   MDB_val d_mdbval;
 217 private:
 218   MDBInVal(){}
 219   char d_memory[sizeof(double)];
 220
 221 };
 222
 223
 224
 225
 226 class MDBROCursor;
 227
 228 class MDBROTransactionImpl
 229 {
 230 protected:
 231   MDBROTransactionImpl(MDBEnv *parent, MDB_txn *txn);
 232
 233 private:
 234   static MDB_txn *openROTransaction(MDBEnv *env, MDB_txn *parent, int flags=0);
 235
 236   MDBEnv* d_parent;
 237   std::vector<MDBROCursor*> d_cursors;
 238
 239 protected:
 240   MDB_txn* d_txn;
 241
 242   void closeROCursors();
 243
 244 public:
 245   explicit MDBROTransactionImpl(MDBEnv* parent, int flags=0);
 246
 247   MDBROTransactionImpl(const MDBROTransactionImpl& src) = delete;
 248   MDBROTransactionImpl &operator=(const MDBROTransactionImpl& src) = delete;
 249
 250   // The move constructor/operator cannot be made safe due to Object Slicing with MDBRWTransaction.
 251   MDBROTransactionImpl(MDBROTransactionImpl&& rhs) = delete;
 252   MDBROTransactionImpl &operator=(MDBROTransactionImpl &&rhs) = delete;
 253
 254   virtual ~MDBROTransactionImpl();
 255
 256   virtual void abort();
 257   virtual void commit();
 258
 259   int get(MDB_dbi dbi, const MDBInVal& key, MDBOutVal& val)
 260   {
 261     if(!d_txn)
 262       throw std::runtime_error("Attempt to use a closed RO transaction for get");
 263
 264     int rc = mdb_get(d_txn, dbi, const_cast<MDB_val*>(&key.d_mdbval),
 265                      const_cast<MDB_val*>(&val.d_mdbval));
 266     if(rc && rc != MDB_NOTFOUND)
 267       throw std::runtime_error("getting data: " + std::string(mdb_strerror(rc)));
 268
 269     return rc;
 270   }
 271
 272   int get(MDB_dbi dbi, const MDBInVal& key, string_view& val)
 273   {
 274     MDBOutVal out;
 275     int rc = get(dbi, key, out);
 276     if(!rc)
 277       val = out.get<string_view>();
 278     return rc;
 279   }
 280
 281
 282   // this is something you can do, readonly
 283   MDBDbi openDB(string_view dbname, int flags)
 284   {
 285     return MDBDbi( d_parent->d_env, d_txn, dbname, flags);
 286   }
 287
 288   MDBROCursor getCursor(const MDBDbi&);
 289   MDBROCursor getROCursor(const MDBDbi&);
 290
 291   operator MDB_txn*()
 292   {
 293     return d_txn;
 294   }
 295
 296   inline operator bool() const {
 297     return d_txn;
 298   }
 299
 300   inline MDBEnv &environment()
 301   {
 302     return *d_parent;
 303   }
 304 };
 305
 306 /*
 307    A cursor in a read-only transaction must be closed explicitly, before or after its transaction ends. It can be reused with mdb_cursor_renew() before finally closing it.
 308
 309    "If the parent transaction commits, the cursor must not be used again."
 310 */
 311
 312 template<class Transaction, class T>
 313 class MDBGenCursor
 314 {
 315 private:
 316   std::vector<T*> *d_registry;
 317   MDB_cursor* d_cursor;
 318
 319 public:
 320   MDBGenCursor():
 321     d_registry(nullptr),
 322     d_cursor(nullptr)
 323   {
 324
 325   }
 326
 327   MDBGenCursor(std::vector<T*> &registry, MDB_cursor *cursor):
 328     d_registry(&registry),
 329     d_cursor(cursor)
 330   {
 331     registry.emplace_back(static_cast<T*>(this));
 332   }
 333
 334 private:
 335   void move_from(MDBGenCursor *src)
 336   {
 337     if (!d_registry) {
 338       return;
 339     }
 340
 341     auto iter = std::find(d_registry->begin(),
 342                           d_registry->end(),
 343                           src);
 344     if (iter != d_registry->end()) {
 345       *iter = static_cast<T*>(this);
 346     } else {
 347       d_registry->emplace_back(static_cast<T*>(this));
 348     }
 349   }
 350
 351 public:
 352   MDBGenCursor(const MDBGenCursor &src) = delete;
 353
 354   MDBGenCursor(MDBGenCursor &&src) noexcept:
 355     d_registry(src.d_registry),
 356     d_cursor(src.d_cursor)
 357   {
 358     move_from(&src);
 359     src.d_registry = nullptr;
 360     src.d_cursor = nullptr;
 361   }
 362
 363   MDBGenCursor &operator=(const MDBGenCursor &src) = delete;
 364
 365   MDBGenCursor &operator=(MDBGenCursor &&src) noexcept
 366   {
 367     d_registry = src.d_registry;
 368     d_cursor = src.d_cursor;
 369     move_from(&src);
 370     src.d_registry = nullptr;
 371     src.d_cursor = nullptr;
 372     return *this;
 373   }
 374
 375   ~MDBGenCursor()
 376   {
 377     close();
 378   }
 379
 380 public:
 381   int get(MDBOutVal& key, MDBOutVal& data, MDB_cursor_op op)
 382   {
 383     int rc = mdb_cursor_get(d_cursor, &key.d_mdbval, &data.d_mdbval, op);
 384     if(rc && rc != MDB_NOTFOUND)
 385        throw std::runtime_error("Unable to get from cursor: " + std::string(mdb_strerror(rc)));
 386     return rc;
 387   }
 388
 389   int find(const MDBInVal& in, MDBOutVal& key, MDBOutVal& data)
 390   {
 391     key.d_mdbval = in.d_mdbval;
 392     int rc=mdb_cursor_get(d_cursor, const_cast<MDB_val*>(&key.d_mdbval), &data.d_mdbval, MDB_SET);
 393     if(rc && rc != MDB_NOTFOUND)
 394        throw std::runtime_error("Unable to find from cursor: " + std::string(mdb_strerror(rc)));
 395     return rc;
 396   }
 397
 398   int lower_bound(const MDBInVal& in, MDBOutVal& key, MDBOutVal& data)
 399   {
 400     key.d_mdbval = in.d_mdbval;
 401
 402     int rc = mdb_cursor_get(d_cursor, const_cast<MDB_val*>(&key.d_mdbval), &data.d_mdbval, MDB_SET_RANGE);
 403     if(rc && rc != MDB_NOTFOUND)
 404        throw std::runtime_error("Unable to lower_bound from cursor: " + std::string(mdb_strerror(rc)));
 405     return rc;
 406   }
 407
 408
 409   int nextprev(MDBOutVal& key, MDBOutVal& data, MDB_cursor_op op)
 410   {
 411     int rc = mdb_cursor_get(d_cursor, const_cast<MDB_val*>(&key.d_mdbval), &data.d_mdbval, op);
 412     if(rc && rc != MDB_NOTFOUND)
 413        throw std::runtime_error("Unable to prevnext from cursor: " + std::string(mdb_strerror(rc)));
 414     return rc;
 415   }
 416
 417   int next(MDBOutVal& key, MDBOutVal& data)
 418   {
 419     return nextprev(key, data, MDB_NEXT);
 420   }
 421
 422   int prev(MDBOutVal& key, MDBOutVal& data)
 423   {
 424     return nextprev(key, data, MDB_PREV);
 425   }
 426
 427   int currentlast(MDBOutVal& key, MDBOutVal& data, MDB_cursor_op op)
 428   {
 429     int rc = mdb_cursor_get(d_cursor, const_cast<MDB_val*>(&key.d_mdbval), &data.d_mdbval, op);
 430     if(rc && rc != MDB_NOTFOUND)
 431        throw std::runtime_error("Unable to next from cursor: " + std::string(mdb_strerror(rc)));
 432     return rc;
 433   }
 434
 435   int current(MDBOutVal& key, MDBOutVal& data)
 436   {
 437     return currentlast(key, data, MDB_GET_CURRENT);
 438   }
 439   int last(MDBOutVal& key, MDBOutVal& data)
 440   {
 441     return currentlast(key, data, MDB_LAST);
 442   }
 443   int first(MDBOutVal& key, MDBOutVal& data)
 444   {
 445     return currentlast(key, data, MDB_FIRST);
 446   }
 447
 448   operator MDB_cursor*()
 449   {
 450     return d_cursor;
 451   }
 452
 453   operator bool() const
 454   {
 455     return d_cursor;
 456   }
 457
 458   void close()
 459   {
 460     if (d_registry) {
 461       auto iter = std::find(d_registry->begin(),
 462                             d_registry->end(),
 463                             static_cast<T*>(this));
 464       if (iter != d_registry->end()) {
 465         d_registry->erase(iter);
 466       }
 467       d_registry = nullptr;
 468     }
 469     if (d_cursor) {
 470       mdb_cursor_close(d_cursor);
 471       d_cursor = nullptr;
 472     }
 473   }
 474 };
 475
 476 class MDBROCursor : public MDBGenCursor<MDBROTransactionImpl, MDBROCursor>
 477 {
 478 public:
 479   MDBROCursor() = default;
 480   using MDBGenCursor<MDBROTransactionImpl, MDBROCursor>::MDBGenCursor;
 481   MDBROCursor(const MDBROCursor &src) = delete;
 482   MDBROCursor(MDBROCursor &&src) = default;
 483   MDBROCursor &operator=(const MDBROCursor &src) = delete;
 484   MDBROCursor &operator=(MDBROCursor &&src) = default;
 485   ~MDBROCursor() = default;
 486
 487 };
 488
 489 class MDBRWCursor;
 490
 491 class MDBRWTransactionImpl: public MDBROTransactionImpl
 492 {
 493 protected:
 494   MDBRWTransactionImpl(MDBEnv* parent, MDB_txn* txn);
 495
 496 private:
 497   static MDB_txn *openRWTransaction(MDBEnv* env, MDB_txn *parent, int flags);
 498
 499 private:
 500   std::vector<MDBRWCursor*> d_rw_cursors;
 501
 502   void closeRWCursors();
 503   inline void closeRORWCursors() {
 504     closeROCursors();
 505     closeRWCursors();
 506   }
 507
 508 public:
 509   explicit MDBRWTransactionImpl(MDBEnv* parent, int flags=0);
 510
 511   MDBRWTransactionImpl(const MDBRWTransactionImpl& rhs) = delete;
 512   MDBRWTransactionImpl(MDBRWTransactionImpl&& rhs) = delete;
 513   MDBRWTransactionImpl &operator=(const MDBRWTransactionImpl& rhs) = delete;
 514   MDBRWTransactionImpl &operator=(MDBRWTransactionImpl&& rhs) = delete;
 515
 516   ~MDBRWTransactionImpl() override;
 517
 518   void commit() override;
 519   void abort() override;
 520
 521   void clear(MDB_dbi dbi);
 522
 523   void put(MDB_dbi dbi, const MDBInVal& key, const MDBInVal& val, int flags=0)
 524   {
 525     if(!d_txn)
 526       throw std::runtime_error("Attempt to use a closed RW transaction for put");
 527     int rc;
 528     if((rc=mdb_put(d_txn, dbi,
 529                    const_cast<MDB_val*>(&key.d_mdbval),
 530                    const_cast<MDB_val*>(&val.d_mdbval), flags)))
 531       throw std::runtime_error("putting data: " + std::string(mdb_strerror(rc)));
 532   }
 533
 534
 535   int del(MDBDbi& dbi, const MDBInVal& key, const MDBInVal& val)
 536   {
 537     int rc;
 538     rc=mdb_del(d_txn, dbi, (MDB_val*)&key.d_mdbval, (MDB_val*)&val.d_mdbval);
 539     if(rc && rc != MDB_NOTFOUND)
 540       throw std::runtime_error("deleting data: " + std::string(mdb_strerror(rc)));
 541     return rc;
 542   }
 543
 544   int del(MDBDbi& dbi, const MDBInVal& key)
 545   {
 546     int rc;
 547     rc=mdb_del(d_txn, dbi, (MDB_val*)&key.d_mdbval, 0);
 548     if(rc && rc != MDB_NOTFOUND)
 549       throw std::runtime_error("deleting data: " + std::string(mdb_strerror(rc)));
 550     return rc;
 551   }
 552
 553
 554   int get(MDBDbi& dbi, const MDBInVal& key, MDBOutVal& val)
 555   {
 556     if(!d_txn)
 557       throw std::runtime_error("Attempt to use a closed RW transaction for get");
 558
 559     int rc = mdb_get(d_txn, dbi, const_cast<MDB_val*>(&key.d_mdbval),
 560                      const_cast<MDB_val*>(&val.d_mdbval));
 561     if(rc && rc != MDB_NOTFOUND)
 562       throw std::runtime_error("getting data: " + std::string(mdb_strerror(rc)));
 563     return rc;
 564   }
 565
 566   int get(MDBDbi& dbi, const MDBInVal& key, string_view& val)
 567   {
 568     MDBOutVal out;
 569     int rc = get(dbi, key, out);
 570     if(!rc)
 571       val = out.get<string_view>();
 572     return rc;
 573   }
 574
 575   MDBDbi openDB(string_view dbname, int flags)
 576   {
 577     return MDBDbi(environment().d_env, d_txn, dbname, flags);
 578   }
 579
 580   MDBRWCursor getRWCursor(const MDBDbi&);
 581   MDBRWCursor getCursor(const MDBDbi&);
 582
 583   MDBRWTransaction getRWTransaction();
 584   MDBROTransaction getROTransaction();
 585 };
 586
 587 /* "A cursor in a write-transaction can be closed before its transaction ends, and will otherwise be closed when its transaction ends"
 588    This is a problem for us since it may means we are closing the cursor twice, which is bad
 589 */
 590 class MDBRWCursor : public MDBGenCursor<MDBRWTransactionImpl, MDBRWCursor>
 591 {
 592 public:
 593   MDBRWCursor() = default;
 594   using MDBGenCursor<MDBRWTransactionImpl, MDBRWCursor>::MDBGenCursor;
 595   MDBRWCursor(const MDBRWCursor &src) = delete;
 596   MDBRWCursor(MDBRWCursor &&src) = default;
 597   MDBRWCursor &operator=(const MDBRWCursor &src) = delete;
 598   MDBRWCursor &operator=(MDBRWCursor &&src) = default;
 599   ~MDBRWCursor() = default;
 600
 601   void put(const MDBOutVal& key, const MDBInVal& data)
 602   {
 603     int rc = mdb_cursor_put(*this,
 604                             const_cast<MDB_val*>(&key.d_mdbval),
 605                             const_cast<MDB_val*>(&data.d_mdbval), MDB_CURRENT);
 606     if(rc)
 607       throw std::runtime_error("mdb_cursor_put: " + std::string(mdb_strerror(rc)));
 608   }
 609
 610
 611   int put(const MDBOutVal& key, const MDBOutVal& data, int flags=0)
 612   {
 613     // XXX check errors
 614     return mdb_cursor_put(*this,
 615                           const_cast<MDB_val*>(&key.d_mdbval),
 616                           const_cast<MDB_val*>(&data.d_mdbval), flags);
 617   }
 618
 619   int del(int flags=0)
 620   {
 621     return mdb_cursor_del(*this, flags);
 622   }
 623
 624 };
 625