diff --git a/Makefile b/Makefile index 8474858..203c4bd 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ CFLAGS:= -Wall -O2 -MMD -MP -ggdb PROGRAMS = lmdb-test basic-example scale-example multi-example rel-example \ - resize-example + resize-example lmdb-typed all: $(PROGRAMS) @@ -21,19 +21,22 @@ clean: lmdb-test: lmdb-test.o lmdb-safe.o - g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS) #-lasan + g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) #-lasan basic-example: basic-example.o lmdb-safe.o - g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS) + g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) scale-example: scale-example.o lmdb-safe.o - g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS) + g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) multi-example: multi-example.o lmdb-safe.o - g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS) + g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) rel-example: rel-example.o lmdb-safe.o - g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS) -lboost_serialization + g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) -lboost_serialization resize-example: resize-example.o lmdb-safe.o - g++ $(CXXVERSIONGLAG) $^ -o $@ -pthread $(LIBS) + g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) + +lmdb-typed: lmdb-typed.o lmdb-safe.o + g++ $(CXXVERSIONFLAG) $^ -o $@ -pthread $(LIBS) -lboost_serialization diff --git a/lmdb-safe.cc b/lmdb-safe.cc index fcb0d7e..6c427e5 100644 --- a/lmdb-safe.cc +++ b/lmdb-safe.cc @@ -13,11 +13,11 @@ static string MDBError(int rc) return mdb_strerror(rc); } -MDBDbi::MDBDbi(MDB_env* env, MDB_txn* txn, const char* dbname, int flags) +MDBDbi::MDBDbi(MDB_env* env, MDB_txn* txn, const string_view dbname, int flags) { // A transaction that uses this function must finish (either commit or abort) before any other transaction in the process may use this function. - int rc = mdb_dbi_open(txn, dbname, flags, &d_dbi); + int rc = mdb_dbi_open(txn, &dbname[0], flags, &d_dbi); if(rc) throw std::runtime_error("Unable to open named database: " + MDBError(rc)); @@ -27,7 +27,7 @@ MDBDbi::MDBDbi(MDB_env* env, MDB_txn* txn, const char* dbname, int flags) MDBEnv::MDBEnv(const char* fname, int flags, int mode) { mdb_env_create(&d_env); - if(mdb_env_set_mapsize(d_env, 4ULL*4096*244140ULL)) // 4GB + if(mdb_env_set_mapsize(d_env, 16ULL*4096*244140ULL)) // 4GB throw std::runtime_error("setting map size"); /* Various other options may also need to be set before opening the handle, e.g. mdb_env_set_mapsize(), mdb_env_set_maxreaders(), mdb_env_set_maxdbs(), @@ -128,7 +128,7 @@ std::shared_ptr getMDBEnv(const char* fname, int flags, int mode) } -MDBDbi MDBEnv::openDB(const char* dbname, int flags) +MDBDbi MDBEnv::openDB(const string_view dbname, int flags) { unsigned int envflags; mdb_env_get_flags(d_env, &envflags); @@ -139,7 +139,7 @@ MDBDbi MDBEnv::openDB(const char* dbname, int flags) if(!(envflags & MDB_RDONLY)) { auto rwt = getRWTransaction(); - MDBDbi ret = rwt.openDB(dbname, flags); + MDBDbi ret = rwt.openDB(&dbname[0], flags); rwt.commit(); return ret; } diff --git a/lmdb-safe.hh b/lmdb-safe.hh index db38e7f..7807e8a 100644 --- a/lmdb-safe.hh +++ b/lmdb-safe.hh @@ -40,7 +40,7 @@ public: { d_dbi = -1; } - explicit MDBDbi(MDB_env* env, MDB_txn* txn, const char* dbname, int flags); + explicit MDBDbi(MDB_env* env, MDB_txn* txn, string_view dbname, int flags); operator const MDB_dbi&() const { @@ -65,7 +65,7 @@ public: // but, elsewhere, docs say database handles do not need to be closed? } - MDBDbi openDB(const char* dbname, int flags); + MDBDbi openDB(const string_view dbname, int flags); MDBRWTransaction getRWTransaction(); MDBROTransaction getROTransaction(); @@ -257,7 +257,7 @@ public: // this is something you can do, readonly - MDBDbi openDB(const char* dbname, int flags) + MDBDbi openDB(string_view dbname, int flags) { return MDBDbi(d_parent->d_env, d_txn, dbname, flags); } @@ -408,7 +408,7 @@ public: } - int del(MDB_dbi dbi, const MDBInVal& key, const MDBInVal& val) + int del(MDBDbi& dbi, const MDBInVal& key, const MDBInVal& val) { int rc; rc=mdb_del(d_txn, dbi, (MDB_val*)&key.d_mdbval, (MDB_val*)&val.d_mdbval); @@ -417,7 +417,7 @@ public: return rc; } - int del(MDB_dbi dbi, const MDBInVal& key) + int del(MDBDbi& dbi, const MDBInVal& key) { int rc; rc=mdb_del(d_txn, dbi, (MDB_val*)&key.d_mdbval, 0); @@ -427,7 +427,7 @@ public: } - int get(MDB_dbi dbi, const MDBInVal& key, MDBOutVal& val) + int get(MDBDbi& dbi, const MDBInVal& key, MDBOutVal& val) { if(!d_txn) throw std::runtime_error("Attempt to use a closed RW transaction for get"); @@ -439,7 +439,7 @@ public: return rc; } - int get(MDB_dbi dbi, const MDBInVal& key, string_view& val) + int get(MDBDbi& dbi, const MDBInVal& key, string_view& val) { MDBOutVal out; int rc = get(dbi, key, out); diff --git a/lmdb-typed.cc b/lmdb-typed.cc new file mode 100644 index 0000000..5cd7595 --- /dev/null +++ b/lmdb-typed.cc @@ -0,0 +1,112 @@ +#include "lmdb-typed.hh" +#include + +unsigned int getMaxID(MDBRWTransaction& txn, MDBDbi& dbi) +{ + auto cursor = txn.getCursor(dbi); + MDBOutVal maxidval, maxcontent; + unsigned int maxid{0}; + if(!cursor.get(maxidval, maxcontent, MDB_LAST)) { + maxid = maxidval.get(); + } + return maxid; +} + + +using namespace std; + +struct DNSResourceRecord +{ + string qname; // index + uint16_t qtype{0}; + uint32_t domain_id{0}; // index + string content; + uint32_t ttl{0}; + string ordername; // index + bool auth{true}; +}; + +template +void serialize(Archive & ar, DNSResourceRecord& g, const unsigned int version) +{ + ar & g.qtype; + ar & g.qname; + ar & g.content; + ar & g.ttl; + ar & g.domain_id; + ar & g.ordername; + ar & g.auth; +} + + +int main() +{ + TypedDBI, + index_on, + index_on + > tdbi(getMDBEnv("./typed.lmdb", MDB_NOSUBDIR, 0600), "records"); + + auto txn = tdbi.getRWTransaction(); + +#if 0 + cout<<"Going to iterate over powerdns.com!"<qname << " " << iter->qtype << " " << iter->content < +#include "lmdb-safe.hh" +#include +#include +#include +#include +#include +#include +#include +#include +#include +using std::cout; +using std::endl; + +unsigned int getMaxID(MDBRWTransaction& txn, MDBDbi& dbi); + + +template +std::string serToString(const T& t) +{ + + std::string serial_str; + boost::iostreams::back_insert_device inserter(serial_str); + boost::iostreams::stream > s(inserter); + boost::archive::binary_oarchive oa(s, boost::archive::no_header | boost::archive::no_codecvt); + + oa << t; + return serial_str; +} + +template +void serFromString(const std::string& str, T& ret) +{ + ret = T(); + std::istringstream istr{str}; + boost::archive::binary_iarchive oi(istr,boost::archive::no_header|boost::archive::no_codecvt ); + oi >> ret; +} + + +/* This is for storing a struct that has to be found using several + of its fields. + + We want a typed lmdb database that we can only: + * insert such structs + * remove them + * mutate them + + All while maintaining indexes on insert, removal and mutation. + + struct DNSResourceRecord + { + string qname; // index + string qtype; + uint32_t domain_id; // index + string content; + string ordername; // index + bool auth; + } + + TypedDBI tdbi; + + DNSResourceRecord rr; + uint32_t id = tdbi.insert(rr); // inserts, creates three index items + + tdbi.modify(id, [](auto& rr) { rr.auth=false; }); + + DNSResourceRecord retrr; + uint32_t id = tdbi.get<1>(qname, retrr); + + // this checks for changes and updates indexes if need be + tdbi.modify(id, [](auto& rr) { rr.ordername="blah"; }); +*/ + + +template +struct index_on +{ + static Type getMember(const Class& c) + { + return c.*PtrToMember; + } + + void put(MDBRWTransaction& txn, const Class& t, uint32_t id) + { + txn.put(d_idx, getMember(t), id); + } + + void del(MDBRWTransaction& txn, const Class& t, uint32_t id) + { + txn.del(d_idx, getMember(t), id); + } + + void openDB(std::shared_ptr& env, string_view str, int flags) + { + d_idx = env->openDB(str, flags); + } + + + typedef Type type; + MDBDbi d_idx; +}; + +struct nullindex_t +{ + template + void put(MDBRWTransaction& txn, const Class& t, uint32_t id) + {} + template + void del(MDBRWTransaction& txn, const Class& t, uint32_t id) + {} + + void openDB(std::shared_ptr& env, string_view str, int flags) + { + + } + typedef uint32_t type; // dummy +}; + +template +class TypedDBI +{ +public: + TypedDBI(std::shared_ptr env, string_view name) + : d_env(env), d_name(name) + { + d_main = d_env->openDB(name, MDB_CREATE | MDB_INTEGERKEY); + d_i1.openDB(d_env, std::string(name)+"_1", MDB_CREATE | MDB_DUPFIXED | MDB_DUPSORT); + d_i2.openDB(d_env, std::string(name)+"_2", MDB_CREATE | MDB_DUPFIXED | MDB_DUPSORT); + d_i3.openDB(d_env, std::string(name)+"_3", MDB_CREATE | MDB_DUPFIXED | MDB_DUPSORT); + d_i4.openDB(d_env, std::string(name)+"_4", MDB_CREATE | MDB_DUPFIXED | MDB_DUPSORT); + } + + I1 d_i1; + I2 d_i2; + I3 d_i3; + I4 d_i4; + + class RWTransaction + { + public: + explicit RWTransaction(TypedDBI* parent) : d_parent(parent), d_txn(d_parent->d_env->getRWTransaction()) + { + } + + RWTransaction(RWTransaction&& rhs) : + d_parent(rhs.d_parent), d_txn(std::move(rhs.d_txn)) + { + rhs.d_parent = 0; + } + + uint32_t insert(const T& t) + { + uint32_t id = getMaxID(d_txn, d_parent->d_main) + 1; + d_txn.put(d_parent->d_main, id, serToString(t)); + + d_parent->d_i1.put(d_txn, t, id); + d_parent->d_i2.put(d_txn, t, id); + d_parent->d_i3.put(d_txn, t, id); + d_parent->d_i4.put(d_txn, t, id); + return id; + } + + bool get(uint32_t id, T& t) + { + MDBOutVal data; + if(d_txn.get(d_parent->d_main, id, data)) + return false; + + serFromString(data.get(), t); + return true; + } + + void del(uint32_t id) + { + T t; + if(!get(id, t)) + return; + + d_txn.del(d_parent->d_main, id); + + d_parent->d_i1.del(d_txn, t, id); + d_parent->d_i2.del(d_txn, t, id); + d_parent->d_i3.del(d_txn, t, id); + d_parent->d_i4.del(d_txn, t, id); + } + + uint32_t get1(const typename I1::type& key, T& out) + { + MDBOutVal id; + if(!d_txn.get(d_parent->d_i1.d_idx, key, id)) + return get(id.get(), out); + return 0; + } + + uint32_t get2(const typename I2::type& key, T& out) + { + MDBOutVal id; + if(!d_txn.get(d_parent->d_i2.d_idx, key, id)) + return get(id.get(), out); + return 0; + } + + uint32_t get3(const typename I3::type& key, T& out) + { + MDBOutVal id; + if(!d_txn.get(d_parent->d_i3.d_idx, key, id)) + return get(id.get(), out); + return 0; + } + + uint32_t get4(const typename I4::type& key, T& out) + { + MDBOutVal id; + if(!d_txn.get(d_parent->d_i4.d_idx, key, id)) + return get(id.get(), out); + return 0; + } + + void commit() + { + d_txn.commit(); + } + + void abort() + { + d_txn.abort(); + } + + + private: + TypedDBI* d_parent; + MDBRWTransaction d_txn; + }; + + RWTransaction getRWTransaction() + { + return RWTransaction(this); + } + +private: + + std::shared_ptr d_env; + MDBDbi d_main; + std::string d_name; +}; + + +#if 0 + struct eiter1_t + {}; + struct iter1_t + { + explicit iter1_t(MDBROTransaction && txn, const MDBDbi& dbi, const MDBDbi& main, const typename I1::type& key) : d_txn(std::move(txn)), d_cursor(d_txn.getCursor(dbi)), d_in(key), d_main(main) + { + d_key.d_mdbval = d_in.d_mdbval; + + MDBOutVal id, data; + if(d_cursor.get(d_key, id, MDB_SET)) { + d_end = true; + return; + } + if(d_txn.get(d_main, id, data)) + throw std::runtime_error("Missing id in constructor"); + + serFromString(data.get(), d_t); + } + + + bool operator!=(const eiter1_t& rhs) + { + return !d_end; + } + + bool operator==(const eiter1_t& rhs) + { + return d_end; + } + + const T& operator*() + { + return d_t; + } + + const T* operator->() + { + return &d_t; + } + + iter1_t& operator++() + { + MDBOutVal id, data; + int rc = d_cursor.get(d_key, id, MDB_NEXT_DUP); + if(rc == MDB_NOTFOUND) { + d_end = true; + } + else { + if(d_txn.get(d_main, id, data)) + throw std::runtime_error("Missing id field"); + + serFromString(data.get(), d_t); + } + return *this; + } + + MDBROTransaction d_txn; + MDBROCursor d_cursor; + MDBOutVal d_key, d_data; + MDBInVal d_in; + bool d_end{false}; + T d_t; + MDBDbi d_main; + }; + + iter1_t find1(const typename I1::type& key) + { + iter1_t ret{std::move(d_env->getROTransaction()), d_idx1.d_ix1 d_main, key}; + return ret; + }; + + eiter1_t end() + { + return eiter1_t(); + } + +#endif