.gitignore | 7 +- INSTALL | 53 +++-- LICENSE | 7 +- Makefile | 6 +- UPGRADING | 5 + bindings/bigloo/LICENSE | 20 +- bindings/d/LICENSE | 20 +- bindings/haskell/LICENSE | 20 +- bindings/newlisp/LICENSE | 20 +- bindings/python/LICENSE | 20 +- bindings/python/nmdb.py | 33 +++- bindings/python/nmdb_ll.c | 193 ++++++++++++++--- bindings/python/setup.py | 10 +- bindings/python3/LICENSE | 34 --- bindings/python3/nmdb.py | 252 --------------------- bindings/python3/nmdb_ll.c | 452 -------------------------------------- bindings/python3/setup.py | 17 -- bindings/ruby/LICENSE | 20 +- doc/design.rst | 13 +- doc/guide.rst | 23 +- doc/network.rst | 17 +- libnmdb/LICENSE | 20 +- libnmdb/Makefile | 29 ++-- libnmdb/doxygen/Doxyfile.base.in | 204 +++++++++++++++++ libnmdb/doxygen/Doxyfile.public | 8 + libnmdb/doxygen/Makefile | 30 +++ libnmdb/internal.h | 29 --- libnmdb/internal.h.in | 83 +++++++ libnmdb/libnmdb.3 | 6 +- libnmdb/libnmdb.c | 119 ++++++++++- libnmdb/libnmdb.skel.pc | 2 +- libnmdb/nmdb.h | 355 ++++++++++++++++++++++++++++++ libnmdb/nmdb.skel.h | 101 --------- libnmdb/sctp.h | 2 + libnmdb/tcp.h | 2 + libnmdb/tipc.h | 2 + libnmdb/udp.h | 2 + nmdb/LICENSE | 194 ++--------------- nmdb/Makefile | 56 +++-- nmdb/be-bdb.c | 86 ++++++-- nmdb/be-null.c | 51 ++++- nmdb/be-qdbm.c | 66 +++++- nmdb/be-tc.c | 62 +++++- nmdb/be-tdb.c | 155 +++++++++++++ nmdb/be.c | 61 +++++ nmdb/be.h | 119 ++++++++--- nmdb/cache.c | 306 +++++++++++++++----------- nmdb/cache.h | 21 ++- nmdb/common.h | 4 + nmdb/dbloop.c | 70 +++++- nmdb/dbloop.h | 4 +- nmdb/hash.h | 164 ++++++++++++++ nmdb/log.c | 18 ++ nmdb/log.h | 3 + nmdb/main.c | 66 +++++- nmdb/net-const.h | 3 + nmdb/net.c | 18 ++- nmdb/nmdb.1 | 25 ++- nmdb/parse.c | 119 ++++++++--- nmdb/sparse.h | 13 +- nmdb/stats.c | 3 + nmdb/stats.h | 4 +- nmdb/tcp.c | 5 +- tests/coverage/README | 21 ++ tests/coverage/coverage | 183 +++++++++++++++ tests/coverage/lcov-start | 21 ++ tests/coverage/lcov-stop | 28 +++ tests/perf/ag.sh | 7 +- tests/python/README | 4 + tests/python/random1-cache.py | 129 ----------- tests/python/random1.py | 295 +++++++++++++++++-------- tests/python/walk.py | 103 +++++++++ tests/python3/README | 4 - tests/python3/random1-cache.py | 129 ----------- tests/python3/random1.py | 154 ------------- utils/nmdb-stats.c | 7 +- 76 files changed, 2954 insertions(+), 2063 deletions(-) diff --git a/.gitignore b/.gitignore index 7cad653..1ce400d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,17 +7,20 @@ .*.swp *.gcno *.gcda +gmon.out # *~ /nmdb/nmdb /utils/nmdb-stats /tests/c/*-*-cache /tests/c/*-*-normal /tests/c/*-*-sync -/libnmdb/nmdb.h +/libnmdb/internal.h /libnmdb/libnmdb.pc +/libnmdb/doxygen/doc.internal +/libnmdb/doxygen/Doxyfile.base /tags /bindings/python/build -/bindings/python3/build /tests/perf/out /tests/perf/ag-data /tests/perf/graph +/tests/coverage/lcov diff --git a/INSTALL b/INSTALL index 64f1615..0f7d4e0 100644 --- a/INSTALL +++ b/INSTALL @@ -4,10 +4,9 @@ Quick guide for the patience-impaired At the top level directory, run: - $ make BACKEND=qdbm ENABLE_TIPC=0 ENABLE_SCTP=0 install + $ make install -to build and install the server without TIPC and SCTP support, and to use the -qdbm backend. +to build and install the server, library and utilities. How to compile and install @@ -19,40 +18,37 @@ The only mandatory requisite to build nmdb is libevent There are several build-time options, in two groups: supported network protocols, and backend databases. -The network protocols can all be enabled at the same time: +The network protocols: + * TCP and UDP, the well-known network protocols. * TIPC (http://tipc.sf.net/), a cluster-oriented network protocol. You will need a Linux kernel >= 2.6.16 with TIPC support (most distributions enable it by default). * SCTP, a network protocol similar to UDP and TCP, offering reliable message - passing over IP, among other very useful things. You will need the - lksctp-tools package. - -The backends, on the other hand, are mutually exclusive and only one can be -selected: - * qdbm (http://qdbm.sf.net/). - * bdb (http://www.oracle.com/database/berkeley-db/). - * null backend, if you don't need a real one. - -By default, all network protocols are enabled, and the qdbm backend is -selected. + passing over IP, among other very useful things. You will need the + libsctp-dev (or equivalent) package. -You can change the defaults by passing parameters to make, like this: +The backend databases: - $ make BACKEND=[qbdm|bdb|null] ENABLE_$PROTO=[1|0] + * qdbm (http://qdbm.sf.net/) + * bdb (http://www.oracle.com/database/berkeley-db/) + * Tokyo Cabinet (http://1978th.net/tokyocabinet/) + * tdb (http://tdb.samba.org/) + * A null backend (to use when you don't need a real one) -Where $PROTO can be TCP, UDP, TIPC or SCTP. +By default, network protocols and backends are automatically detected +according to the available libraries. -For instance, to build with bdb backend and without TIPC and UDP support, use: +You can change the defaults by passing parameters to make, like this: - $ make BACKEND=bdb ENABLE_TIPC=0 + $ make BE_ENABLE_$BACKEND=[1|0] ENABLE_$PROTO=[1|0] +Where $PROTO can be TCP, UDP, TIPC or SCTP, and $BACKEND can be QDBM, BDB, TC, +TDB or NULL. -Tests ------ +For instance, to build with bdb backend and without TIPC support, use: -To run some tests, start the server and then go to the "tests/c/" directory. -Run "make.sh build" and then run any of the generated tests. + $ make BE_ENABLE_BDB=1 ENABLE_TIPC=0 Bindings @@ -67,3 +63,12 @@ The other bindings do not have a properly defined install procedure, and you'll need knowledge of the language to install them. +Tests +----- + +Tests are available in the "tests/" directory. Some are written in C +("tests/c/"), some in Python ("tests/python/"). + +Python tests are useful for stress and coverage tests, while the C ones are +useful for performance measurements. + diff --git a/LICENSE b/LICENSE index e1e5faa..0c8c297 100644 --- a/LICENSE +++ b/LICENSE @@ -3,11 +3,12 @@ Each sub-project in nmdb source tree has an independant license, and can be treated as independant projects from a licensing point of view. Licenses are always in a file named "LICENSE" in the top directory of each -sub-project, and apply to ALL the source code files in that directory tree (it -includes subdirectories). +sub-project, and apply to ALL the source code files in that directory tree +(that is, including subdirectories). As a brief resume, here's how each sub-project is licensed: - * nmdb: OSL 3.0 + + * nmdb: BOLA (Public domain) * libnmdb: BOLA (Public domain) * bindings/bigloo: BOLA (Public domain) * bindings/d: BOLA (Public domain) diff --git a/Makefile b/Makefile index 5ef8582..7c61c5e 100644 --- a/Makefile +++ b/Makefile @@ -33,13 +33,13 @@ python_clean: cd bindings/python && rm -rf build/ python3: - cd bindings/python3 && python3 setup.py build + cd bindings/python && python3 setup.py build python3_install: - cd bindings/python3 && python3 setup.py install + cd bindings/python && python3 setup.py install python3_clean: - cd bindings/python3 && rm -rf build/ + cd bindings/python && rm -rf build/ .PHONY: default all clean nmdb libnmdb utils \ diff --git a/UPGRADING b/UPGRADING index bd906cd..f839937 100644 --- a/UPGRADING +++ b/UPGRADING @@ -5,6 +5,11 @@ While normally nothing should be done, sometimes things change in incompatible ways. Here's the listing for the releases, you should always check it before upgrading. +0.22 -> 0.23 + * The server signal handling for SIGUSR1 has been remapped: to reopen the + file descriptors send SIGHUP instead of SIGUSR1. SIGUSR1 now puts the + server in read-only mode. + 0.20 -> 0.21 * nmdb_get() and nmdb_cache_get() now return -1 instead of 0 when there is a missing key. This was done to allow setting a value to be "" (no data). All diff --git a/bindings/bigloo/LICENSE b/bindings/bigloo/LICENSE index f3a9498..987d3b9 100644 --- a/bindings/bigloo/LICENSE +++ b/bindings/bigloo/LICENSE @@ -2,12 +2,11 @@ I don't like licenses, because I don't like having to worry about all this legal stuff just for a simple piece of software I don't really mind anyone using. But I also believe that it's important that people share and give back; -so I'm placing this library under the following license, so you feel guilty if -you don't ;) +so I'm placing this work under the following license. -BOLA - Buena Onda License Agreement ------------------------------------ +BOLA - Buena Onda License Agreement (v1.1) +------------------------------------------ This work is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of @@ -16,19 +15,16 @@ this work. To all effects and purposes, this work is to be considered Public Domain. -However, if you want to be "Buena onda", you should: +However, if you want to be "buena onda", you should: 1. Not take credit for it, and give proper recognition to the authors. 2. Share your modifications, so everybody benefits from them. -4. Do something nice for the authors. -5. Help someone who needs it: sign up for some volunteer work or help your +3. Do something nice for the authors. +4. Help someone who needs it: sign up for some volunteer work or help your neighbour paint the house. -6. Don't waste. Anything, but specially energy that comes from natural +5. Don't waste. Anything, but specially energy that comes from natural non-renewable resources. Extra points if you discover or invent something to replace them. -7. Be tolerant. Everything that's good in nature comes from cooperation. - -The order is important, and the further you go the more "Buena onda" you are. -Make the world a better place: be "Buena onda". +6. Be tolerant. Everything that's good in nature comes from cooperation. diff --git a/bindings/d/LICENSE b/bindings/d/LICENSE index f3a9498..987d3b9 100644 --- a/bindings/d/LICENSE +++ b/bindings/d/LICENSE @@ -2,12 +2,11 @@ I don't like licenses, because I don't like having to worry about all this legal stuff just for a simple piece of software I don't really mind anyone using. But I also believe that it's important that people share and give back; -so I'm placing this library under the following license, so you feel guilty if -you don't ;) +so I'm placing this work under the following license. -BOLA - Buena Onda License Agreement ------------------------------------ +BOLA - Buena Onda License Agreement (v1.1) +------------------------------------------ This work is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of @@ -16,19 +15,16 @@ this work. To all effects and purposes, this work is to be considered Public Domain. -However, if you want to be "Buena onda", you should: +However, if you want to be "buena onda", you should: 1. Not take credit for it, and give proper recognition to the authors. 2. Share your modifications, so everybody benefits from them. -4. Do something nice for the authors. -5. Help someone who needs it: sign up for some volunteer work or help your +3. Do something nice for the authors. +4. Help someone who needs it: sign up for some volunteer work or help your neighbour paint the house. -6. Don't waste. Anything, but specially energy that comes from natural +5. Don't waste. Anything, but specially energy that comes from natural non-renewable resources. Extra points if you discover or invent something to replace them. -7. Be tolerant. Everything that's good in nature comes from cooperation. - -The order is important, and the further you go the more "Buena onda" you are. -Make the world a better place: be "Buena onda". +6. Be tolerant. Everything that's good in nature comes from cooperation. diff --git a/bindings/haskell/LICENSE b/bindings/haskell/LICENSE index f3a9498..987d3b9 100644 --- a/bindings/haskell/LICENSE +++ b/bindings/haskell/LICENSE @@ -2,12 +2,11 @@ I don't like licenses, because I don't like having to worry about all this legal stuff just for a simple piece of software I don't really mind anyone using. But I also believe that it's important that people share and give back; -so I'm placing this library under the following license, so you feel guilty if -you don't ;) +so I'm placing this work under the following license. -BOLA - Buena Onda License Agreement ------------------------------------ +BOLA - Buena Onda License Agreement (v1.1) +------------------------------------------ This work is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of @@ -16,19 +15,16 @@ this work. To all effects and purposes, this work is to be considered Public Domain. -However, if you want to be "Buena onda", you should: +However, if you want to be "buena onda", you should: 1. Not take credit for it, and give proper recognition to the authors. 2. Share your modifications, so everybody benefits from them. -4. Do something nice for the authors. -5. Help someone who needs it: sign up for some volunteer work or help your +3. Do something nice for the authors. +4. Help someone who needs it: sign up for some volunteer work or help your neighbour paint the house. -6. Don't waste. Anything, but specially energy that comes from natural +5. Don't waste. Anything, but specially energy that comes from natural non-renewable resources. Extra points if you discover or invent something to replace them. -7. Be tolerant. Everything that's good in nature comes from cooperation. - -The order is important, and the further you go the more "Buena onda" you are. -Make the world a better place: be "Buena onda". +6. Be tolerant. Everything that's good in nature comes from cooperation. diff --git a/bindings/newlisp/LICENSE b/bindings/newlisp/LICENSE index f3a9498..987d3b9 100644 --- a/bindings/newlisp/LICENSE +++ b/bindings/newlisp/LICENSE @@ -2,12 +2,11 @@ I don't like licenses, because I don't like having to worry about all this legal stuff just for a simple piece of software I don't really mind anyone using. But I also believe that it's important that people share and give back; -so I'm placing this library under the following license, so you feel guilty if -you don't ;) +so I'm placing this work under the following license. -BOLA - Buena Onda License Agreement ------------------------------------ +BOLA - Buena Onda License Agreement (v1.1) +------------------------------------------ This work is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of @@ -16,19 +15,16 @@ this work. To all effects and purposes, this work is to be considered Public Domain. -However, if you want to be "Buena onda", you should: +However, if you want to be "buena onda", you should: 1. Not take credit for it, and give proper recognition to the authors. 2. Share your modifications, so everybody benefits from them. -4. Do something nice for the authors. -5. Help someone who needs it: sign up for some volunteer work or help your +3. Do something nice for the authors. +4. Help someone who needs it: sign up for some volunteer work or help your neighbour paint the house. -6. Don't waste. Anything, but specially energy that comes from natural +5. Don't waste. Anything, but specially energy that comes from natural non-renewable resources. Extra points if you discover or invent something to replace them. -7. Be tolerant. Everything that's good in nature comes from cooperation. - -The order is important, and the further you go the more "Buena onda" you are. -Make the world a better place: be "Buena onda". +6. Be tolerant. Everything that's good in nature comes from cooperation. diff --git a/bindings/python/LICENSE b/bindings/python/LICENSE index f3a9498..987d3b9 100644 --- a/bindings/python/LICENSE +++ b/bindings/python/LICENSE @@ -2,12 +2,11 @@ I don't like licenses, because I don't like having to worry about all this legal stuff just for a simple piece of software I don't really mind anyone using. But I also believe that it's important that people share and give back; -so I'm placing this library under the following license, so you feel guilty if -you don't ;) +so I'm placing this work under the following license. -BOLA - Buena Onda License Agreement ------------------------------------ +BOLA - Buena Onda License Agreement (v1.1) +------------------------------------------ This work is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of @@ -16,19 +15,16 @@ this work. To all effects and purposes, this work is to be considered Public Domain. -However, if you want to be "Buena onda", you should: +However, if you want to be "buena onda", you should: 1. Not take credit for it, and give proper recognition to the authors. 2. Share your modifications, so everybody benefits from them. -4. Do something nice for the authors. -5. Help someone who needs it: sign up for some volunteer work or help your +3. Do something nice for the authors. +4. Help someone who needs it: sign up for some volunteer work or help your neighbour paint the house. -6. Don't waste. Anything, but specially energy that comes from natural +5. Don't waste. Anything, but specially energy that comes from natural non-renewable resources. Extra points if you discover or invent something to replace them. -7. Be tolerant. Everything that's good in nature comes from cooperation. - -The order is important, and the further you go the more "Buena onda" you are. -Make the world a better place: be "Buena onda". +6. Be tolerant. Everything that's good in nature comes from cooperation. diff --git a/bindings/python/nmdb.py b/bindings/python/nmdb.py index e185d55..fbad7a9 100644 --- a/bindings/python/nmdb.py +++ b/bindings/python/nmdb.py @@ -183,8 +183,8 @@ class GenericDB (object): elif r == 1: # no match, because the value didn't have the right # format - raise TypeError, \ - "The value must be a NULL-terminated string" + raise TypeError( + "The value must be a NULL-terminated string") elif r == 0: # not in raise KeyError @@ -197,6 +197,31 @@ class GenericDB (object): def normal_incr(self, key, increment = 1): return self.generic_incr(self._db.incr, key, increment) + def firstkey(self): + try: + r = self._db.firstkey() + except: + raise NetworkError + if r == -1: + # No keys, or unsupported + raise KeyError + if self.autopickle: + r = pickle.loads(r) + return r + + def nextkey(self, key): + if self.autopickle: + key = pickle.dumps(key, protocol = -1) + try: + r = self._db.nextkey(key) + except: + raise NetworkError + if r == -1: + # No next key, or unsupported + raise KeyError + if self.autopickle: + r = pickle.loads(r) + return r # The following functions will assume the existance of self.set, # self.get, and self.delete, which are supposed to be set by our @@ -240,6 +265,8 @@ class DB (GenericDB): delete = GenericDB.normal_delete cas = GenericDB.normal_cas incr = GenericDB.normal_incr + firstkey = GenericDB.firstkey + nextkey = GenericDB.nextkey class SyncDB (GenericDB): get = GenericDB.normal_get @@ -247,5 +274,7 @@ class SyncDB (GenericDB): delete = GenericDB.delete_sync cas = GenericDB.normal_cas incr = GenericDB.normal_incr + firstkey = GenericDB.firstkey + nextkey = GenericDB.nextkey diff --git a/bindings/python/nmdb_ll.c b/bindings/python/nmdb_ll.c index c4bdd91..6c329a3 100644 --- a/bindings/python/nmdb_ll.c +++ b/bindings/python/nmdb_ll.c @@ -1,12 +1,14 @@ /* - * Python bindings for libnmdb + * Python 2 and 3 bindings for libnmdb * Alberto Bertogli (albertito@blitiri.com.ar) * * This is the low-level module, used by the python one to construct * friendlier objects. */ +#define PY_SSIZE_T_CLEAN 1 + #include #include @@ -19,7 +21,6 @@ typedef struct { PyObject_HEAD; nmdb_t *db; } nmdbobject; -static PyTypeObject nmdbType; /* * The nmdb object @@ -92,7 +93,7 @@ static PyObject *db_add_udp_server(nmdbobject *db, PyObject *args) static PyObject *db_cache_set(nmdbobject *db, PyObject *args) { unsigned char *key, *val; - int ksize, vsize; + size_t ksize, vsize; int rv; if (!PyArg_ParseTuple(args, "s#s#:cache_set", &key, &ksize, @@ -111,8 +112,8 @@ static PyObject *db_cache_set(nmdbobject *db, PyObject *args) static PyObject *db_cache_get(nmdbobject *db, PyObject *args) { unsigned char *key, *val; - int ksize, vsize; - long rv; + size_t ksize, vsize; + ssize_t rv; PyObject *r; if (!PyArg_ParseTuple(args, "s#:cache_get", &key, &ksize)) { @@ -136,7 +137,11 @@ static PyObject *db_cache_get(nmdbobject *db, PyObject *args) /* Miss, handled in the high-level module. */ r = PyLong_FromLong(-1); } else { +#ifdef PYTHON3 + r = PyBytes_FromStringAndSize((char *) val, rv); +#elif PYTHON2 r = PyString_FromStringAndSize((char *) val, rv); +#endif } free(val); @@ -147,7 +152,7 @@ static PyObject *db_cache_get(nmdbobject *db, PyObject *args) static PyObject *db_cache_delete(nmdbobject *db, PyObject *args) { unsigned char *key; - int ksize; + size_t ksize; int rv; if (!PyArg_ParseTuple(args, "s#:cache_delete", &key, &ksize)) { @@ -165,7 +170,7 @@ static PyObject *db_cache_delete(nmdbobject *db, PyObject *args) static PyObject *db_cache_cas(nmdbobject *db, PyObject *args) { unsigned char *key, *oldval, *newval; - int ksize, ovsize, nvsize; + size_t ksize, ovsize, nvsize; int rv; if (!PyArg_ParseTuple(args, "s#s#s#:cache_cas", &key, &ksize, @@ -186,7 +191,7 @@ static PyObject *db_cache_cas(nmdbobject *db, PyObject *args) static PyObject *db_cache_incr(nmdbobject *db, PyObject *args) { unsigned char *key; - int ksize; + size_t ksize; int rv; long long int increment; int64_t newval; @@ -200,7 +205,7 @@ static PyObject *db_cache_incr(nmdbobject *db, PyObject *args) rv = nmdb_cache_incr(db->db, key, ksize, increment, &newval); Py_END_ALLOW_THREADS - return Py_BuildValue("LL", rv, newval); + return Py_BuildValue("iL", rv, newval); } @@ -208,7 +213,7 @@ static PyObject *db_cache_incr(nmdbobject *db, PyObject *args) static PyObject *db_set(nmdbobject *db, PyObject *args) { unsigned char *key, *val; - int ksize, vsize; + size_t ksize, vsize; int rv; if (!PyArg_ParseTuple(args, "s#s#:set", &key, &ksize, @@ -227,8 +232,8 @@ static PyObject *db_set(nmdbobject *db, PyObject *args) static PyObject *db_get(nmdbobject *db, PyObject *args) { unsigned char *key, *val; - int ksize, vsize; - long rv; + size_t ksize, vsize; + ssize_t rv; PyObject *r; if (!PyArg_ParseTuple(args, "s#:get", &key, &ksize)) { @@ -252,7 +257,11 @@ static PyObject *db_get(nmdbobject *db, PyObject *args) /* Miss, handled in the high-level module. */ r = PyLong_FromLong(-1); } else { +#ifdef PYTHON3 + r = PyBytes_FromStringAndSize((char *) val, rv); +#elif PYTHON2 r = PyString_FromStringAndSize((char *) val, rv); +#endif } free(val); @@ -263,7 +272,7 @@ static PyObject *db_get(nmdbobject *db, PyObject *args) static PyObject *db_delete(nmdbobject *db, PyObject *args) { unsigned char *key; - int ksize; + size_t ksize; int rv; if (!PyArg_ParseTuple(args, "s#:delete", &key, &ksize)) { @@ -281,7 +290,7 @@ static PyObject *db_delete(nmdbobject *db, PyObject *args) static PyObject *db_cas(nmdbobject *db, PyObject *args) { unsigned char *key, *oldval, *newval; - int ksize, ovsize, nvsize; + size_t ksize, ovsize, nvsize; int rv; if (!PyArg_ParseTuple(args, "s#s#s#:cas", &key, &ksize, @@ -301,7 +310,7 @@ static PyObject *db_cas(nmdbobject *db, PyObject *args) static PyObject *db_incr(nmdbobject *db, PyObject *args) { unsigned char *key; - int ksize; + size_t ksize; int rv; long long int increment; int64_t newval; @@ -322,7 +331,7 @@ static PyObject *db_incr(nmdbobject *db, PyObject *args) static PyObject *db_set_sync(nmdbobject *db, PyObject *args) { unsigned char *key, *val; - int ksize, vsize; + size_t ksize, vsize; int rv; if (!PyArg_ParseTuple(args, "s#s#:set_sync", &key, &ksize, @@ -341,7 +350,7 @@ static PyObject *db_set_sync(nmdbobject *db, PyObject *args) static PyObject *db_delete_sync(nmdbobject *db, PyObject *args) { unsigned char *key; - int ksize; + size_t ksize; int rv; if (!PyArg_ParseTuple(args, "s#:delete_sync", &key, &ksize)) { @@ -355,6 +364,86 @@ static PyObject *db_delete_sync(nmdbobject *db, PyObject *args) return PyLong_FromLong(rv); } +/* firstkey */ +static PyObject *db_firstkey(nmdbobject *db, PyObject *args) +{ + unsigned char *key; + size_t ksize; + ssize_t rv; + PyObject *r; + + if (!PyArg_ParseTuple(args, "")) { + return NULL; + } + + /* ksize is enough to hold the any value */ + ksize = 128 * 1024; + key = malloc(ksize); + if (key== NULL) + return PyErr_NoMemory(); + + Py_BEGIN_ALLOW_THREADS + rv = nmdb_firstkey(db->db, key, ksize); + Py_END_ALLOW_THREADS + + if (rv <= -2) { + /* FIXME: define a better exception */ + r = PyErr_SetFromErrno(PyExc_IOError); + } else if (rv == -1) { + /* No first key or unsupported, handled in the high-level + * module. */ + r = PyLong_FromLong(-1); + } else { +#ifdef PYTHON3 + r = PyBytes_FromStringAndSize((char *) key, rv); +#elif PYTHON2 + r = PyString_FromStringAndSize((char *) key, rv); +#endif + } + + free(key); + return r; +} + +/* nextkey */ +static PyObject *db_nextkey(nmdbobject *db, PyObject *args) +{ + unsigned char *key, *newkey; + size_t ksize, nksize; + ssize_t rv; + PyObject *r; + + if (!PyArg_ParseTuple(args, "s#:nextkey", &key, &ksize)) { + return NULL; + } + + /* nksize is enough to hold the any value */ + nksize = 128 * 1024; + newkey = malloc(nksize); + if (newkey == NULL) + return PyErr_NoMemory(); + + Py_BEGIN_ALLOW_THREADS + rv = nmdb_nextkey(db->db, key, ksize, newkey, nksize); + Py_END_ALLOW_THREADS + + if (rv <= -2) { + /* FIXME: define a better exception */ + r = PyErr_SetFromErrno(PyExc_IOError); + } else if (rv == -1) { + /* End, handled in the high-level module. */ + r = PyLong_FromLong(-1); + } else { +#ifdef PYTHON3 + r = PyBytes_FromStringAndSize((char *) newkey, rv); +#elif PYTHON2 + r = PyString_FromStringAndSize((char *) newkey, rv); +#endif + } + + free(newkey); + return r; +} /* nmdb method table */ @@ -378,25 +467,12 @@ static PyMethodDef nmdb_methods[] = { { "incr", (PyCFunction) db_incr, METH_VARARGS, NULL }, { "set_sync", (PyCFunction) db_set_sync, METH_VARARGS, NULL }, { "delete_sync", (PyCFunction) db_delete_sync, METH_VARARGS, NULL }, + { "firstkey", (PyCFunction) db_firstkey, METH_VARARGS, NULL }, + { "nextkey", (PyCFunction) db_nextkey, METH_VARARGS, NULL }, { NULL } }; -static PyObject *db_getattr(nmdbobject *db, char *name) -{ - return Py_FindMethod(nmdb_methods, (PyObject *)db, name); -} - -static PyTypeObject nmdbType = { - PyObject_HEAD_INIT(NULL) - 0, - "nmdb_ll.nmdb", - sizeof(nmdbobject), - 0, - (destructor) db_dealloc, - 0, - (getattrfunc) db_getattr, -}; /* @@ -404,11 +480,16 @@ static PyTypeObject nmdbType = { */ /* new, returns an nmdb object */ +static PyTypeObject nmdbType; static PyObject *db_new(PyObject *self, PyObject *args) { nmdbobject *db; +#ifdef PYTHON3 + db = (nmdbobject *) nmdbType.tp_alloc(&nmdbType, 0); +#elif PYTHON2 db = PyObject_New(nmdbobject, &nmdbType); +#endif if (db == NULL) return NULL; @@ -435,6 +516,53 @@ static PyMethodDef nmdb_functions[] = { { NULL } }; + +#ifdef PYTHON3 +static PyTypeObject nmdbType = { + PyObject_HEAD_INIT(NULL) + .tp_name = "nmdb_ll.nmdb", + .tp_itemsize = sizeof(nmdbobject), + .tp_dealloc = (destructor) db_dealloc, + .tp_methods = nmdb_methods, +}; + +static PyModuleDef nmdb_module = { + PyModuleDef_HEAD_INIT, + .m_name = "nmdb_ll", + .m_doc = NULL, + .m_size = -1, + .m_methods = nmdb_functions, +}; + +PyMODINIT_FUNC PyInit_nmdb_ll(void) +{ + PyObject *m; + + if (PyType_Ready(&nmdbType) < 0) + return NULL; + + m = PyModule_Create(&nmdb_module); + + return m; +} + +#elif PYTHON2 +static PyObject *db_getattr(nmdbobject *db, char *name) +{ + return Py_FindMethod(nmdb_methods, (PyObject *)db, name); +} + +static PyTypeObject nmdbType = { + PyObject_HEAD_INIT(NULL) + 0, + "nmdb_ll.nmdb", + sizeof(nmdbobject), + 0, + (destructor) db_dealloc, + 0, + (getattrfunc) db_getattr, +}; + PyMODINIT_FUNC initnmdb_ll(void) { PyObject *m; @@ -444,5 +572,6 @@ PyMODINIT_FUNC initnmdb_ll(void) m = Py_InitModule("nmdb_ll", nmdb_functions); } +#endif diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 54c1bcd..dce1df3 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -1,9 +1,17 @@ +import sys from distutils.core import setup, Extension +if sys.version_info[0] == 2: + ver_define = ('PYTHON2', '1') +elif sys.version_info[0] == 3: + ver_define = ('PYTHON3', '1') + + nmdb_ll = Extension("nmdb_ll", libraries = ['nmdb'], - sources = ['nmdb_ll.c']) + sources = ['nmdb_ll.c'], + define_macros = [ver_define]) setup( name = 'nmdb', diff --git a/bindings/python3/LICENSE b/bindings/python3/LICENSE deleted file mode 100644 index f3a9498..0000000 --- a/bindings/python3/LICENSE +++ /dev/null @@ -1,34 +0,0 @@ - -I don't like licenses, because I don't like having to worry about all this -legal stuff just for a simple piece of software I don't really mind anyone -using. But I also believe that it's important that people share and give back; -so I'm placing this library under the following license, so you feel guilty if -you don't ;) - - -BOLA - Buena Onda License Agreement ------------------------------------ - -This work is provided 'as-is', without any express or implied warranty. In no -event will the authors be held liable for any damages arising from the use of -this work. - -To all effects and purposes, this work is to be considered Public Domain. - - -However, if you want to be "Buena onda", you should: - -1. Not take credit for it, and give proper recognition to the authors. -2. Share your modifications, so everybody benefits from them. -4. Do something nice for the authors. -5. Help someone who needs it: sign up for some volunteer work or help your - neighbour paint the house. -6. Don't waste. Anything, but specially energy that comes from natural - non-renewable resources. Extra points if you discover or invent something - to replace them. -7. Be tolerant. Everything that's good in nature comes from cooperation. - -The order is important, and the further you go the more "Buena onda" you are. -Make the world a better place: be "Buena onda". - - diff --git a/bindings/python3/nmdb.py b/bindings/python3/nmdb.py deleted file mode 100644 index 2c0b88c..0000000 --- a/bindings/python3/nmdb.py +++ /dev/null @@ -1,252 +0,0 @@ - -""" -libnmdb python 3 wrapper - -This module is a wrapper for the libnmdb, the C library used to implement -clients to the nmdb server. - -It provides three similar classes: DB, SyncDB and Cache. They all present the -same dictionary-alike interface, but differ in how they interact with the -server. - -The DB class allows you to set, get and delete (key, value) pairs from the -database; the SyncDB class works like DB, but does so in a synchronous way; and -the Cache class affects only the cache and do not impact the backend database. - -Note that mixing cache sets with DB sets can create inconsistencies between -the database and the cache. You shouldn't do that unless you know what you're -doing. - -The classes use pickle to allow you to store and retrieve python objects in a -transparent way. To disable it, set .autopickle to False. - -Here is an example using the DB class: - ->>> import nmdb ->>> db = nmdb.DB() ->>> db.add_tipc_server() ->>> db[1] = { 'english': 'one', 'castellano': 'uno', 'quechua': 'huk' } ->>> print(db[1]) -{'english': 'one', 'castellano': 'uno', 'quechua': 'huk'} ->>> db[(1, 2)] = (True, False) ->>> print(db[(1, 2)]) -(True, False) ->>> del db[(1, 2)] ->>> print(db[(1, 2)]) -Traceback (most recent call last): - File "", line 1, in - File "/usr/local/lib/python3.0/dist-packages/nmdb.py", line 206, in __getitem__ - return self.get(key) - File "/usr/local/lib/python3.0/dist-packages/nmdb.py", line 102, in normal_get - return self.generic_get(self._db.get, key) - File "/usr/local/lib/python3.0/dist-packages/nmdb.py", line 93, in generic_get - raise KeyError -KeyError ->>> -""" - -import pickle -import nmdb_ll - - -class NetworkError (Exception): - pass - - -class GenericDB: - def __init__(self): - self._db = nmdb_ll.nmdb() - self.autopickle = True - - def add_tipc_server(self, port = -1): - "Adds a TIPC server to the server pool." - rv = self._db.add_tipc_server(port) - if not rv: - raise NetworkError - return rv - - def add_tcp_server(self, addr, port = -1): - "Adds a TCP server to the server pool." - rv = self._db.add_tcp_server(addr, port) - if not rv: - raise NetworkError - return rv - - def add_udp_server(self, addr, port = -1): - "Adds an UDP server to the server pool." - rv = self._db.add_udp_server(addr, port) - if not rv: - raise NetworkError - return rv - - - def generic_get(self, getf, key): - "d[k] Returns the value associated with the key k." - if self.autopickle: - key = pickle.dumps(key, protocol = -1) - try: - r = getf(key) - except: - raise NetworkError - if r == -1: - # For key errors, get returns -1 instead of a string - # so we know it's a miss. - raise KeyError - if self.autopickle: - r = pickle.loads(r) - return r - - def cache_get(self, key): - return self.generic_get(self._db.cache_get, key) - - def normal_get(self, key): - return self.generic_get(self._db.get, key) - - - def generic_set(self, setf, key, val): - "d[k] = v Associates the value v to the key k." - if self.autopickle: - key = pickle.dumps(key, protocol = -1) - val = pickle.dumps(val, protocol = -1) - r = setf(key, val) - if r <= 0: - raise NetworkError - return 1 - - def cache_set(self, key, val): - return self.generic_set(self._db.cache_set, key, val) - - def normal_set(self, key, val): - return self.generic_set(self._db.set, key, val) - - def set_sync(self, key, val): - return self.generic_set(self._db.set_sync, key, val) - - - def generic_delete(self, delf, key): - "del d[k] Deletes the key k." - if self.autopickle: - key = pickle.dumps(key, protocol = -1) - r = delf(key) - if r < 0: - raise NetworkError - elif r == 0: - raise KeyError - return 1 - - def cache_delete(self, key): - return self.generic_delete(self._db.cache_delete, key) - - def normal_delete(self, key): - return self.generic_delete(self._db.delete, key) - - def delete_sync(self, key): - return self.generic_delete(self._db.delete_sync, key) - - - def generic_cas(self, casf, key, oldval, newval): - "Perform a compare-and-swap." - if self.autopickle: - key = pickle.dumps(key, protocol = -1) - oldval = pickle.dumps(oldval, protocol = -1) - newval = pickle.dumps(newval, protocol = -1) - r = casf(key, oldval, newval) - if r == 2: - # success - return 2 - elif r == 1: - # no match - return 1 - elif r == 0: - # not in - raise KeyError - else: - raise NetworkError - - def cache_cas(self, key, oldv, newv): - return self.generic_cas(self._db.cache_cas, key, - oldval, newval) - - def normal_cas(self, key, oldval, newval): - return self.generic_cas(self._db.cas, key, - oldval, newval) - - - def generic_incr(self, incrf, key, increment): - """Atomically increment the value associated with the given - key by the given increment.""" - if self.autopickle: - key = pickle.dumps(key, protocol = -1) - r, v = incrf(key, increment) - if r == 2: - # success - return v - elif r == 1: - # no match, because the value didn't have the right - # format - raise TypeError("The value must be a " + \ - "NULL-terminated string") - elif r == 0: - # not in - raise KeyError - else: - raise NetworkError - - def cache_incr(self, key, increment = 1): - return self.generic_incr(self._db.cache_incr, key, increment) - - def normal_incr(self, key, increment = 1): - return self.generic_incr(self._db.incr, key, increment) - - - # The following functions will assume the existance of self.set, - # self.get, and self.delete, which are supposed to be set by our - # subclasses. - - def __getitem__(self, key): - return self.get(key) - - def __setitem__(self, key, val): - return self.set(key, val) - - def __delitem__(self, key): - return self.delete(key) - - def __contains__(self, key): - "Returns True if the key is in the database, False otherwise." - try: - r = self.get(key) - except KeyError: - return False - if not r: - return False - return True - - def has_key(self, key): - "Returns True if the key is in the database, False otherwise." - return self.__contains__(key) - - - -class Cache (GenericDB): - get = GenericDB.cache_get - set = GenericDB.cache_set - delete = GenericDB.cache_delete - cas = GenericDB.cache_cas - incr = GenericDB.cache_incr - -class DB (GenericDB): - get = GenericDB.normal_get - set = GenericDB.normal_set - delete = GenericDB.normal_delete - cas = GenericDB.normal_cas - incr = GenericDB.normal_incr - -class SyncDB (GenericDB): - get = GenericDB.normal_get - set = GenericDB.set_sync - delete = GenericDB.delete_sync - cas = GenericDB.normal_cas - incr = GenericDB.normal_incr - - diff --git a/bindings/python3/nmdb_ll.c b/bindings/python3/nmdb_ll.c deleted file mode 100644 index a6a4ab8..0000000 --- a/bindings/python3/nmdb_ll.c +++ /dev/null @@ -1,452 +0,0 @@ - -/* - * Python 3 bindings for libnmdb - * Alberto Bertogli (albertito@blitiri.com.ar) - * - * This is the low-level module, used by the python one to construct - * friendlier objects. - */ - -#include -#include - - -/* - * Type definitions - */ - -typedef struct { - PyObject_HEAD; - nmdb_t *db; -} nmdbobject; -static PyTypeObject nmdbType; - -/* - * The nmdb object - */ - -/* delete */ -static void db_dealloc(nmdbobject *db) -{ - if (db->db) { - nmdb_free(db->db); - } - PyObject_Del(db); -} - - -/* add tipc server */ -static PyObject *db_add_tipc_server(nmdbobject *db, PyObject *args) -{ - int port; - int rv; - - if (!PyArg_ParseTuple(args, "i:add_tipc_server", &port)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_add_tipc_server(db->db, port); - Py_END_ALLOW_THREADS - - return PyLong_FromLong(rv); -} - -/* add tcp server */ -static PyObject *db_add_tcp_server(nmdbobject *db, PyObject *args) -{ - int port; - char *addr; - int rv; - - if (!PyArg_ParseTuple(args, "si:add_tcp_server", &addr, &port)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_add_tcp_server(db->db, addr, port); - Py_END_ALLOW_THREADS - - return PyLong_FromLong(rv); -} - -/* add udp server */ -static PyObject *db_add_udp_server(nmdbobject *db, PyObject *args) -{ - int port; - char *addr; - int rv; - - if (!PyArg_ParseTuple(args, "si:add_udp_server", &addr, &port)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_add_udp_server(db->db, addr, port); - Py_END_ALLOW_THREADS - - return PyLong_FromLong(rv); -} - -/* cache set */ -static PyObject *db_cache_set(nmdbobject *db, PyObject *args) -{ - unsigned char *key, *val; - int ksize, vsize; - int rv; - - if (!PyArg_ParseTuple(args, "s#s#:cache_set", &key, &ksize, - &val, &vsize)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_cache_set(db->db, key, ksize, val, vsize); - Py_END_ALLOW_THREADS - - return PyLong_FromLong(rv); -} - -/* cache get */ -static PyObject *db_cache_get(nmdbobject *db, PyObject *args) -{ - unsigned char *key, *val; - int ksize, vsize; - long rv; - PyObject *r; - - if (!PyArg_ParseTuple(args, "s#:cache_get", &key, &ksize)) { - return NULL; - } - - /* vsize is enough to hold the any value */ - vsize = 128 * 1024; - val = malloc(vsize); - if (val == NULL) - return PyErr_NoMemory(); - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_cache_get(db->db, key, ksize, val, vsize); - Py_END_ALLOW_THREADS - - if (rv <= -2) { - /* FIXME: define a better exception */ - r = PyErr_SetFromErrno(PyExc_IOError); - } else if (rv == -1) { - /* Miss, handled in the high-level module. */ - r = PyLong_FromLong(-1); - } else { - r = PyBytes_FromStringAndSize((char *) val, rv); - } - - free(val); - return r; -} - -/* cache delete */ -static PyObject *db_cache_delete(nmdbobject *db, PyObject *args) -{ - unsigned char *key; - int ksize; - int rv; - - if (!PyArg_ParseTuple(args, "s#:cache_delete", &key, &ksize)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_cache_del(db->db, key, ksize); - Py_END_ALLOW_THREADS - - return PyLong_FromLong(rv); -} - -/* cache cas */ -static PyObject *db_cache_cas(nmdbobject *db, PyObject *args) -{ - unsigned char *key, *oldval, *newval; - int ksize, ovsize, nvsize; - int rv; - - if (!PyArg_ParseTuple(args, "s#s#s#:cache_cas", &key, &ksize, - &oldval, &ovsize, - &newval, &nvsize)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_cache_cas(db->db, key, ksize, oldval, ovsize, - newval, nvsize); - Py_END_ALLOW_THREADS - - return PyLong_FromLong(rv); -} - -/* cache increment */ -static PyObject *db_cache_incr(nmdbobject *db, PyObject *args) -{ - unsigned char *key; - int ksize; - int rv; - long long int increment; - int64_t newval; - - if (!PyArg_ParseTuple(args, "s#L:cache_incr", &key, &ksize, - &increment)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_cache_incr(db->db, key, ksize, increment, &newval); - Py_END_ALLOW_THREADS - - return Py_BuildValue("LL", rv, newval); -} - - -/* db set */ -static PyObject *db_set(nmdbobject *db, PyObject *args) -{ - unsigned char *key, *val; - int ksize, vsize; - int rv; - - if (!PyArg_ParseTuple(args, "s#s#:set", &key, &ksize, - &val, &vsize)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_set(db->db, key, ksize, val, vsize); - Py_END_ALLOW_THREADS - - return PyLong_FromLong(rv); -} - -/* db get */ -static PyObject *db_get(nmdbobject *db, PyObject *args) -{ - unsigned char *key, *val; - int ksize, vsize; - long rv; - PyObject *r; - - if (!PyArg_ParseTuple(args, "s#:get", &key, &ksize)) { - return NULL; - } - - /* vsize is enough to hold the any value */ - vsize = 128 * 1024; - val = malloc(vsize); - if (val == NULL) - return PyErr_NoMemory(); - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_get(db->db, key, ksize, val, vsize); - Py_END_ALLOW_THREADS - - if (rv <= -2) { - /* FIXME: define a better exception */ - r = PyErr_SetFromErrno(PyExc_IOError); - } else if (rv == -1) { - /* Miss, handled in the high-level module. */ - r = PyLong_FromLong(-1); - } else { - r = PyBytes_FromStringAndSize((char *) val, rv); - } - - free(val); - return r; -} - -/* db delete */ -static PyObject *db_delete(nmdbobject *db, PyObject *args) -{ - unsigned char *key; - int ksize; - int rv; - - if (!PyArg_ParseTuple(args, "s#:delete", &key, &ksize)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_del(db->db, key, ksize); - Py_END_ALLOW_THREADS - - return PyLong_FromLong(rv); -} - -/* db cas */ -static PyObject *db_cas(nmdbobject *db, PyObject *args) -{ - unsigned char *key, *oldval, *newval; - int ksize, ovsize, nvsize; - int rv; - - if (!PyArg_ParseTuple(args, "s#s#s#:cas", &key, &ksize, - &oldval, &ovsize, - &newval, &nvsize)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_cas(db->db, key, ksize, oldval, ovsize, newval, nvsize); - Py_END_ALLOW_THREADS - - return PyLong_FromLong(rv); -} - -/* db increment */ -static PyObject *db_incr(nmdbobject *db, PyObject *args) -{ - unsigned char *key; - int ksize; - int rv; - long long int increment; - int64_t newval; - - if (!PyArg_ParseTuple(args, "s#L:incr", &key, &ksize, &increment)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_incr(db->db, key, ksize, increment, &newval); - Py_END_ALLOW_THREADS - - return Py_BuildValue("LL", rv, newval); -} - - -/* db set sync */ -static PyObject *db_set_sync(nmdbobject *db, PyObject *args) -{ - unsigned char *key, *val; - int ksize, vsize; - int rv; - - if (!PyArg_ParseTuple(args, "s#s#:set_sync", &key, &ksize, - &val, &vsize)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_set_sync(db->db, key, ksize, val, vsize); - Py_END_ALLOW_THREADS - - return PyLong_FromLong(rv); -} - -/* db delete sync */ -static PyObject *db_delete_sync(nmdbobject *db, PyObject *args) -{ - unsigned char *key; - int ksize; - int rv; - - if (!PyArg_ParseTuple(args, "s#:delete_sync", &key, &ksize)) { - return NULL; - } - - Py_BEGIN_ALLOW_THREADS - rv = nmdb_del_sync(db->db, key, ksize); - Py_END_ALLOW_THREADS - - return PyLong_FromLong(rv); -} - - - -/* nmdb method table */ - -static PyMethodDef nmdb_methods[] = { - { "add_tipc_server", (PyCFunction) db_add_tipc_server, - METH_VARARGS, NULL }, - { "add_tcp_server", (PyCFunction) db_add_tcp_server, - METH_VARARGS, NULL }, - { "add_udp_server", (PyCFunction) db_add_udp_server, - METH_VARARGS, NULL }, - { "cache_set", (PyCFunction) db_cache_set, METH_VARARGS, NULL }, - { "cache_get", (PyCFunction) db_cache_get, METH_VARARGS, NULL }, - { "cache_delete", (PyCFunction) db_cache_delete, METH_VARARGS, NULL }, - { "cache_cas", (PyCFunction) db_cache_cas, METH_VARARGS, NULL }, - { "cache_incr", (PyCFunction) db_cache_incr, METH_VARARGS, NULL }, - { "set", (PyCFunction) db_set, METH_VARARGS, NULL }, - { "get", (PyCFunction) db_get, METH_VARARGS, NULL }, - { "delete", (PyCFunction) db_delete, METH_VARARGS, NULL }, - { "cas", (PyCFunction) db_cas, METH_VARARGS, NULL }, - { "incr", (PyCFunction) db_incr, METH_VARARGS, NULL }, - { "set_sync", (PyCFunction) db_set_sync, METH_VARARGS, NULL }, - { "delete_sync", (PyCFunction) db_delete_sync, METH_VARARGS, NULL }, - - { NULL } -}; - -/* new, returns an nmdb object */ -static PyObject *db_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - nmdbobject *db; - - db = (nmdbobject *) type->tp_alloc(type, 0); - if (db == NULL) - return NULL; - - if (!PyArg_ParseTuple(args, ":new")) { - return NULL; - } - - db->db = nmdb_init(); - if (db->db == NULL) { - return PyErr_NoMemory(); - } - - /* XXX: is this necessary? */ - if (PyErr_Occurred()) { - nmdb_free(db->db); - return NULL; - } - - return (PyObject *) db; -} - - -static PyTypeObject nmdbType = { - PyObject_HEAD_INIT(NULL) - .tp_name = "nmdb_ll.nmdb", - .tp_itemsize = sizeof(nmdbobject), - .tp_dealloc = (destructor) db_dealloc, - .tp_methods = nmdb_methods, - .tp_new = db_new, -}; - - - -/* - * The module - */ - -static PyModuleDef nmdb_module = { - PyModuleDef_HEAD_INIT, - .m_name = "nmdb_ll", - .m_doc = NULL, - .m_size = -1, -}; - - -PyMODINIT_FUNC PyInit_nmdb_ll(void) -{ - PyObject *m; - - if (PyType_Ready(&nmdbType) < 0) - return NULL; - - m = PyModule_Create(&nmdb_module); - - Py_INCREF(&nmdbType); - PyModule_AddObject(m, "nmdb", (PyObject *) &nmdbType); - - return m; -} - - - diff --git a/bindings/python3/setup.py b/bindings/python3/setup.py deleted file mode 100644 index 54c1bcd..0000000 --- a/bindings/python3/setup.py +++ /dev/null @@ -1,17 +0,0 @@ - -from distutils.core import setup, Extension - -nmdb_ll = Extension("nmdb_ll", - libraries = ['nmdb'], - sources = ['nmdb_ll.c']) - -setup( - name = 'nmdb', - description = "libnmdb bindings", - author = "Alberto Bertogli", - author_email = "albertito@blitiri.com.ar", - url = "http://blitiri.com.ar/p/nmdb", - py_modules = ['nmdb'], - ext_modules = [nmdb_ll] -) - diff --git a/bindings/ruby/LICENSE b/bindings/ruby/LICENSE index f3a9498..987d3b9 100644 --- a/bindings/ruby/LICENSE +++ b/bindings/ruby/LICENSE @@ -2,12 +2,11 @@ I don't like licenses, because I don't like having to worry about all this legal stuff just for a simple piece of software I don't really mind anyone using. But I also believe that it's important that people share and give back; -so I'm placing this library under the following license, so you feel guilty if -you don't ;) +so I'm placing this work under the following license. -BOLA - Buena Onda License Agreement ------------------------------------ +BOLA - Buena Onda License Agreement (v1.1) +------------------------------------------ This work is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of @@ -16,19 +15,16 @@ this work. To all effects and purposes, this work is to be considered Public Domain. -However, if you want to be "Buena onda", you should: +However, if you want to be "buena onda", you should: 1. Not take credit for it, and give proper recognition to the authors. 2. Share your modifications, so everybody benefits from them. -4. Do something nice for the authors. -5. Help someone who needs it: sign up for some volunteer work or help your +3. Do something nice for the authors. +4. Help someone who needs it: sign up for some volunteer work or help your neighbour paint the house. -6. Don't waste. Anything, but specially energy that comes from natural +5. Don't waste. Anything, but specially energy that comes from natural non-renewable resources. Extra points if you discover or invent something to replace them. -7. Be tolerant. Everything that's good in nature comes from cooperation. - -The order is important, and the further you go the more "Buena onda" you are. -Make the world a better place: be "Buena onda". +6. Be tolerant. Everything that's good in nature comes from cooperation. diff --git a/doc/design.rst b/doc/design.rst index 4470fec..bb8e467 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -123,8 +123,8 @@ operation. A specific solution could have been used, and the database backend code is isolated enough to allow this to happen in the future if necessity arises. -Several backends are supported (at the moment QDBM_, BDB_, tokyocabinet_ and a -null backend); the selection is done at build time. +Several backends are supported (at the moment QDBM_, BDB_, tokyocabinet_, tdb_ +and a null backend); the selection is done at build time. The processing is performed by taking requests from the aforementioned queue, and acting upon the database accordingly, which involves calling the backend's @@ -163,13 +163,13 @@ The cache layer is implemented by a modified hash table, to make eviction efficient and cheap. The hash table is quite normal: several buckets (the size is decided at -initialization time), and each bucket containing a linked list with the -objects assigned to it. +initialization time), and each bucket containing a list with the objects +assigned to it. There a some tricks, though: - In order to keep a bound on the number of objects in the cache, the number - of elements in each linked list is limited to 4. + of elements in each list is limited to 4. - Whenever a lookup is made, the entry that matched is promoted to the head of the list containing it. - When inserting a new element in the cache, it's always inserted to the top @@ -204,5 +204,6 @@ pattern involves handling lots of different keys. .. _memcached: http://www.danga.com/memcached/ .. _QDBM: http://qdbm.sf.net .. _BDB: http://www.oracle.com/technology/products/berkeley-db/db/ -.. _tokyocabinet: http://tokyocabinet.sf.net/index.html +.. _tokyocabinet: http://1978th.net/tokyocabinet/ +.. _tdb: http://tdb.samba.org diff --git a/doc/guide.rst b/doc/guide.rst index 3d94cfd..5af4056 100644 --- a/doc/guide.rst +++ b/doc/guide.rst @@ -31,11 +31,9 @@ Prerequisites Before you install nmdb, you will need the following software: - libevent_, a library for fast event handling. -- Either QDBM_, BDB_ or tokyocabinet_ for the database backend. - -And, if you're going to use TIPC_: - -- `Linux kernel`_ 2.6.16 or newer, compiled with TIPC_ support. +- One or more of QDBM_, BDB_, tokyocabinet_ or tdb_ for the database backend. +- If you want TIPC_ support, `Linux kernel`_ 2.6.16 or newer. +- If you want SCTP_ support, libsctp-dev (or equivalent package). Compiling and installing @@ -49,8 +47,9 @@ To install the server and the C library, run ``make install; ldconfig``. To install the Python module, run ``make python_install`` after installing the C library. -If you want to disable support for some protocol (i.e. TIPC), you can do so by -running ``make ENABLE_TIPC=0 install``. +The build system autodetects the available backends and protocols, but if you +want to manually override it, you can you can do so by running, for example, +``make ENABLE_TIPC=0 BE_ENABLE_BDB=1 install`` to disable TIPC and enable BDB. Quick start @@ -141,8 +140,8 @@ Cache size in it, and not by byte size. Backend database - The backend database engine can be selected at build time; QDBM_ is the - default. + The backend database engine can be selected at run time via a command-line + option. If for some reason (hardware failure, for instance) the database becomes corrupt, you should use your database utilities to fix it. It shouldn't @@ -259,7 +258,7 @@ the default port. The Python module ------------------- +----------------- The Python module it's quite easy to use, because its interface is very similar to a dictionary. It has similar limitations regarding the key (it must @@ -416,11 +415,13 @@ know at albertito@blitiri.com.ar. .. _nmdb: http://blitiri.com.ar/p/nmdb/ .. _libevent: http://www.monkey.org/~provos/libevent/ .. _TIPC: http://tipc.sf.net +.. _SCTP: http://en.wikipedia.org/wiki/Stream_Control_Transmission_Protocol .. _memcached: http://www.danga.com/memcached/ .. _`Linux kernel`: http://kernel.org .. _tetrations: http://en.wikipedia.org/wiki/Tetration .. _QDBM: http://qdbm.sf.net .. _BDB: http://www.oracle.com/technology/products/berkeley-db/db/ -.. _tokyocabinet: http://tokyocabinet.sf.net/index.html +.. _tokyocabinet: http://1978th.net/tokyocabinet/ +.. _tdb: http://tdb.samba.org diff --git a/doc/network.rst b/doc/network.rst index 13f0d36..7d2dc34 100644 --- a/doc/network.rst +++ b/doc/network.rst @@ -24,7 +24,7 @@ payload. They look like this:: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Version | Request ID | + | Ver | Request ID | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Request code | Flags | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ @@ -65,6 +65,9 @@ REQ_SET 0x102 REQ_DEL 0x103 REQ_CAS 0x104 REQ_INCR 0x105 +REQ_STATS 0x106 +REQ_FIRSTKEY 0x107 +REQ_NEXTKEY 0x108 ============== ====== @@ -101,6 +104,10 @@ REQ_CAS REQ_INCR First the key size (32 bits), then the key, and then the increment as a signed network byte order 64 bit integer. +REQ_FIRSTKEY + No payload. +REQ_NEXTKEY + The key size and then the key. Replies @@ -157,9 +164,10 @@ REP_CACHE_HIT REP_OK Depending on the request, this reply does or doesn't have an associated value. For *REQ_SET**, *REQ_DEL** and *REQ_CAS** there is no payload. But - for *REQ_GET* the first 32 bits are the value size, and then the value; and - for *REQ_INCR* the first 32 bits are the payload size, and then the - post-increment value as a signed 64-bit integer in network byte order. + for *REQ_GET* and *REQ_NEXTKEY* the first 32 bits are the value size, and + then the value; and for *REQ_INCR* the first 32 bits are the payload size, + and then the post-increment value as a signed 64-bit integer in network byte + order. Reply error codes @@ -174,6 +182,7 @@ ERR_BROKEN 0x103 Broken request ERR_UNKREQ 0x104 Unknown request ERR_MEM 0x105 Memory allocation error ERR_DB 0x106 Database error +ERR_RO 0x107 Server in read-only mode ============ ====== ========================= diff --git a/libnmdb/LICENSE b/libnmdb/LICENSE index f3a9498..987d3b9 100644 --- a/libnmdb/LICENSE +++ b/libnmdb/LICENSE @@ -2,12 +2,11 @@ I don't like licenses, because I don't like having to worry about all this legal stuff just for a simple piece of software I don't really mind anyone using. But I also believe that it's important that people share and give back; -so I'm placing this library under the following license, so you feel guilty if -you don't ;) +so I'm placing this work under the following license. -BOLA - Buena Onda License Agreement ------------------------------------ +BOLA - Buena Onda License Agreement (v1.1) +------------------------------------------ This work is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of @@ -16,19 +15,16 @@ this work. To all effects and purposes, this work is to be considered Public Domain. -However, if you want to be "Buena onda", you should: +However, if you want to be "buena onda", you should: 1. Not take credit for it, and give proper recognition to the authors. 2. Share your modifications, so everybody benefits from them. -4. Do something nice for the authors. -5. Help someone who needs it: sign up for some volunteer work or help your +3. Do something nice for the authors. +4. Help someone who needs it: sign up for some volunteer work or help your neighbour paint the house. -6. Don't waste. Anything, but specially energy that comes from natural +5. Don't waste. Anything, but specially energy that comes from natural non-renewable resources. Extra points if you discover or invent something to replace them. -7. Be tolerant. Everything that's good in nature comes from cooperation. - -The order is important, and the further you go the more "Buena onda" you are. -Make the world a better place: be "Buena onda". +6. Be tolerant. Everything that's good in nature comes from cooperation. diff --git a/libnmdb/Makefile b/libnmdb/Makefile index 831cc3f..7e2beaf 100644 --- a/libnmdb/Makefile +++ b/libnmdb/Makefile @@ -1,15 +1,13 @@ -ENABLE_TIPC = 1 ENABLE_TCP = 1 ENABLE_UDP = 1 -ENABLE_SCTP = 1 +ENABLE_TIPC := $(shell if echo "\#include " | \ + $(CPP) - > /dev/null 2>&1; then echo 1; else echo 0; fi) +ENABLE_SCTP := $(shell if echo "\#include " | \ + $(CPP) - > /dev/null 2>&1; then echo 1; else echo 0; fi) CFLAGS += -std=c99 -pedantic -Wall -O3 ALL_CFLAGS = -D_XOPEN_SOURCE=500 -fPIC $(CFLAGS) -ALL_CFLAGS += -DENABLE_TIPC=$(ENABLE_TIPC) \ - -DENABLE_TCP=$(ENABLE_TCP) \ - -DENABLE_UDP=$(ENABLE_UDP) \ - -DENABLE_SCTP=$(ENABLE_SCTP) ifdef DEBUG ALL_CFLAGS += -g @@ -38,14 +36,14 @@ default: all all: libs libnmdb.pc -nmdb.h: nmdb.skel.h - @echo "generating nmdb.h" - @cat nmdb.skel.h | \ +internal.h: internal.h.in + @echo "generating internal.h" + @cat internal.h.in | \ sed 's/++CONFIG_ENABLE_TIPC++/$(ENABLE_TIPC)/g' | \ sed 's/++CONFIG_ENABLE_TCP++/$(ENABLE_TCP)/g' | \ sed 's/++CONFIG_ENABLE_UDP++/$(ENABLE_UDP)/g' | \ sed 's/++CONFIG_ENABLE_SCTP++/$(ENABLE_SCTP)/g' \ - > nmdb.h + > internal.h libnmdb.pc: libnmdb.skel.pc @echo "generating libnmdb.pc" @@ -55,10 +53,10 @@ libnmdb.pc: libnmdb.skel.pc libs: libnmdb.so libnmdb.a -libnmdb.so: nmdb.h $(OBJS) +libnmdb.so: internal.h $(OBJS) $(NICE_CC) $(ALL_CFLAGS) -shared -fPIC $(OBJS) -o libnmdb.so -libnmdb.a: nmdb.h $(OBJS) +libnmdb.a: internal.h $(OBJS) $(AR) cr libnmdb.a $(OBJS) @@ -80,14 +78,17 @@ install-man: install: install-lib install-man +doxygen: + $(MAKE) -C doxygen/ .c.o: $(NICE_CC) $(ALL_CFLAGS) -c $< -o $@ clean: - rm -f nmdb.h libnmdb.pc $(OBJS) libnmdb.so libnmdb.a + rm -f internal.h libnmdb.pc $(OBJS) libnmdb.so libnmdb.a rm -f *.bb *.bbg *.da *.gcov *.gcda *.gcno gmon.out + $(MAKE) -C doxygen $@ -.PHONY: default all libs install-lib install-man install clean +.PHONY: default all libs install-lib install-man install doxygen clean diff --git a/libnmdb/doxygen/Doxyfile.base.in b/libnmdb/doxygen/Doxyfile.base.in new file mode 100644 index 0000000..9a74040 --- /dev/null +++ b/libnmdb/doxygen/Doxyfile.base.in @@ -0,0 +1,204 @@ +DOXYFILE_ENCODING = UTF-8 +PROJECT_NAME = libnmdb +PROJECT_NUMBER = ++VERSION++ +OUTPUT_DIRECTORY = +CREATE_SUBDIRS = NO +OUTPUT_LANGUAGE = English +BRIEF_MEMBER_DESC = YES +REPEAT_BRIEF = YES +ABBREVIATE_BRIEF = +ALWAYS_DETAILED_SEC = NO +INLINE_INHERITED_MEMB = NO +FULL_PATH_NAMES = NO +STRIP_FROM_PATH = +STRIP_FROM_INC_PATH = +SHORT_NAMES = NO +JAVADOC_AUTOBRIEF = YES +QT_AUTOBRIEF = YES +MULTILINE_CPP_IS_BRIEF = NO +INHERIT_DOCS = YES +SEPARATE_MEMBER_PAGES = NO +TAB_SIZE = 8 +ALIASES = +OPTIMIZE_OUTPUT_FOR_C = YES +OPTIMIZE_OUTPUT_JAVA = NO +OPTIMIZE_FOR_FORTRAN = NO +OPTIMIZE_OUTPUT_VHDL = NO +EXTENSION_MAPPING = +BUILTIN_STL_SUPPORT = NO +CPP_CLI_SUPPORT = NO +SIP_SUPPORT = NO +IDL_PROPERTY_SUPPORT = YES +DISTRIBUTE_GROUP_DOC = NO +SUBGROUPING = YES +TYPEDEF_HIDES_STRUCT = NO +SYMBOL_CACHE_SIZE = 0 +EXTRACT_ALL = NO +EXTRACT_PRIVATE = NO +EXTRACT_STATIC = NO +EXTRACT_LOCAL_CLASSES = YES +EXTRACT_LOCAL_METHODS = NO +EXTRACT_ANON_NSPACES = NO +HIDE_UNDOC_MEMBERS = NO +HIDE_UNDOC_CLASSES = NO +HIDE_FRIEND_COMPOUNDS = NO +HIDE_IN_BODY_DOCS = NO +INTERNAL_DOCS = NO +CASE_SENSE_NAMES = YES +HIDE_SCOPE_NAMES = NO +SHOW_INCLUDE_FILES = YES +INLINE_INFO = YES +SORT_MEMBER_DOCS = NO +SORT_BRIEF_DOCS = NO +SORT_GROUP_NAMES = NO +SORT_BY_SCOPE_NAME = NO +GENERATE_TODOLIST = YES +GENERATE_TESTLIST = YES +GENERATE_BUGLIST = YES +GENERATE_DEPRECATEDLIST= YES +ENABLED_SECTIONS = +MAX_INITIALIZER_LINES = 30 +SHOW_USED_FILES = YES +SHOW_DIRECTORIES = YES +SHOW_FILES = YES +SHOW_NAMESPACES = YES +FILE_VERSION_FILTER = +LAYOUT_FILE = +QUIET = YES +WARNINGS = YES +WARN_IF_UNDOCUMENTED = YES +WARN_IF_DOC_ERROR = YES +WARN_NO_PARAMDOC = YES +WARN_FORMAT = "$file:$line: $text" +WARN_LOGFILE = +INPUT = ../ +INPUT_ENCODING = UTF-8 +FILE_PATTERNS = *.c *.h *.doxy +RECURSIVE = YES +EXCLUDE = +EXCLUDE_SYMLINKS = NO +EXCLUDE_PATTERNS = +EXCLUDE_SYMBOLS = +EXAMPLE_PATH = +EXAMPLE_PATTERNS = +EXAMPLE_RECURSIVE = NO +IMAGE_PATH = +INPUT_FILTER = +FILTER_PATTERNS = +FILTER_SOURCE_FILES = NO +SOURCE_BROWSER = YES +INLINE_SOURCES = NO +STRIP_CODE_COMMENTS = YES +REFERENCED_BY_RELATION = YES +REFERENCES_RELATION = NO +REFERENCES_LINK_SOURCE = NO +USE_HTAGS = NO +VERBATIM_HEADERS = YES +ALPHABETICAL_INDEX = NO +COLS_IN_ALPHA_INDEX = 5 +IGNORE_PREFIX = +GENERATE_HTML = YES +HTML_OUTPUT = html +HTML_FILE_EXTENSION = .html +HTML_HEADER = +HTML_FOOTER = +HTML_STYLESHEET = +HTML_ALIGN_MEMBERS = YES +HTML_DYNAMIC_SECTIONS = YES +GENERATE_DOCSET = NO +DOCSET_FEEDNAME = "Doxygen generated docs" +DOCSET_BUNDLE_ID = org.doxygen.Project +GENERATE_HTMLHELP = NO +CHM_FILE = +HHC_LOCATION = +GENERATE_CHI = NO +CHM_INDEX_ENCODING = +BINARY_TOC = NO +TOC_EXPAND = YES +GENERATE_QHP = NO +QCH_FILE = +QHP_NAMESPACE = +QHP_VIRTUAL_FOLDER = doc +QHP_CUST_FILTER_NAME = +QHP_CUST_FILTER_ATTRS = +QHP_SECT_FILTER_ATTRS = +QHG_LOCATION = +DISABLE_INDEX = NO +ENUM_VALUES_PER_LINE = 4 +GENERATE_TREEVIEW = NONE +TREEVIEW_WIDTH = 250 +FORMULA_FONTSIZE = 10 +GENERATE_LATEX = NO +LATEX_OUTPUT = latex +LATEX_CMD_NAME = latex +MAKEINDEX_CMD_NAME = makeindex +COMPACT_LATEX = NO +PAPER_TYPE = a4wide +EXTRA_PACKAGES = +LATEX_HEADER = +PDF_HYPERLINKS = YES +USE_PDFLATEX = YES +LATEX_BATCHMODE = NO +LATEX_HIDE_INDICES = NO +GENERATE_RTF = NO +RTF_OUTPUT = rtf +COMPACT_RTF = NO +RTF_HYPERLINKS = NO +RTF_STYLESHEET_FILE = +RTF_EXTENSIONS_FILE = +GENERATE_MAN = NO +MAN_OUTPUT = man +MAN_EXTENSION = .3 +MAN_LINKS = NO +GENERATE_XML = NO +XML_OUTPUT = xml +XML_SCHEMA = +XML_DTD = +XML_PROGRAMLISTING = YES +GENERATE_AUTOGEN_DEF = NO +GENERATE_PERLMOD = NO +PERLMOD_LATEX = NO +PERLMOD_PRETTY = YES +PERLMOD_MAKEVAR_PREFIX = +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = NO +EXPAND_ONLY_PREDEF = NO +SEARCH_INCLUDES = YES +INCLUDE_PATH = +INCLUDE_FILE_PATTERNS = +PREDEFINED = +EXPAND_AS_DEFINED = +SKIP_FUNCTION_MACROS = YES +TAGFILES = +GENERATE_TAGFILE = +ALLEXTERNALS = NO +EXTERNAL_GROUPS = YES +PERL_PATH = /usr/bin/perl +CLASS_DIAGRAMS = YES +MSCGEN_PATH = +HIDE_UNDOC_RELATIONS = YES +HAVE_DOT = YES +DOT_FONTNAME = FreeSans +DOT_FONTSIZE = 10 +DOT_FONTPATH = +CLASS_GRAPH = YES +COLLABORATION_GRAPH = YES +GROUP_GRAPHS = YES +UML_LOOK = NO +TEMPLATE_RELATIONS = NO +INCLUDE_GRAPH = YES +INCLUDED_BY_GRAPH = YES +CALL_GRAPH = YES +CALLER_GRAPH = YES +GRAPHICAL_HIERARCHY = YES +DIRECTORY_GRAPH = YES +DOT_IMAGE_FORMAT = png +DOT_PATH = +DOTFILE_DIRS = +DOT_GRAPH_MAX_NODES = 50 +MAX_DOT_GRAPH_DEPTH = 0 +DOT_TRANSPARENT = YES +DOT_MULTI_TARGETS = NO +GENERATE_LEGEND = YES +DOT_CLEANUP = YES +SEARCHENGINE = NO diff --git a/libnmdb/doxygen/Doxyfile.public b/libnmdb/doxygen/Doxyfile.public new file mode 100644 index 0000000..9c9c209 --- /dev/null +++ b/libnmdb/doxygen/Doxyfile.public @@ -0,0 +1,8 @@ +@INCLUDE = Doxyfile.base +PROJECT_NAME = "libnmdb (public)" +OUTPUT_DIRECTORY = doc.public +EXTRACT_ALL = YES +EXTRACT_STATIC = NO +INTERNAL_DOCS = NO +INPUT = ../nmdb.h + diff --git a/libnmdb/doxygen/Makefile b/libnmdb/doxygen/Makefile new file mode 100644 index 0000000..71a2a2f --- /dev/null +++ b/libnmdb/doxygen/Makefile @@ -0,0 +1,30 @@ + +ifneq ($(V), 1) + NICE_DOXYGEN = @echo " DOXYGEN $@"; doxygen +else + NICE_DOXYGEN = doxygen +endif + + +default: all + +all: public + +# $(LIB_VER) must be defined externally if we want the generated docs to +# specify a version number. Usually, this Makefile will be invoked by +# libnmdb's, which has that variable properly defined. +Doxyfile.base: Doxyfile.base.in + @echo "generating Doxyfile.base" + @cat Doxyfile.base.in | \ + sed 's@++VERSION++@$(LIB_VER)@g' \ + > Doxyfile.base + +public: Doxyfile.base + $(NICE_DOXYGEN) Doxyfile.public + +clean: + rm -rf doc.public Doxyfile.base + + +.PHONY: all clean default doxygen public + diff --git a/libnmdb/internal.h b/libnmdb/internal.h deleted file mode 100644 index d80424c..0000000 --- a/libnmdb/internal.h +++ /dev/null @@ -1,29 +0,0 @@ - -#ifndef _INTERNAL_H -#define _INTERNAL_H - -/* Different connection types. Used internally to differentiate between the - * different protocols in struct nmdb_srv. */ -#define TIPC_CONN 1 -#define TCP_CONN 2 -#define UDP_CONN 3 -#define SCTP_CONN 4 - -/* The ID code for requests is hardcoded for now, until asynchronous requests - * are implemented. */ -#define ID_CODE 1 - -/* For a given buffer, how much into it should the generic library code write - * the message contents. */ -#define TIPC_MSG_OFFSET 0 -#define TCP_MSG_OFFSET 4 -#define UDP_MSG_OFFSET 0 -#define SCTP_MSG_OFFSET 0 - -/* Functions used internally but shared among the different files. */ -int compare_servers(const void *s1, const void *s2); -ssize_t srecv(int fd, unsigned char *buf, size_t count, int flags); -ssize_t ssend(int fd, const unsigned char *buf, size_t count, int flags); - -#endif - diff --git a/libnmdb/internal.h.in b/libnmdb/internal.h.in new file mode 100644 index 0000000..dc875f9 --- /dev/null +++ b/libnmdb/internal.h.in @@ -0,0 +1,83 @@ + +#ifndef _INTERNAL_H +#define _INTERNAL_H + +/* Different connection types. Used internally to differentiate between the + * different protocols in struct nmdb_srv. */ +#define TIPC_CONN 1 +#define TCP_CONN 2 +#define UDP_CONN 3 +#define SCTP_CONN 4 + +/* The ID code for requests is hardcoded for now, until asynchronous requests + * are implemented. */ +#define ID_CODE 1 + +/* For a given buffer, how much into it should the generic library code write + * the message contents. */ +#define TIPC_MSG_OFFSET 0 +#define TCP_MSG_OFFSET 4 +#define UDP_MSG_OFFSET 0 +#define SCTP_MSG_OFFSET 0 + +/* Defined to 0 or 1 at libnmdb build time according the build configuration, + * not to be used externally. */ +#define ENABLE_TIPC ++CONFIG_ENABLE_TIPC++ +#define ENABLE_TCP ++CONFIG_ENABLE_TCP++ +#define ENABLE_UDP ++CONFIG_ENABLE_UDP++ +#define ENABLE_SCTP ++CONFIG_ENABLE_SCTP++ + +/* Functions used internally but shared among the different files. */ +int compare_servers(const void *s1, const void *s2); +ssize_t srecv(int fd, unsigned char *buf, size_t count, int flags); +ssize_t ssend(int fd, const unsigned char *buf, size_t count, int flags); + + +/* + * Server and connection internal types. + */ + +#include /* socket defines */ +#include /* socklen_t */ + +#if ENABLE_TIPC +#include /* struct sockaddr_tipc */ +#endif + +#if (ENABLE_TCP || ENABLE_UDP || ENABLE_SCTP) +#include /* struct sockaddr_in */ +#endif + +/* A connection to an nmdb server. */ +struct nmdb_srv { + int fd; + int type; + union { + +#if ENABLE_TIPC + struct { + unsigned int port; + struct sockaddr_tipc srvsa; + socklen_t srvlen; + } tipc; +#endif + +#if (ENABLE_TCP || ENABLE_UDP || ENABLE_SCTP) + struct { + struct sockaddr_in srvsa; + socklen_t srvlen; + } in; +#endif + + } info; +}; + +/* A connection to one or many nmdb servers. */ +struct nmdb_conn { + unsigned int nservers; + struct nmdb_srv *servers; +}; + + +#endif + diff --git a/libnmdb/libnmdb.3 b/libnmdb/libnmdb.3 index 226255f..b1d814f 100644 --- a/libnmdb/libnmdb.3 +++ b/libnmdb/libnmdb.3 @@ -1,7 +1,7 @@ .TH libnmdb 3 "11/Sep/2006" .SH NAME libnmdb - Library for interacting with a nmdb server -.SH SYNOPSYS +.SH SYNOPSIS .nf .B #include .sp @@ -141,8 +141,8 @@ variant is used), or -2 on failure. .BR nmdb_del () is used to remove a given key (and it's associated value). The normal variant -returns 1 if it was queued successfuly, or < 0 on failure. The cache and -synchronous variant return 1 if the key was removed successfuly, 0 if the key +returns 1 if it was queued successfully, or < 0 on failure. The cache and +synchronous variant return 1 if the key was removed successfully, 0 if the key was not in the database/cache, or < 0 on failure. .BR nmdb_cas () diff --git a/libnmdb/libnmdb.c b/libnmdb/libnmdb.c index 6cb612d..3ad8e0c 100644 --- a/libnmdb/libnmdb.c +++ b/libnmdb/libnmdb.c @@ -8,15 +8,22 @@ #include /* close() */ #include "nmdb.h" +#include "internal.h" #include "net-const.h" #include "tipc.h" #include "tcp.h" #include "udp.h" #include "sctp.h" -#include "internal.h" #include "netutils.h" +/* Possible flags, notice it may make no sense to mix them, consult the + * documentation before doing weird things. Values should be kept in sync with + * the internal net-const.h */ +#define NMDB_CACHE_ONLY 1 +#define NMDB_SYNC 2 + + /* Compares two servers by their connection identifiers. It is used internally * to keep the server array sorted with qsort(). */ int compare_servers(const void *s1, const void *s2) @@ -650,6 +657,116 @@ int nmdb_cache_incr(nmdb_t *db, const unsigned char *key, size_t ksize, } +ssize_t nmdb_firstkey(nmdb_t *db, unsigned char *key, size_t ksize) +{ + ssize_t rv, t; + unsigned char *buf, *p; + size_t bufsize, payload_offset, psize = 0; + uint32_t reply; + struct nmdb_srv *srv; + + if (db->nservers != 1) { + return -2; + } + srv = &(db->servers[0]); + + buf = new_packet(srv, REQ_FIRSTKEY, 0, &bufsize, &payload_offset, -1); + if (buf == NULL) + return -2; + + t = srv_send(srv, buf, payload_offset); + if (t <= 0) { + rv = -2; + goto exit; + } + + reply = get_rep(srv, buf, bufsize, &p, &psize); + + if (reply == REP_NOTIN) { + rv = -1; + goto exit; + } else if (reply == REP_ERR) { + rv = -2; + goto exit; + } else if (reply != REP_OK) { + /* invalid response */ + rv = -2; + goto exit; + } + + /* we've got an answer */ + rv = * (uint32_t *) p; + rv = ntohl(rv); + if (rv > (psize - 4) || rv > ksize) { + /* the value is too big for the packet size, or it is too big + * to fit in the buffer we were given */ + rv = -2; + goto exit; + } + memcpy(key, p + 4, rv); + +exit: + free(buf); + return rv; + +} + +ssize_t nmdb_nextkey(nmdb_t *db, const unsigned char *key, size_t ksize, + unsigned char *newkey, size_t nksize) +{ + ssize_t rv, t; + unsigned char *buf, *p; + size_t bufsize, reqsize, payload_offset, psize = 0; + uint32_t reply; + struct nmdb_srv *srv; + + if (db->nservers != 1) + return -2; + srv = &(db->servers[0]); + + buf = new_packet(srv, REQ_NEXTKEY, 0, &bufsize, &payload_offset, -1); + if (buf == NULL) + return -1; + reqsize = payload_offset; + reqsize += append_1v(buf + payload_offset, key, ksize); + + t = srv_send(srv, buf, reqsize); + if (t <= 0) { + rv = -2; + goto exit; + } + + reply = get_rep(srv, buf, bufsize, &p, &psize); + + if (reply == REP_NOTIN) { + rv = -1; + goto exit; + } else if (reply == REP_ERR) { + rv = -2; + goto exit; + } else if (reply != REP_OK) { + /* invalid response */ + rv = -2; + goto exit; + } + + /* we've got an answer */ + rv = * (uint32_t *) p; + rv = ntohl(rv); + if (rv > (psize - 4) || rv > nksize) { + /* the value is too big for the packet size, or it is too big + * to fit in the buffer we were given */ + rv = -2; + goto exit; + } + memcpy(newkey, p + 4, rv); + +exit: + free(buf); + return rv; +} + + /* Request servers' statistics, return the aggregated results in buf, with the * number of servers in nservers and the number of stats per server in nstats. * Used in the "nmdb-stats" utility, matches the server version. diff --git a/libnmdb/libnmdb.skel.pc b/libnmdb/libnmdb.skel.pc index f0f3c01..1a12d88 100644 --- a/libnmdb/libnmdb.skel.pc +++ b/libnmdb/libnmdb.skel.pc @@ -6,7 +6,7 @@ includedir=${prefix}/include Name: libnmdb Description: Library for interacting with a nmdb server URL: http://blitiri.com.ar/p/nmdb/ -Version: 0.22 +Version: 0.23 Libs: -L${libdir} -lnmdb Cflags: -I${includedir} diff --git a/libnmdb/nmdb.h b/libnmdb/nmdb.h new file mode 100644 index 0000000..3c5d49a --- /dev/null +++ b/libnmdb/nmdb.h @@ -0,0 +1,355 @@ + +/* Header for the libnmdb library. */ + +#ifndef _NMDB_H +#define _NMDB_H + +#include /* size_t, ssize_t */ + +/** Opaque type representing a connection with one or more servers. */ +typedef struct nmdb_conn nmdb_t; + + +/** + * @addtogroup connection Connection API + * Functions used to connect to the servers. + */ + +/** Create a new nmdb_t pointer, used to talk with the server. + * + * @returns a new connection instance. + * @ingroup connection + */ +nmdb_t *nmdb_init(); + +/** Add a TIPC server. + * + * @param db connection instance. + * @param port TIPC port to connect to (-1 means use the default). + * @returns 1 on success, 0 on failure. + * @ingroup connection + */ +int nmdb_add_tipc_server(nmdb_t *db, int port); + +/** Add a TCP server. + * + * @param db connection instance. + * @param addr host to connect to. + * @param port port to connect to (-1 means use the default). + * @returns 1 on success, 0 on failure. + * @ingroup connection + */ +int nmdb_add_tcp_server(nmdb_t *db, const char *addr, int port); + +/** Add an UDP server. + * + * @param db connection instance. + * @param addr host to connect to. + * @param port port to connect to (-1 means use the default). + * @returns 1 on success, 0 on failure. + * @ingroup connection + */ +int nmdb_add_udp_server(nmdb_t *db, const char *addr, int port); + +/** Add a SCTP server. + * + * @param db connection instance. + * @param addr host to connect to. + * @param port port to connect to (-1 means use the default). + * @returns 1 on success, 0 on failure. + * @ingroup connection + */ +int nmdb_add_sctp_server(nmdb_t *db, const char *addr, int port); + +/** Free a nmdb_t structure created by nmdb_init(). + * It also closes all the connections opened to the servers. + * + * @param db connection instance. + * @returns 1 on success, 0 on failure. + * @ingroup connection + */ +int nmdb_free(nmdb_t *db); + + +/** + * @addtogroup database Database API + * Functions that affect the database and the cache. + * + * @addtogroup cache Cache API + * Functions that affect only the cache. + */ + +/** Get the value associated with a key. + * + * @param db connection instance. + * @param key the key. + * @param ksize the key size. + * @param[out] val buffer where the value will be stored. + * @param vsize size of the value buffer. + * @returns the size of the value written to the given buffer, -1 if the key + * is not in the database, or -2 if there was an error. + * @ingroup database + */ +ssize_t nmdb_get(nmdb_t *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t vsize); + +/** Get the value associated with a key from cache. + * This is just like nmdb_get(), except it only queries the caches, and never + * the database. + * + * @param db connection instance. + * @param key the key. + * @param ksize the key size. + * @param[out] val buffer where the value will be stored. + * @param vsize size of the value buffer. + * @returns the size of the value written to the given buffer, -1 if the key + * is not in the cache, or -2 if there was an error. + * @ingroup cache + */ +ssize_t nmdb_cache_get(nmdb_t *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t vsize); + +/** Set the value associated with a key. + * It returns after the command has been acknowledged by the server, but does + * not wait for the database to confirm it. In any case, further GET requests + * will return this value, even if this set has not reached the backend yet. + * + * @param db connection instance. + * @param key the key. + * @param ksize the key size. + * @param val the value + * @param vsize size of the value. + * @returns 1 on success, < 0 on error. + * @ingroup database + */ +int nmdb_set(nmdb_t *db, const unsigned char *key, size_t ksize, + const unsigned char *val, size_t vsize); + +/** Set the value associated with a key, synchronously. + * It works just like nmdb_set(), except it returns only after the database + * confirms it has stored the value. + * + * @param db connection instance. + * @param key the key. + * @param ksize the key size. + * @param val the value + * @param vsize size of the value. + * @returns 1 on success, < 0 on error. + * @ingroup database + */ +int nmdb_set_sync(nmdb_t *db, const unsigned char *key, size_t ksize, + const unsigned char *val, size_t vsize); + +/** Set the value associated with a key, but only in the cache. + * This command sets the key's value in the cache, but does NOT affect the + * backend database. Combining it with nmdb_set() and/or nmdb_set_sync() is + * not recommended, because consistency issues may arise. + * + * @param db connection instance. + * @param key the key. + * @param ksize the key size. + * @param val the value + * @param vsize size of the value. + * @returns 1 on success, < 0 on error. + * @ingroup cache + */ +int nmdb_cache_set(nmdb_t *db, const unsigned char *key, size_t ksize, + const unsigned char *val, size_t vsize); + +/** Delete a key. + * It returns after the command has been acknowledged by the server, but does + * not wait for the database to confirm it. In any case, further GET requests + * will return the key is missing, even if this delete has not reached the + * backend yet. + * + * @param db connection instance. + * @param key the key. + * @param ksize the key size. + * @returns 1 on success, < 0 on error. + * @ingroup database + */ +int nmdb_del(nmdb_t *db, const unsigned char *key, size_t ksize); + +/** Delete a key synchronously. + * It works just like nmdb_del(), except it returns only after the database + * confirms it has deleted the key. + * + * @param db connection instance. + * @param key the key. + * @param ksize the key size. + * @returns 1 on success, < 0 on error. + * @ingroup database + */ +int nmdb_del_sync(nmdb_t *db, const unsigned char *key, size_t ksize); + +/** Delete a key, but only from the cache. + * This command deletes a key from the cache, but does NOT affect the backend + * database. Combining it with nmdb_del() and/or nmdb_del_sync() is not + * recommended, because consistency issues may arise. + * + * @param db connection instance. + * @param key the key. + * @param ksize the key size. + * @returns 1 on success, < 0 on error. + * @ingroup cache + */ +int nmdb_cache_del(nmdb_t *db, const unsigned char *key, size_t ksize); + +/** Perform an atomic compare-and-swap. + * This command will set the value associated to a key to newval, provided it + * currently is oldval, in an atomic way. + * Equivalent to atomically doing: + * + * if get(key) == oldval, then set(key, newval) + * + * @param db connection instance. + * @param key the key. + * @param ksize the key size. + * @param oldval the old, expected value. + * @param ovsize size of the expected value. + * @param newval the new value to set. + * @param nvsize size of the new value. + * @returns 2 on success, 1 if the current value does not match oldval, 0 if + * the key is not in the database, or < 0 on error. + * @ingroup database + */ +int nmdb_cas(nmdb_t *db, const unsigned char *key, size_t ksize, + const unsigned char *oldval, size_t ovsize, + const unsigned char *newval, size_t nvsize); + +/** Perform an atomic compare-and-swap only on the cache. + * This command works just like nmdb_cas(), except it affects only the cache, + * and not the backend database. + * Equivalent to atomically doing: + * + * if get_cache(key) == oldval, then set_cache(key, newval) + * + * @param db connection instance. + * @param key the key. + * @param ksize the key size. + * @param oldval the old, expected value. + * @param ovsize size of the expected value. + * @param newval the new value to set. + * @param nvsize size of the new value. + * @returns 2 on success, 1 if the current value does not match oldval, 0 if + * the key is not in the database, or < 0 on error. + * @ingroup cache + */ +int nmdb_cache_cas(nmdb_t *db, const unsigned char *key, size_t ksize, + const unsigned char *oldval, size_t ovsize, + const unsigned char *newval, size_t nvsize); + +/** Atomically increment the value associated with a key. + * This command atomically increments the value associated with a key in the + * given increment. However, there are requirements on the current value: it + * must be a NULL-terminated string with only a number in base 10 in it (i.e. + * it must be parseable by strtoll(3)). + * + * @param db connection instance. + * @param key the key. + * @param ksize the key size. + * @param increment the value to add to the current one (can be negative). + * @param[out] newval pointer to an integer that will be set to the new + * value. + * @returns 2 if the increment was successful, 1 if the current value was not + * in an appropriate format, 0 if the key is not in the database, or < 0 + * on error. + * @ingroup database + */ +int nmdb_incr(nmdb_t *db, const unsigned char *key, size_t ksize, + int64_t increment, int64_t *newval); + +/** Atomically increment the value associated with a key only in the cache. + * This command works just like nmdb_incr(), except it affects only the cache, + * and not the backend database. + * + * @param db connection instance. + * @param key the key. + * @param ksize the key size. + * @param increment the value to add to the current one (can be negative). + * @param[out] newval pointer to an integer that will be set to the new + * value. + * @returns 2 if the increment was successful, 1 if the current value was not + * in an appropriate format, 0 if the key is not in the database, or < 0 + * on error. + * @ingroup cache + */ +int nmdb_cache_incr(nmdb_t *db, const unsigned char *key, size_t ksize, + int64_t increment, int64_t *newval); + + +/** + * @addtogroup utility Functions used in nmdb utilities + * These functions are used almost exclusively by nmdb utilities, although + * they may be used by external applications. They often require some + * knowledge about nmdb's inner workings so they should be used with care. + */ + +/** Get the first key. + * Returns the first key in the database, which can then be used to get the + * following one with nmdb_nextkey(). Together, they can be used to iterate + * over all the keys of a *single server*. It has some caveats: + * + * - It will fail if db has more than one server. + * - If the database is being modified during iteration, the walk can result + * in skipping nodes or walking the same one twice. + * - There is absolutely no guarantee about the order of the keys. + * - The order is not stable and must not be relied upon. + * + * This is almost exclusively used for replication utilities. + * + * @param db connection instance. + * @param[out] key the first key. + * @param ksize the key size. + * @returns -2 on error, -1 if the database is empty, or the key size on + * success. + * @ingroup utility + */ +ssize_t nmdb_firstkey(nmdb_t *db, unsigned char *key, size_t ksize); + +/** Get the key that follows the one given. + * Together with nmdb_firstkey(), they can be used to iterate This function, + * along with nmdb_firstkey(), are used to iterate over all the keys of a + * *single server*. It has some caveats: + * + * - It will fail if db has more than one server. + * - If the database is being modified during iteration, the walk can result + * in skipping nodes or walking the same one twice. + * - There is absolutely no guarantee about the order of the keys. + * - The order is not stable and must not be relied upon. + * + * This is almost exclusively used for replication utilities. + * + * @param db connection instance. + * @param key the current key. + * @param ksize the current key size. + * @param[out] newkey the key that follows the current one. + * @param nksize the newkey size. + * @returns -2 on error, -1 if the database is empty, or the new key size on + * success. + * @ingroup utility + */ +ssize_t nmdb_nextkey(nmdb_t *db, const unsigned char *key, size_t ksize, + unsigned char *newkey, size_t nksize); + + +/** Request servers' statistics. + * This API is used by nmdb-stats, and likely to change in the future. Do not + * rely on it. + * + * @param db connection instance. + * @param[out] buf buffer used to store the results. + * @param bsize size of the buffer. + * @param[out] nservers number of servers queried. + * @param [out] nstats number of stats per server. + * @returns 1 if success, -1 if there was an error in the server, -2 if there + * was a network error, -3 if the buffer was too small, -4 if the server + * replies were of different size (indicates different server versions, + * not supported at the time) + * @ingroup utility + */ +int nmdb_stats(nmdb_t *db, unsigned char *buf, size_t bsize, + unsigned int *nservers, unsigned int *nstats); + +#endif + diff --git a/libnmdb/nmdb.skel.h b/libnmdb/nmdb.skel.h deleted file mode 100644 index 448fb15..0000000 --- a/libnmdb/nmdb.skel.h +++ /dev/null @@ -1,101 +0,0 @@ - -/* Header for the libnmdb library. */ - -#ifndef _NMDB_H -#define _NMDB_H - -/* Defined to 0 or 1 at libnmdb build time according the build configuration, - * not to be used externally. */ -#define _ENABLE_TIPC ++CONFIG_ENABLE_TIPC++ -#define _ENABLE_TCP ++CONFIG_ENABLE_TCP++ -#define _ENABLE_UDP ++CONFIG_ENABLE_UDP++ -#define _ENABLE_SCTP ++CONFIG_ENABLE_SCTP++ - - -#include /* socket defines */ -#include /* socklen_t */ - -#if _ENABLE_TIPC -#include /* struct sockaddr_tipc */ -#endif - -#if (_ENABLE_TCP || _ENABLE_UDP || _ENABLE_SCTP) -#include /* struct sockaddr_in */ -#endif - -struct nmdb_srv { - int fd; - int type; - union { - -#if _ENABLE_TIPC - struct { - unsigned int port; - struct sockaddr_tipc srvsa; - socklen_t srvlen; - } tipc; -#endif - -#if (_ENABLE_TCP || _ENABLE_UDP || _ENABLE_SCTP) - struct { - struct sockaddr_in srvsa; - socklen_t srvlen; - } in; -#endif - - } info; -}; - -typedef struct nmdb_t { - unsigned int nservers; - struct nmdb_srv *servers; -} nmdb_t; - - -/* Possible flags, notice it may make no sense to mix them, consult the - * documentation before doing weird things. Values should be kept in sync with - * the internal net-const.h */ -#define NMDB_CACHE_ONLY 1 -#define NMDB_SYNC 2 - - -nmdb_t *nmdb_init(); -int nmdb_add_tipc_server(nmdb_t *db, int port); -int nmdb_add_tcp_server(nmdb_t *db, const char *addr, int port); -int nmdb_add_udp_server(nmdb_t *db, const char *addr, int port); -int nmdb_add_sctp_server(nmdb_t *db, const char *addr, int port); -int nmdb_free(nmdb_t *db); - -ssize_t nmdb_get(nmdb_t *db, const unsigned char *key, size_t ksize, - unsigned char *val, size_t vsize); -ssize_t nmdb_cache_get(nmdb_t *db, const unsigned char *key, size_t ksize, - unsigned char *val, size_t vsize); - -int nmdb_set(nmdb_t *db, const unsigned char *key, size_t ksize, - const unsigned char *val, size_t vsize); -int nmdb_set_sync(nmdb_t *db, const unsigned char *key, size_t ksize, - const unsigned char *val, size_t vsize); -int nmdb_cache_set(nmdb_t *db, const unsigned char *key, size_t ksize, - const unsigned char *val, size_t vsize); - -int nmdb_del(nmdb_t *db, const unsigned char *key, size_t ksize); -int nmdb_del_sync(nmdb_t *db, const unsigned char *key, size_t ksize); -int nmdb_cache_del(nmdb_t *db, const unsigned char *key, size_t ksize); - -int nmdb_cas(nmdb_t *db, const unsigned char *key, size_t ksize, - const unsigned char *oldval, size_t ovsize, - const unsigned char *newval, size_t nvsize); -int nmdb_cache_cas(nmdb_t *db, const unsigned char *key, size_t ksize, - const unsigned char *oldval, size_t ovsize, - const unsigned char *newval, size_t nvsize); - -int nmdb_incr(nmdb_t *db, const unsigned char *key, size_t ksize, - int64_t increment, int64_t *newval); -int nmdb_cache_incr(nmdb_t *db, const unsigned char *key, size_t ksize, - int64_t increment, int64_t *newval); - -int nmdb_stats(nmdb_t *db, unsigned char *buf, size_t bsize, - unsigned int *nservers, unsigned int *nstats); - -#endif - diff --git a/libnmdb/sctp.h b/libnmdb/sctp.h index 21eb074..b44ebc9 100644 --- a/libnmdb/sctp.h +++ b/libnmdb/sctp.h @@ -2,6 +2,8 @@ #ifndef _SCTP_H #define _SCTP_H +#include "internal.h" + int sctp_srv_send(struct nmdb_srv *srv, unsigned char *buf, size_t bsize); uint32_t sctp_get_rep(struct nmdb_srv *srv, unsigned char *buf, size_t bsize, diff --git a/libnmdb/tcp.h b/libnmdb/tcp.h index 1459d42..881ee73 100644 --- a/libnmdb/tcp.h +++ b/libnmdb/tcp.h @@ -2,6 +2,8 @@ #ifndef _TCP_H #define _TCP_H +#include "internal.h" + int tcp_srv_send(struct nmdb_srv *srv, unsigned char *buf, size_t bsize); uint32_t tcp_get_rep(struct nmdb_srv *srv, unsigned char *buf, size_t bsize, diff --git a/libnmdb/tipc.h b/libnmdb/tipc.h index 82b8cb2..db29a70 100644 --- a/libnmdb/tipc.h +++ b/libnmdb/tipc.h @@ -2,6 +2,8 @@ #ifndef _TIPC_H #define _TIPC_H +#include "internal.h" + int tipc_srv_send(struct nmdb_srv *srv, const unsigned char *buf, size_t bsize); uint32_t tipc_get_rep(struct nmdb_srv *srv, diff --git a/libnmdb/udp.h b/libnmdb/udp.h index c989a83..74444eb 100644 --- a/libnmdb/udp.h +++ b/libnmdb/udp.h @@ -2,6 +2,8 @@ #ifndef _UDP_H #define _UDP_H +#include "internal.h" + int udp_srv_send(struct nmdb_srv *srv, unsigned char *buf, size_t bsize); uint32_t udp_get_rep(struct nmdb_srv *srv, unsigned char *buf, size_t bsize, diff --git a/nmdb/LICENSE b/nmdb/LICENSE index 0ed89f2..987d3b9 100644 --- a/nmdb/LICENSE +++ b/nmdb/LICENSE @@ -1,184 +1,30 @@ -This project, "nmdb", is copyrighted by Alberto Bertogli and licensed under -the Open Software License version 3.0 as obtained from -http://www.opensource.org (and included here-in for easy reference) (that -license itself is copyrighted by Lawrence Rosen). +I don't like licenses, because I don't like having to worry about all this +legal stuff just for a simple piece of software I don't really mind anyone +using. But I also believe that it's important that people share and give back; +so I'm placing this work under the following license. - Alberto Bertogli - 11/September/2006 ------------------------------------------------------------------------------ +BOLA - Buena Onda License Agreement (v1.1) +------------------------------------------ +This work is provided 'as-is', without any express or implied warranty. In no +event will the authors be held liable for any damages arising from the use of +this work. -Open Software License (“OSL”) v. 3.0 +To all effects and purposes, this work is to be considered Public Domain. -This Open Software License (the "License") applies to any original work of -authorship (the "Original Work") whose owner (the "Licensor") has placed the -following licensing notice adjacent to the copyright notice for the Original -Work: -Licensed under the Open Software License version 3.0 +However, if you want to be "buena onda", you should: -1) Grant of Copyright License. Licensor grants You a worldwide, royalty-free, -non-exclusive, sublicensable license, for the duration of the copyright, to do -the following: +1. Not take credit for it, and give proper recognition to the authors. +2. Share your modifications, so everybody benefits from them. +3. Do something nice for the authors. +4. Help someone who needs it: sign up for some volunteer work or help your + neighbour paint the house. +5. Don't waste. Anything, but specially energy that comes from natural + non-renewable resources. Extra points if you discover or invent something + to replace them. +6. Be tolerant. Everything that's good in nature comes from cooperation. -a) to reproduce the Original Work in copies, either alone or as part of a -collective work; -b) to translate, adapt, alter, transform, modify, or arrange the Original -Work, thereby creating derivative works ("Derivative Works") based upon the -Original Work; - -c) to distribute or communicate copies of the Original Work and Derivative -Works to the public, with the proviso that copies of Original Work or -Derivative Works that You distribute or communicate shall be licensed under -this Open Software License; - -d) to perform the Original Work publicly; and - -e) to display the Original Work publicly. - -2) Grant of Patent License. Licensor grants You a worldwide, royalty-free, -non-exclusive, sublicensable license, under patent claims owned or controlled -by the Licensor that are embodied in the Original Work as furnished by the -Licensor, for the duration of the patents, to make, use, sell, offer for sale, -have made, and import the Original Work and Derivative Works. - -3) Grant of Source Code License. The term "Source Code" means the preferred -form of the Original Work for making modifications to it and all available -documentation describing how to modify the Original Work. Licensor agrees to -provide a machine-readable copy of the Source Code of the Original Work along -with each copy of the Original Work that Licensor distributes. Licensor -reserves the right to satisfy this obligation by placing a machine-readable -copy of the Source Code in an information repository reasonably calculated to -permit inexpensive and convenient access by You for as long as Licensor -continues to distribute the Original Work. - -4) Exclusions From License Grant. Neither the names of Licensor, nor the names -of any contributors to the Original Work, nor any of their trademarks or -service marks, may be used to endorse or promote products derived from this -Original Work without express prior permission of the Licensor. Except as -expressly stated herein, nothing in this License grants any license to -Licensor’s trademarks, copyrights, patents, trade secrets or any other -intellectual property. No patent license is granted to make, use, sell, offer -for sale, have made, or import embodiments of any patent claims other than the -licensed claims defined in Section 2. No license is granted to the trademarks -of Licensor even if such marks are included in the Original Work. Nothing in -this License shall be interpreted to prohibit Licensor from licensing under -terms different from this License any Original Work that Licensor otherwise -would have a right to license. - -5) External Deployment. The term "External Deployment" means the use, -distribution, or communication of the Original Work or Derivative Works in any -way such that the Original Work or Derivative Works may be used by anyone -other than You, whether those works are distributed or communicated to those -persons or made available as an application intended for use over a network. -As an express condition for the grants of license hereunder, You must treat -any External Deployment by You of the Original Work or a Derivative Work as a -distribution under section 1(c). - -6) Attribution Rights. You must retain, in the Source Code of any Derivative -Works that You create, all copyright, patent, or trademark notices from the -Source Code of the Original Work, as well as any notices of licensing and any -descriptive text identified therein as an "Attribution Notice." You must cause -the Source Code for any Derivative Works that You create to carry a prominent -Attribution Notice reasonably calculated to inform recipients that You have -modified the Original Work. - -7) Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that -the copyright in and to the Original Work and the patent rights granted herein -by Licensor are owned by the Licensor or are sublicensed to You under the -terms of this License with the permission of the contributor(s) of those -copyrights and patent rights. Except as expressly stated in the immediately -preceding sentence, the Original Work is provided under this License on an "AS -IS" BASIS and WITHOUT WARRANTY, either express or implied, including, without -limitation, the warranties of non-infringement, merchantability or fitness for -a particular purpose. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK -IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an essential part of this -License. No license to the Original Work is granted by this License except -under this disclaimer. - -8) Limitation of Liability. Under no circumstances and under no legal theory, -whether in tort (including negligence), contract, or otherwise, shall the -Licensor be liable to anyone for any indirect, special, incidental, or -consequential damages of any character arising as a result of this License or -the use of the Original Work including, without limitation, damages for loss -of goodwill, work stoppage, computer failure or malfunction, or any and all -other commercial damages or losses. This limitation of liability shall not -apply to the extent applicable law prohibits such limitation. - -9) Acceptance and Termination. If, at any time, You expressly assented to this -License, that assent indicates your clear and irrevocable acceptance of this -License and all of its terms and conditions. If You distribute or communicate -copies of the Original Work or a Derivative Work, You must make a reasonable -effort under the circumstances to obtain the express assent of recipients to -the terms of this License. This License conditions your rights to undertake -the activities listed in Section 1, including your right to create Derivative -Works based upon the Original Work, and doing so without honoring these terms -and conditions is prohibited by copyright law and international treaty. -Nothing in this License is intended to affect copyright exceptions and -limitations (including “fair use” or “fair dealing”). This License shall -terminate immediately and You may no longer exercise any of the rights granted -to You by this License upon your failure to honor the conditions in Section -1(c). - -10) Termination for Patent Action. This License shall terminate automatically -and You may no longer exercise any of the rights granted to You by this -License as of the date You commence an action, including a cross-claim or -counterclaim, against Licensor or any licensee alleging that the Original Work -infringes a patent. This termination provision shall not apply for an action -alleging patent infringement by combinations of the Original Work with other -software or hardware. - -11) Jurisdiction, Venue and Governing Law. Any action or suit relating to this -License may be brought only in the courts of a jurisdiction wherein the -Licensor resides or in which Licensor conducts its primary business, and under -the laws of that jurisdiction excluding its conflict-of-law provisions. The -application of the United Nations Convention on Contracts for the -International Sale of Goods is expressly excluded. Any use of the Original -Work outside the scope of this License or after its termination shall be -subject to the requirements and penalties of copyright or patent law in the -appropriate jurisdiction. This section shall survive the termination of this -License. - -12) Attorneys’ Fees. In any action to enforce the terms of this License or -seeking damages relating thereto, the prevailing party shall be entitled to -recover its costs and expenses, including, without limitation, reasonable -attorneys' fees and costs incurred in connection with such action, including -any appeal of such action. This section shall survive the termination of this -License. - -13) Miscellaneous. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent necessary -to make it enforceable. - -14) Definition of "You" in This License. "You" throughout this License, -whether in upper or lower case, means an individual or a legal entity -exercising rights under, and complying with all of the terms of, this License. -For legal entities, "You" includes any entity that controls, is controlled by, -or is under common control with you. For purposes of this definition, -"control" means (i) the power, direct or indirect, to cause the direction or -management of such entity, whether by contract or otherwise, or (ii) ownership -of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial -ownership of such entity. - -15) Right to Use. You may use the Original Work in all ways not otherwise -restricted or conditioned by this License or by law, and Licensor promises not -to interfere with or be responsible for such uses by You. - -16) Modification of This License. This License is Copyright © 2005 Lawrence -Rosen. Permission is granted to copy, distribute, or communicate this License -without modification. Nothing in this License permits You to modify this -License as applied to the Original Work or to Derivative Works. However, You -may modify the text of this License and copy, distribute or communicate your -modified version (the "Modified License") and apply it to other original works -of authorship subject to the following conditions: (i) You may not indicate in -any way that your Modified License is the "Open Software License" or "OSL" and -you may not use those names in the name of your Modified License; (ii) You -must replace the notice specified in the first paragraph above with the notice -"Licensed under " or with a notice of your own -that is not confusingly similar to the notice in this License; and (iii) You -may not claim that your original works are open source software unless your -Modified License has been approved by Open Source Initiative (OSI) and You -comply with its license review and certification process. diff --git a/nmdb/Makefile b/nmdb/Makefile index 66a9f14..4f7f870 100644 --- a/nmdb/Makefile +++ b/nmdb/Makefile @@ -1,19 +1,36 @@ # Protocols to enable -ENABLE_TIPC = 1 ENABLE_TCP = 1 ENABLE_UDP = 1 -ENABLE_SCTP = 1 +ENABLE_TIPC := $(shell if echo "\#include " | \ + $(CPP) - > /dev/null 2>&1; then echo 1; else echo 0; fi) +ENABLE_SCTP := $(shell if echo "\#include " | \ + $(CPP) - > /dev/null 2>&1; then echo 1; else echo 0; fi) + +# Backends to enable +BE_ENABLE_QDBM := $(shell if `pkg-config --exists qdbm`; \ + then echo 1; else echo 0; fi) +BE_ENABLE_BDB := $(shell if echo "\#include " | \ + $(CPP) - > /dev/null 2>&1; then echo 1; else echo 0; fi) +BE_ENABLE_TC := $(shell if `pkg-config --exists tokyocabinet`; \ + then echo 1; else echo 0; fi) +BE_ENABLE_TDB := $(shell if `pkg-config --exists tdb`; \ + then echo 1; else echo 0; fi) +BE_ENABLE_NULL := 1 -# Backend to use, can be qdbm, bdb, tc, or null -BACKEND = qdbm CFLAGS += -std=c99 -pedantic -Wall -O3 ALL_CFLAGS = -D_XOPEN_SOURCE=600 $(CFLAGS) ALL_CFLAGS += -DENABLE_TIPC=$(ENABLE_TIPC) \ -DENABLE_TCP=$(ENABLE_TCP) \ -DENABLE_UDP=$(ENABLE_UDP) \ - -DENABLE_SCTP=$(ENABLE_SCTP) + -DENABLE_SCTP=$(ENABLE_SCTP) \ + -DBE_ENABLE_QDBM=$(BE_ENABLE_QDBM) \ + -DBE_ENABLE_BDB=$(BE_ENABLE_BDB) \ + -DBE_ENABLE_TC=$(BE_ENABLE_TC) \ + -DBE_ENABLE_TDB=$(BE_ENABLE_TDB) \ + -DBE_ENABLE_NULL=$(BE_ENABLE_NULL) \ + ifdef DEBUG ALL_CFLAGS += -g @@ -32,7 +49,8 @@ endif PREFIX=/usr/local -OBJS = cache.o dbloop.o queue.o log.o net.o netutils.o parse.o stats.o main.o +OBJS = cache.o dbloop.o queue.o log.o net.o netutils.o parse.o stats.o main.o \ + be.o be-bdb.o be-null.o be-qdbm.o be-tc.o be-tdb.o LIBS = -levent -lpthread -lrt @@ -60,27 +78,23 @@ else OBJS += sctp-stub.o endif -# Use series of ifeq-endif instead of else-ifeq because otherwise the nesting -# is a mess. Using "else ifeq ..." in the same line is only supported from -# gmake 3.81, which is too new. -ifeq ($(BACKEND), qdbm) - OBJS += be-qdbm.o - ALL_CFLAGS += `pkg-config qdbm --cflags` -DBACKEND_QDBM + +ifeq ($(BE_ENABLE_QDBM), 1) + ALL_CFLAGS += `pkg-config qdbm --cflags` LIBS += `pkg-config qdbm --libs-only-L` -lqdbm endif -ifeq ($(BACKEND), bdb) - OBJS += be-bdb.o - ALL_CFLAGS += -DBACKEND_BDB +ifeq ($(BE_ENABLE_BDB), 1) LIBS += -ldb endif -ifeq ($(BACKEND), tc) - OBJS += be-tc.o - ALL_CFLAGS += `pkg-config tokyocabinet --cflags` -DBACKEND_TC +ifeq ($(BE_ENABLE_TC), 1) + ALL_CFLAGS += `pkg-config tokyocabinet --cflags` LIBS += `pkg-config tokyocabinet --libs` endif -ifeq ($(BACKEND), null) - OBJS += be-null.o - ALL_CFLAGS += -DBACKEND_NULL +ifeq ($(BE_ENABLE_TDB), 1) + ALL_CFLAGS += `pkg-config tdb --cflags` + LIBS += `pkg-config tdb --libs` +endif +ifeq ($(BE_ENABLE_NULL), 1) endif diff --git a/nmdb/be-bdb.c b/nmdb/be-bdb.c index 4a930f8..55abd26 100644 --- a/nmdb/be-bdb.c +++ b/nmdb/be-bdb.c @@ -1,77 +1,115 @@ +#if BE_ENABLE_BDB + #include /* memset() */ +#include /* NULL */ +#include /* malloc() and friends */ + +/* typedefs to work around db.h include bug */ +typedef unsigned int u_int; +typedef unsigned long u_long; +#include + #include "be.h" -db_t *db_open(const char *name, int flags) +int bdb_set(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t vsize); +int bdb_get(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t *vsize); +int bdb_del(struct db_conn *db, const unsigned char *key, size_t ksize); +int bdb_close(struct db_conn *db); + + +struct db_conn *bdb_open(const char *name, int flags) { int rv; - db_t *db; + struct db_conn *db; + DB *bdb_db; - rv = db_create(&db, NULL, 0); + rv = db_create(&bdb_db, NULL, 0); if (rv != 0) return NULL; - rv = db->open(db, NULL, name, NULL, DB_HASH, DB_CREATE, 0); + rv = bdb_db->open(bdb_db, NULL, name, NULL, DB_HASH, DB_CREATE, 0); if (rv != 0) { - db->close(db, 0); + bdb_db->close(bdb_db, 0); return NULL; } + db = malloc(sizeof(struct db_conn)); + if (db == NULL) { + bdb_db->close(bdb_db, 0); + return NULL; + } + + db->conn = bdb_db; + db->set = bdb_set; + db->get = bdb_get; + db->del = bdb_del; + db->firstkey = NULL; + db->nextkey = NULL; + db->close = bdb_close; + return db; } -int db_close(db_t *db) +int bdb_close(struct db_conn *db) { int rv; + DB *bdb_db = (DB *) db->conn; - rv = db->close(db, 0); + rv = bdb_db->close(bdb_db, 0); if (rv != 0) return 0; + + free(db); return 1; } -int db_set(db_t *db, const unsigned char *key, size_t ksize, +int bdb_set(struct db_conn *db, const unsigned char *key, size_t ksize, unsigned char *val, size_t vsize) { int rv; DBT k, v; + DB *bdb_db = (DB *) db->conn; memset(&k, 0, sizeof(DBT)); memset(&v, 0, sizeof(DBT)); /* we can't maintain "const"ness here because bdb's prototypes; the - * same applies to get and del */ - k.data = key; + * same applies to get and del, so we just cast */ + k.data = (unsigned char *) key; k.size = ksize; - v.data = val; + v.data = (unsigned char *) val; v.size = vsize; - rv = db->put(db, NULL, &k, &v, 0); + rv = bdb_db->put(bdb_db, NULL, &k, &v, 0); if (rv != 0) return 0; return 1; } -int db_get(db_t *db, const unsigned char *key, size_t ksize, +int bdb_get(struct db_conn *db, const unsigned char *key, size_t ksize, unsigned char *val, size_t *vsize) { int rv; DBT k, v; + DB *bdb_db = (DB *) db->conn; memset(&k, 0, sizeof(DBT)); memset(&v, 0, sizeof(DBT)); - k.data = key; + k.data = (unsigned char *) key; k.size = ksize; v.data = val; v.ulen = *vsize; v.flags = DB_DBT_USERMEM; /* we supplied the memory */ - rv = db->get(db, NULL, &k, &v, 0); + rv = bdb_db->get(bdb_db, NULL, &k, &v, 0); if (rv != 0) { return 0; } else { @@ -80,20 +118,32 @@ int db_get(db_t *db, const unsigned char *key, size_t ksize, } } -int db_del(db_t *db, const unsigned char *key, size_t ksize) +int bdb_del(struct db_conn *db, const unsigned char *key, size_t ksize) { int rv; DBT k, v; + DB *bdb_db = (DB *) db->conn; memset(&k, 0, sizeof(DBT)); memset(&v, 0, sizeof(DBT)); - k.data = key; + k.data = (unsigned char *) key; k.size = ksize; - rv = db->del(db, NULL, &k, 0); + rv = bdb_db->del(bdb_db, NULL, &k, 0); if (rv != 0) return 0; return 1; } +#else + +#include /* NULL */ + +struct db_conn *bdb_open(const char *name, int flags) +{ + return NULL; +} + +#endif + diff --git a/nmdb/be-null.c b/nmdb/be-null.c index c358fa0..7b8d85e 100644 --- a/nmdb/be-null.c +++ b/nmdb/be-null.c @@ -1,37 +1,72 @@ +#if BE_ENABLE_NULL + #include /* size_t */ +#include /* malloc() and friends */ #include "be.h" -db_t *db_open(const char *name, int flags) +int null_set(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t vsize); +int null_get(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t *vsize); +int null_del(struct db_conn *db, const unsigned char *key, size_t ksize); +int null_close(struct db_conn *db); + + +struct db_conn *null_open(const char *name, int flags) { - /* Use a dumb not-null pointer because it is never looked at outside - * the functions defined here */ - return (db_t *) 1; + struct db_conn *db; + + db = malloc(sizeof(struct db_conn)); + if (db == NULL) + return NULL; + + db->conn = NULL; + db->set = null_set; + db->get = null_get; + db->del = null_del; + db->firstkey = NULL; + db->nextkey = NULL; + db->close = null_close; + + return db; } -int db_close(db_t *db) +int null_close(struct db_conn *db) { + free(db); return 1; } -int db_set(db_t *db, const unsigned char *key, size_t ksize, +int null_set(struct db_conn *db, const unsigned char *key, size_t ksize, unsigned char *val, size_t vsize) { return 1; } -int db_get(db_t *db, const unsigned char *key, size_t ksize, +int null_get(struct db_conn *db, const unsigned char *key, size_t ksize, unsigned char *val, size_t *vsize) { return 0; } -int db_del(db_t *db, const unsigned char *key, size_t ksize) +int null_del(struct db_conn *db, const unsigned char *key, size_t ksize) { return 0; } +#else + +#include /* NULL */ + +struct db_conn *null_open(const char *name, int flags) +{ + return NULL; +} + +#endif + diff --git a/nmdb/be-qdbm.c b/nmdb/be-qdbm.c index 3c98f9b..4a1291e 100644 --- a/nmdb/be-qdbm.c +++ b/nmdb/be-qdbm.c @@ -1,38 +1,73 @@ +#if BE_ENABLE_QDBM + #include /* QDBM's Depot API */ #include #include "be.h" -db_t *db_open(const char *name, int flags) +int qdbm_set(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t vsize); +int qdbm_get(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t *vsize); +int qdbm_del(struct db_conn *db, const unsigned char *key, size_t ksize); +int qdbm_close(struct db_conn *db); + + +struct db_conn *qdbm_open(const char *name, int flags) { int f; + struct db_conn *db; + DEPOT *qdbm_db; f = DP_OREADER | DP_OWRITER | DP_ONOLCK | DP_OCREAT; - return dpopen(name, f, 0); + qdbm_db = dpopen(name, f, 0); + if (qdbm_db == NULL) + return NULL; + + db = malloc(sizeof(struct db_conn)); + if (db == NULL) { + dpclose(qdbm_db); + return NULL; + } + + db->conn = qdbm_db; + db->set = qdbm_set; + db->get = qdbm_get; + db->del = qdbm_del; + db->firstkey = NULL; + db->nextkey = NULL; + db->close = qdbm_close; + + return db; } -int db_close(db_t *db) +int qdbm_close(struct db_conn *db) { - return dpclose(db); + int rv; + + rv = dpclose(db->conn); + free(db); + return rv; } -int db_set(db_t *db, const unsigned char *key, size_t ksize, +int qdbm_set(struct db_conn *db, const unsigned char *key, size_t ksize, unsigned char *val, size_t vsize) { - return dpput(db, (char *) key, ksize, (char *) val, vsize, DP_DOVER); + return dpput(db->conn, (char *) key, ksize, + (char *) val, vsize, DP_DOVER); } -int db_get(db_t *db, const unsigned char *key, size_t ksize, +int qdbm_get(struct db_conn *db, const unsigned char *key, size_t ksize, unsigned char *val, size_t *vsize) { int rv; - rv = dpgetwb(db, (char *) key, ksize, 0, *vsize, (char *) val); + rv = dpgetwb(db->conn, (char *) key, ksize, 0, *vsize, (char *) val); if (rv >= 0) { *vsize = rv; return 1; @@ -41,8 +76,19 @@ int db_get(db_t *db, const unsigned char *key, size_t ksize, } } -int db_del(db_t *db, const unsigned char *key, size_t ksize) +int qdbm_del(struct db_conn *db, const unsigned char *key, size_t ksize) { - return dpout(db, (char *) key, ksize); + return dpout(db->conn, (char *) key, ksize); } +#else + +#include /* NULL */ + +struct db_conn *qdbm_open(const char *name, int flags) +{ + return NULL; +} + +#endif + diff --git a/nmdb/be-tc.c b/nmdb/be-tc.c index 11f5267..319a844 100644 --- a/nmdb/be-tc.c +++ b/nmdb/be-tc.c @@ -1,42 +1,69 @@ +#if BE_ENABLE_TC + #include /* Tokyo Cabinet's hash API */ #include #include "be.h" -db_t *db_open(const char *name, int flags) +int tc_set(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t vsize); +int tc_get(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t *vsize); +int tc_del(struct db_conn *db, const unsigned char *key, size_t ksize); +int tc_close(struct db_conn *db); + + +struct db_conn *tc_open(const char *name, int flags) { - db_t *db = tchdbnew(); + struct db_conn *db; + TCHDB *tc_db = tchdbnew(); - if (!tchdbopen(db, name, HDBOWRITER | HDBOCREAT)) + if (!tchdbopen(tc_db, name, HDBOWRITER | HDBOCREAT)) return NULL; + db = malloc(sizeof(struct db_conn)); + if (db == NULL) { + tchdbclose(tc_db); + tchdbdel(tc_db); + return NULL; + } + + db->conn = tc_db; + db->set = tc_set; + db->get = tc_get; + db->del = tc_del; + db->firstkey = NULL; + db->nextkey = NULL; + db->close = tc_close; + return db; } -int db_close(db_t *db) +int tc_close(struct db_conn *db) { - int r = tchdbclose(db); - tchdbdel(db); + int r = tchdbclose(db->conn); + tchdbdel(db->conn); + free(db); return r; } -int db_set(db_t *db, const unsigned char *key, size_t ksize, +int tc_set(struct db_conn *db, const unsigned char *key, size_t ksize, unsigned char *val, size_t vsize) { - return tchdbput(db, key, ksize, val, vsize); + return tchdbput(db->conn, key, ksize, val, vsize); } -int db_get(db_t *db, const unsigned char *key, size_t ksize, +int tc_get(struct db_conn *db, const unsigned char *key, size_t ksize, unsigned char *val, size_t *vsize) { int rv; - rv = tchdbget3(db, key, ksize, val, *vsize); + rv = tchdbget3(db->conn, key, ksize, val, *vsize); if (rv >= 0) { *vsize = rv; return 1; @@ -45,8 +72,19 @@ int db_get(db_t *db, const unsigned char *key, size_t ksize, } } -int db_del(db_t *db, const unsigned char *key, size_t ksize) +int tc_del(struct db_conn *db, const unsigned char *key, size_t ksize) { - return tchdbout(db, key, ksize); + return tchdbout(db->conn, key, ksize); } +#else + +#include /* NULL */ + +struct db_conn *tc_open(const char *name, int flags) +{ + return NULL; +} + +#endif + diff --git a/nmdb/be-tdb.c b/nmdb/be-tdb.c new file mode 100644 index 0000000..0c3fca1 --- /dev/null +++ b/nmdb/be-tdb.c @@ -0,0 +1,155 @@ + +#if BE_ENABLE_TDB + +#include /* memcpy() */ +#include /* malloc() and friends */ + +/* tdb.h needs mode_t defined externally, and it is defined in one of these +(which are the ones required for open() */ +#include +#include +#include +#include + +#include "be.h" + +/* Local operations names are prepended with an 'x' so they don't collide with + * tdb real functions */ + +int xtdb_set(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t vsize); +int xtdb_get(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t *vsize); +int xtdb_del(struct db_conn *db, const unsigned char *key, size_t ksize); +int xtdb_firstkey(struct db_conn *db, unsigned char *key, size_t *ksize); +int xtdb_nextkey(struct db_conn *db, + const unsigned char *key, size_t ksize, + unsigned char *nextkey, size_t *nksize); +int xtdb_close(struct db_conn *db); + + +struct db_conn *xtdb_open(const char *name, int flags) +{ + struct db_conn *db; + TDB_CONTEXT *tdb_db; + + tdb_db = tdb_open(name, 0, 0, O_CREAT | O_RDWR, 0640); + if (tdb_db == NULL) + return NULL; + + db = malloc(sizeof(struct db_conn)); + if (db == NULL) { + tdb_close(tdb_db); + return NULL; + } + + db->conn = tdb_db; + db->set = xtdb_set; + db->get = xtdb_get; + db->del = xtdb_del; + db->firstkey = xtdb_firstkey; + db->nextkey = xtdb_nextkey; + db->close = xtdb_close; + + return db; +} + +int xtdb_close(struct db_conn *db) +{ + int rv; + + rv = tdb_close(db->conn); + free(db); + return rv == 0; +} + +int xtdb_set(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t vsize) +{ + TDB_DATA k, v; + + /* we can't maintain "const"ness here because tdb's prototypes; the + * same applies to get and del, so we just cast */ + k.dptr = (unsigned char *) key; + k.dsize = ksize; + v.dptr = (unsigned char *) val; + v.dsize = vsize; + + return tdb_store(db->conn, k, v, TDB_REPLACE) == 0; +} + +int xtdb_get(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t *vsize) +{ + TDB_DATA k, v; + + k.dptr = (unsigned char *) key; + k.dsize = ksize; + + v = tdb_fetch(db->conn, k); + if (v.dptr == NULL) + return 0; + + if (v.dsize > *vsize) + return 0; + + *vsize = v.dsize; + memcpy(val, v.dptr, v.dsize); + free(v.dptr); + return 1; +} + +int xtdb_del(struct db_conn *db, const unsigned char *key, size_t ksize) +{ + TDB_DATA k; + + k.dptr = (unsigned char *) key; + k.dsize = ksize; + + return tdb_delete(db->conn, k) == 0; +} + +int xtdb_firstkey(struct db_conn *db, unsigned char *key, size_t *ksize) +{ + TDB_DATA k; + + k = tdb_firstkey(db->conn); + if (k.dptr == NULL) + return 0; + + *ksize = k.dsize; + memcpy(key, k.dptr, k.dsize); + free(k.dptr); + return 1; +} + +int xtdb_nextkey(struct db_conn *db, + const unsigned char *key, size_t ksize, + unsigned char *nextkey, size_t *nksize) +{ + TDB_DATA pk, nk; + + pk.dptr = (unsigned char *) key; + pk.dsize = ksize; + + nk = tdb_nextkey(db->conn, pk); + if (nk.dptr == NULL) + return 0; + + *nksize = nk.dsize; + memcpy(nextkey, nk.dptr, nk.dsize); + free(nk.dptr); + return 1; +} + +#else + +#include /* NULL */ + +struct db_conn *xtdb_open(const char *name, int flags) +{ + return NULL; +} + +#endif + diff --git a/nmdb/be.c b/nmdb/be.c new file mode 100644 index 0000000..d1349e4 --- /dev/null +++ b/nmdb/be.c @@ -0,0 +1,61 @@ + +#include /* strcmp() */ +#include "be.h" + +/* Openers for each supported backend, defined on each be-X.c file */ +struct db_conn *qdbm_open(const char *name, int flags); +struct db_conn *bdb_open(const char *name, int flags); +struct db_conn *tc_open(const char *name, int flags); +struct db_conn *xtdb_open(const char *name, int flags); +struct db_conn *null_open(const char *name, int flags); + + +struct db_conn *db_open(enum backend_type type, const char *name, int flags) +{ + switch (type) { + case BE_QDBM: + return qdbm_open(name, flags); + case BE_BDB: + return bdb_open(name, flags); + case BE_TC: + return tc_open(name, flags); + case BE_TDB: + return xtdb_open(name, flags); + case BE_NULL: + return null_open(name, flags); + default: + return NULL; + } +} + +enum backend_type be_type_from_str(const char *name) +{ + if (strcmp(name, "qdbm") == 0) + return BE_ENABLE_QDBM ? BE_QDBM : BE_UNSUPPORTED; + if (strcmp(name, "bdb") == 0) + return BE_ENABLE_BDB ? BE_BDB : BE_UNSUPPORTED; + if (strcmp(name, "tc") == 0) + return BE_ENABLE_TC ? BE_TC : BE_UNSUPPORTED; + if (strcmp(name, "tdb") == 0) + return BE_ENABLE_TDB ? BE_TDB : BE_UNSUPPORTED; + if (strcmp(name, "null") == 0) + return BE_ENABLE_NULL ? BE_NULL : BE_UNSUPPORTED; + return BE_UNKNOWN; +} + + +const char *be_str_from_type(enum backend_type type) +{ + if (type == BE_QDBM) + return "qdbm"; + if (type == BE_BDB) + return "bdb"; + if (type == BE_TC) + return "tc"; + if (type == BE_TDB) + return "tdb"; + if (type == BE_NULL) + return "null"; + return "unknown"; +} + diff --git a/nmdb/be.h b/nmdb/be.h index d333b59..37a8210 100644 --- a/nmdb/be.h +++ b/nmdb/be.h @@ -2,41 +2,102 @@ #ifndef _BE_H #define _BE_H -/* Depending on our backend, we define db_t to be different things so the - * generic code doesn't have to care about which backend we're using. */ -#if defined BACKEND_QDBM - #include - typedef DEPOT db_t; - -#elif defined BACKEND_BDB - /* typedefs to work around db.h include bug */ - typedef unsigned int u_int; - typedef unsigned long u_long; - #include - typedef DB db_t; - -#elif defined BACKEND_TC - #include - typedef TCHDB db_t; - -#elif defined BACKEND_NULL - typedef int db_t; +#include /* size_t */ +struct db_conn { + /* This is where the backend sets a reference to the connection, which + * will be properly casted when needed */ + void *conn; + + /* Operations */ + int (*set)(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t vsize); + int (*get)(struct db_conn *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t *vsize); + int (*del)(struct db_conn *db, const unsigned char *key, size_t ksize); + int (*firstkey)(struct db_conn *db, unsigned char *key, size_t *ksize); + int (*nextkey)(struct db_conn *db, + const unsigned char *key, size_t ksize, + unsigned char *nextkey, size_t *nksize); + int (*close)(struct db_conn *db); +}; + +enum backend_type { + /* The first two are special, used to indicate unknown and unsupported + * backends */ + BE_UNKNOWN = -2, + BE_UNSUPPORTED = -1, + BE_QDBM = 1, + BE_BDB, + BE_TC, + BE_TDB, + BE_NULL, +}; + +/* Generic opener that knows about all the backends */ +struct db_conn *db_open(enum backend_type type, const char *name, int flags); + +/* Returns the backend type for the given name. */ +enum backend_type be_type_from_str(const char *name); + +/* Returns the backend name for the given type. */ +const char *be_str_from_type(enum backend_type type); + +/* String containing a list of all supported backends */ +#if BE_ENABLE_QDBM + #define _QDBM_SUPP "qdbm " #else - #error "Unknown backend" - /* Define it anyway, so this is the only warning/error the user sees. */ - typedef int db_t; + #define _QDBM_SUPP "" #endif +#if BE_ENABLE_BDB + #define _BDB_SUPP "bdb " +#else + #define _BDB_SUPP "" +#endif -db_t *db_open(const char *name, int flags); -int db_close(db_t *db); -int db_set(db_t *db, const unsigned char *key, size_t ksize, - unsigned char *val, size_t vsize); -int db_get(db_t *db, const unsigned char *key, size_t ksize, - unsigned char *val, size_t *vsize); -int db_del(db_t *db, const unsigned char *key, size_t ksize); +#if BE_ENABLE_TC + #define _TC_SUPP "tc " +#else + #define _TC_SUPP "" +#endif +#if BE_ENABLE_TDB + #define _TDB_SUPP "tdb " +#else + #define _TDB_SUPP "" #endif +#if BE_ENABLE_NULL + #define _NULL_SUPP "null " +#else + #define _NULL_SUPP "" +#endif + +#define SUPPORTED_BE _QDBM_SUPP _BDB_SUPP _TC_SUPP _TDB_SUPP _NULL_SUPP + + +/* Default backend */ +#if BE_ENABLE_TDB + #define DEFAULT_BE BE_TDB + #define DEFAULT_BE_NAME "tdb" +#elif BE_ENABLE_TC + #define DEFAULT_BE BE_TC + #define DEFAULT_BE_NAME "tc" +#elif BE_ENABLE_QDBM + #define DEFAULT_BE BE_QDBM + #define DEFAULT_BE_NAME "qdbm" +#elif BE_ENABLE_BDB + #define DEFAULT_BE BE_BDB + #define DEFAULT_BE_NAME "bdb" +#elif BE_ENABLE_NULL + #warning "using null backend as the default" + #define DEFAULT_BE BE_NULL + #define DEFAULT_BE_NAME "null" +#else + #error "no backend available" +#endif + + +#endif diff --git a/nmdb/cache.c b/nmdb/cache.c index 71f2f93..c94bd64 100644 --- a/nmdb/cache.c +++ b/nmdb/cache.c @@ -10,12 +10,13 @@ #include /* for malloc() */ #include /* for memcpy()/memcmp() */ #include /* snprintf() */ +#include "hash.h" /* hash() */ #include "cache.h" struct cache *cache_create(size_t numobjs, unsigned int flags) { - size_t i; + size_t i, j; struct cache *cd; struct cache_chain *c; @@ -28,9 +29,8 @@ struct cache *cache_create(size_t numobjs, unsigned int flags) /* We calculate the hash size so we have 4 objects per bucket; 4 being * an arbitrary number. It's long enough to make LRU useful, and small * enough to make lookups fast. */ - cd->chainlen = 4; cd->numobjs = numobjs; - cd->hashlen = numobjs / cd->chainlen; + cd->hashlen = numobjs / CHAINLEN; cd->table = (struct cache_chain *) malloc(sizeof(struct cache_chain) * cd->hashlen); @@ -44,12 +44,19 @@ struct cache *cache_create(size_t numobjs, unsigned int flags) c->len = 0; c->first = NULL; c->last = NULL; + for (j = 0; j < CHAINLEN; j++) { + memset(&(c->entries[j]), 0, + sizeof(struct cache_entry)); + + /* mark them as unused */ + c->entries[j].ksize = -1; + c->entries[j].vsize = -1; + } } return cd; } - int cache_free(struct cache *cd) { size_t i; @@ -66,7 +73,6 @@ int cache_free(struct cache *cd) n = e->next; free(e->key); free(e->val); - free(e); e = n; } } @@ -76,33 +82,24 @@ int cache_free(struct cache *cd) return 1; } - -/* - * The hash function used is the "One at a time" function, which seems simple, - * fast and popular. Others for future consideration if speed becomes an issue - * include: - * * FNV Hash (http://www.isthe.com/chongo/tech/comp/fnv/) - * * SuperFastHash (http://www.azillionmonkeys.com/qed/hash.html) - * * Judy dynamic arrays (http://judy.sf.net) - * - * A good comparison can be found at - * http://eternallyconfuzzled.com/tuts/hashing.html#existing - */ - -static uint32_t hash(const unsigned char *key, const size_t ksize) +static struct cache_entry *alloc_entry(struct cache_chain *c) { - uint32_t h = 0; - size_t i; + int i; - for (i = 0; i < ksize; i++) { - h += key[i]; - h += (h << 10); - h ^= (h >> 6); + for (i = 0; i < CHAINLEN; i++) { + if (c->entries[i].ksize == -1) + return &(c->entries[i]); } - h += (h << 3); - h ^= (h >> 11); - h += (h << 15); - return h; + + return NULL; +} + +static void free_entry(struct cache_entry *e) +{ + e->key = NULL; + e->ksize = -1; + e->val = NULL; + e->vsize = -1; } @@ -164,11 +161,103 @@ int cache_get(struct cache *cd, const unsigned char *key, size_t ksize, return 1; } +/* Creates a new cache entry, with the given key and value */ +static struct cache_entry *new_entry(struct cache_chain *c, + const unsigned char *key, size_t ksize, + const unsigned char *val, size_t vsize) +{ + struct cache_entry *new; + + new = alloc_entry(c); + if (new == NULL) { + return NULL; + } + + new->ksize = ksize; + new->vsize = vsize; + + new->key = malloc(ksize); + if (new->key == NULL) { + free(new); + return NULL; + } + memcpy(new->key, key, ksize); + + new->val = malloc(vsize); + if (new->val == NULL) { + free(new->key); + free_entry(new); + return NULL; + } + memcpy(new->val, val, vsize); + new->prev = NULL; + new->next = NULL; + + return new; +} + +static int insert_in_full_chain(struct cache_chain *c, + const unsigned char *key, size_t ksize, + const unsigned char *val, size_t vsize) +{ + /* To insert in a full chain, we evict the last entry and place the + * new one at the beginning. + * + * As an optimization, we avoid re-allocating the entry by reusing the + * last one, and when possible we reuse its key and value as well. */ + struct cache_entry *e = c->last; + + if (ksize == e->ksize) { + memcpy(e->key, key, ksize); + } else { + free(e->key); + e->key = malloc(ksize); + if (e->key == NULL) { + goto error; + } + e->ksize = ksize; + memcpy(e->key, key, ksize); + } + + if (vsize == e->vsize) { + memcpy(e->val, val, vsize); + } else { + free(e->val); + e->val = malloc(vsize); + if (e->val == NULL) { + goto error; + } + e->vsize = vsize; + memcpy(e->val, val, vsize); + } + + /* move the entry from the last to the first position */ + c->last = e->prev; + c->last->next = NULL; + + e->prev = NULL; + e->next = c->first; + + c->first->prev = e; + c->first = e; + + return 0; + +error: + /* on errors, remove the entry just in case */ + c->last = e->prev; + c->last->next = NULL; + free(e->key); + free(e->val); + free_entry(e); + + return -1; +} + int cache_set(struct cache *cd, const unsigned char *key, size_t ksize, const unsigned char *val, size_t vsize) { - int rv = 1; uint32_t h = 0; struct cache_chain *c; struct cache_entry *e, *new; @@ -180,71 +269,44 @@ int cache_set(struct cache *cd, const unsigned char *key, size_t ksize, e = find_in_chain(c, key, ksize); if (e == NULL) { - /* not found, create a new cache entry */ - new = malloc(sizeof(struct cache_entry)); - if (new == NULL) { - rv = 0; - goto exit; - } - - new->ksize = ksize; - new->vsize = vsize; - - new->key = malloc(ksize); - if (new->key == NULL) { - free(new); - rv = 0; - goto exit; - } - memcpy(new->key, key, ksize); - - new->val = malloc(vsize); - if (new->val == NULL) { - free(new->key); - free(new); - rv = 0; - goto exit; - } - memcpy(new->val, val, vsize); - new->prev = NULL; - new->next = NULL; - - /* and put it in */ - if (c->len == 0) { - /* line is empty, just put it there */ - c->first = new; - c->last = new; - c->len = 1; - } else if (c->len <= cd->chainlen) { - /* slots are still available, put the entry first */ - new->next = c->first; - c->first->prev = new; - c->first = new; - c->len += 1; + if (c->len == CHAINLEN) { + /* chain is full */ + if (insert_in_full_chain(c, key, ksize, + val, vsize) != 0) + return -1; } else { - /* chain is full, we need to evict the last one */ - e = c->last; - c->last = e->prev; - c->last->next = NULL; - free(e->key); - free(e->val); - free(e); - - new->next = c->first; - c->first->prev = new; - c->first = new; + new = new_entry(c, key, ksize, val, vsize); + if (new == NULL) + return -1; + + if (c->len == 0) { + /* line is empty, just put it there */ + c->first = new; + c->last = new; + c->len = 1; + } else if (c->len < CHAINLEN) { + /* slots are still available, put the entry + * first */ + new->next = c->first; + c->first->prev = new; + c->first = new; + c->len += 1; + } } } else { /* we've got a match, just replace the value in place */ - v = malloc(vsize); - if (v == NULL) { - rv = 0; - goto exit; + if (vsize == e->vsize) { + memcpy(e->val, val, vsize); + } else { + v = malloc(vsize); + if (v == NULL) + return -1; + + free(e->val); + e->val = v; + e->vsize = vsize; + memcpy(e->val, val, vsize); } - free(e->val); - e->val = v; - e->vsize = vsize; - memcpy(e->val, val, vsize); /* promote the entry to the top of the list if necessary */ if (c->first != e) { @@ -261,8 +323,7 @@ int cache_set(struct cache *cd, const unsigned char *key, size_t ksize, } } -exit: - return rv; + return 0; } @@ -300,7 +361,7 @@ int cache_del(struct cache *cd, const unsigned char *key, size_t ksize) free(e->key); free(e->val); - free(e); + free_entry(e); c->len -= 1; @@ -310,46 +371,42 @@ exit: /* Performs a cache compare-and-swap. - * Returns -2 if there was an error, -1 if the key is not in the cache, 0 if - * the old value does not match, and 1 if the CAS was successful. */ + * Returns -3 if there was an error, -2 if the key is not in the cache, -1 if + * the old value does not match, and 0 if the CAS was successful. */ int cache_cas(struct cache *cd, const unsigned char *key, size_t ksize, const unsigned char *oldval, size_t ovsize, const unsigned char *newval, size_t nvsize) { - int rv = 1; struct cache_entry *e; unsigned char *buf; e = find_in_cache(cd, key, ksize); - if (e == NULL) { - rv = -1; - goto exit; - } + if (e == NULL) + return -2; - if (e->vsize != ovsize) { - rv = 0; - goto exit; - } + if (e->vsize != ovsize) + return -1; - if (memcmp(e->val, oldval, ovsize) != 0) { - rv = 0; - goto exit; - } + if (memcmp(e->val, oldval, ovsize) != 0) + return -1; - buf = malloc(nvsize); - if (buf == NULL) { - rv = -2; - goto exit; - } + if (ovsize == nvsize) { + /* since they have the same size, avoid the malloc() and just + * copy the new value */ + memcpy(e->val, newval, nvsize); + } else { + buf = malloc(nvsize); + if (buf == NULL) + return -3; - memcpy(buf, newval, nvsize); - free(e->val); - e->val = buf; - e->vsize = nvsize; + memcpy(buf, newval, nvsize); + free(e->val); + e->val = buf; + e->vsize = nvsize; + } -exit: - return rv; + return 0; } @@ -357,7 +414,7 @@ exit: * The increment is a signed 64 bit value, and the value size must be >= 8 * bytes. * Returns: - * 1 if the increment succeeded. + * 0 if the increment succeeded. * -1 if the value was not in the cache. * -2 if the value was not null terminated. * -3 if there was a memory error. @@ -381,9 +438,10 @@ int cache_incr(struct cache *cd, const unsigned char *key, size_t ksize, val = e->val; vsize = e->vsize; - /* the value must be a NULL terminated string, otherwise strtoll might - * cause a segmentation fault */ - if (val && val[vsize - 1] != '\0') + /* The value must be a 0-terminated string, otherwise strtoll might + * cause a segmentation fault. Note that val should never be NULL, but + * it doesn't hurt to check just in case */ + if (val == NULL || val[vsize - 1] != '\0') return -2; intval = strtoll((char *) val, NULL, 10); @@ -404,7 +462,7 @@ int cache_incr(struct cache *cd, const unsigned char *key, size_t ksize, snprintf((char *) val, vsize, "%23lld", (long long int) intval); *newval = intval; - return 1; + return 0; } diff --git a/nmdb/cache.h b/nmdb/cache.h index 039dd91..9d70c00 100644 --- a/nmdb/cache.h +++ b/nmdb/cache.h @@ -8,6 +8,8 @@ #include /* for int64_t */ +#define CHAINLEN 4 + struct cache { /* set directly by initialization */ size_t numobjs; @@ -15,18 +17,11 @@ struct cache { /* calculated */ size_t hashlen; - size_t chainlen; /* the cache data itself */ struct cache_chain *table; }; -struct cache_chain { - size_t len; - struct cache_entry *first; - struct cache_entry *last; -}; - struct cache_entry { unsigned char *key; unsigned char *val; @@ -37,6 +32,18 @@ struct cache_entry { struct cache_entry *next; }; +struct cache_chain { + size_t len; + + /* the entries live inside the chain, to avoid allocation costs and + * make it more cache-friendly */ + struct cache_entry entries[CHAINLEN]; + + /* these point to elements of the entries array */ + struct cache_entry *first; + struct cache_entry *last; +}; + struct cache *cache_create(size_t numobjs, unsigned int flags); int cache_free(struct cache *cd); diff --git a/nmdb/common.h b/nmdb/common.h index a1b129e..06f0789 100644 --- a/nmdb/common.h +++ b/nmdb/common.h @@ -13,6 +13,7 @@ extern struct cache *cache_table; extern struct queue *op_queue; /* Settings */ +#include "be.h" struct settings { int tipc_lower; int tipc_upper; @@ -25,8 +26,11 @@ struct settings { int numobjs; int foreground; int passive; + int read_only; char *dbname; char *logfname; + enum backend_type backend; + char *pidfile; }; extern struct settings settings; diff --git a/nmdb/dbloop.c b/nmdb/dbloop.c index bb20504..7a4f4fd 100644 --- a/nmdb/dbloop.c +++ b/nmdb/dbloop.c @@ -18,7 +18,7 @@ static void *db_loop(void *arg); -static void process_op(db_t *db, struct queue_entry *e); +static void process_op(struct db_conn *db, struct queue_entry *e); /* Used to signal the loop that it should exit when the queue becomes empty. @@ -26,7 +26,7 @@ static void process_op(db_t *db, struct queue_entry *e); static int loop_should_stop = 0; -pthread_t *db_loop_start(db_t *db) +pthread_t *db_loop_start(struct db_conn *db) { pthread_t *thread; @@ -53,9 +53,9 @@ static void *db_loop(void *arg) int rv; struct timespec ts; struct queue_entry *e; - db_t *db; + struct db_conn *db; - db = (db_t *) arg; + db = (struct db_conn *) arg; for (;;) { /* Condition waits are specified with absolute timeouts, see @@ -102,11 +102,11 @@ static void *db_loop(void *arg) return NULL; } -static void process_op(db_t *db, struct queue_entry *e) +static void process_op(struct db_conn *db, struct queue_entry *e) { int rv; if (e->operation == REQ_SET) { - rv = db_set(db, e->key, e->ksize, e->val, e->vsize); + rv = db->set(db, e->key, e->ksize, e->val, e->vsize); if (!(e->req->flags & FLAGS_SYNC)) return; @@ -125,7 +125,7 @@ static void process_op(db_t *db, struct queue_entry *e) e->req->reply_err(e->req, ERR_MEM); return; } - rv = db_get(db, e->key, e->ksize, val, &vsize); + rv = db->get(db, e->key, e->ksize, val, &vsize); if (rv == 0) { e->req->reply_mini(e->req, REP_NOTIN); free(val); @@ -135,7 +135,7 @@ static void process_op(db_t *db, struct queue_entry *e) free(val); } else if (e->operation == REQ_DEL) { - rv = db_del(db, e->key, e->ksize); + rv = db->del(db, e->key, e->ksize); if (!(e->req->flags & FLAGS_SYNC)) return; @@ -155,7 +155,7 @@ static void process_op(db_t *db, struct queue_entry *e) e->req->reply_err(e->req, ERR_MEM); return; } - rv = db_get(db, e->key, e->ksize, dbval, &dbvsize); + rv = db->get(db, e->key, e->ksize, dbval, &dbvsize); if (rv == 0) { e->req->reply_mini(e->req, REP_NOTIN); free(dbval); @@ -165,7 +165,8 @@ static void process_op(db_t *db, struct queue_entry *e) if (e->vsize == dbvsize && memcmp(e->val, dbval, dbvsize) == 0) { /* Swap */ - rv = db_set(db, e->key, e->ksize, e->newval, e->nvsize); + rv = db->set(db, e->key, e->ksize, + e->newval, e->nvsize); if (!rv) { e->req->reply_err(e->req, ERR_DB); return; @@ -189,7 +190,7 @@ static void process_op(db_t *db, struct queue_entry *e) e->req->reply_err(e->req, ERR_MEM); return; } - rv = db_get(db, e->key, e->ksize, dbval, &dbvsize); + rv = db->get(db, e->key, e->ksize, dbval, &dbvsize); if (rv == 0) { e->req->reply_mini(e->req, REP_NOTIN); free(dbval); @@ -215,7 +216,7 @@ static void process_op(db_t *db, struct queue_entry *e) snprintf((char *) dbval, dbvsize, "%23lld", (long long int) intval); - rv = db_set(db, e->key, e->ksize, dbval, dbvsize); + rv = db->set(db, e->key, e->ksize, dbval, dbvsize); if (!rv) { e->req->reply_err(e->req, ERR_DB); return; @@ -227,6 +228,51 @@ static void process_op(db_t *db, struct queue_entry *e) free(dbval); + } else if (e->operation == REQ_FIRSTKEY) { + unsigned char *key; + size_t ksize = 64 * 1024; + + if (db->firstkey == NULL) { + e->req->reply_err(e->req, ERR_DB); + return; + } + + key = malloc(ksize); + if (key == NULL) { + e->req->reply_err(e->req, ERR_MEM); + return; + } + rv = db->firstkey(db, key, &ksize); + if (rv == 0) { + e->req->reply_mini(e->req, REP_NOTIN); + free(key); + return; + } + e->req->reply_long(e->req, REP_OK, key, ksize); + free(key); + + } else if (e->operation == REQ_NEXTKEY) { + unsigned char *newkey; + size_t nksize = 64 * 1024; + + if (db->nextkey == NULL) { + e->req->reply_err(e->req, ERR_DB); + return; + } + + newkey = malloc(nksize); + if (newkey == NULL) { + e->req->reply_err(e->req, ERR_MEM); + return; + } + rv = db->nextkey(db, e->key, e->ksize, newkey, &nksize); + if (rv == 0) { + e->req->reply_mini(e->req, REP_NOTIN); + free(newkey); + return; + } + e->req->reply_long(e->req, REP_OK, newkey, nksize); + free(newkey); } else { wlog("Unknown op 0x%x\n", e->operation); } diff --git a/nmdb/dbloop.h b/nmdb/dbloop.h index a4a9d63..ddb3664 100644 --- a/nmdb/dbloop.h +++ b/nmdb/dbloop.h @@ -3,9 +3,9 @@ #define _DBLOOP_H #include /* for pthread_t */ -#include "be.h" /* for db_t */ +#include "be.h" /* for struct db_conn */ -pthread_t *db_loop_start(db_t *db); +pthread_t *db_loop_start(struct db_conn *db); void db_loop_stop(pthread_t *thread); #endif diff --git a/nmdb/hash.h b/nmdb/hash.h new file mode 100644 index 0000000..706c5a6 --- /dev/null +++ b/nmdb/hash.h @@ -0,0 +1,164 @@ + +#ifndef _HASH_H +#define _HASH_H + +/* + * Hash function used in the cache. + * + * This is kept here instead of a .c file so the compiler can inline if it + * decides it's worth it. + * + * The hash function used is Austin Appleby's MurmurHash2. It used to be + * Jenkins's one-at-a-time, but this one is much faster. However, we keep + * the others that were tested for future comparison purposes. + */ +#define hash(k, s) murmurhash2(k, s) + +/* MurmurHash2, by Austin Appleby. The one we use. + * It has been modify to fit into the coding style, to work on uint32_t + * instead of ints, and the seed was fixed to a random number because it's not + * an issue for us. The author placed it in the public domain, so it's safe. + * http://sites.google.com/site/murmurhash/ */ +static uint32_t murmurhash2(const unsigned char *key, size_t len) +{ + const uint32_t m = 0x5bd1e995; + const int r = 24; + const uint32_t seed = 0x34a4b627; + + // Initialize the hash to a 'random' value + uint32_t h = seed ^ len; + + // Mix 4 bytes at a time into the hash + while (len >= 4) { + uint32_t k = *(uint32_t *) key; + + k *= m; + k ^= k >> r; + k *= m; + + h *= m; + h ^= k; + + key += 4; + len -= 4; + } + + // Handle the last few bytes of the input array + switch (len) { + case 3: h ^= key[2] << 16; + case 2: h ^= key[1] << 8; + case 1: h ^= key[0]; + h *= m; + } + + // Do a few final mixes of the hash to ensure the last few + // bytes are well-incorporated. + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; +} + + +/* Unused functions, left for comparison purposes */ +#if 0 + +/* Paul Hsieh's SuperFastHash, which is really fast, but a tad slower than + * MurmurHash2. + * http://www.azillionmonkeys.com/qed/hash.html */ +static uint32_t superfast(const unsigned char *data, size_t len) +{ + uint32_t h = len, tmp; + int rem; + + /* this is the same as (*((const uint16_t *) (d))) on little endians, + * but we keep the generic version since gcc compiles decent code for + * both and there's no noticeable difference in performance */ + #define get16bits(d) ( \ + ( ( (uint32_t) ( ((const uint8_t *)(d))[1] ) ) << 8 ) \ + + (uint32_t) ( ((const uint8_t *)(d))[0] ) ) + + if (len <= 0 || data == NULL) + return 0; + + rem = len & 3; + len >>= 2; + + /* Main loop */ + for (; len > 0; len--) { + h += get16bits(data); + tmp = (get16bits(data+2) << 11) ^ h; + h = (h << 16) ^ tmp; + data += 2*sizeof (uint16_t); + h += h >> 11; + } + + /* Handle end cases */ + switch (rem) { + case 3: h += get16bits(data); + h ^= h << 16; + h ^= data[sizeof(uint16_t)] << 18; + h += h >> 11; + break; + case 2: h += get16bits(data); + h ^= h << 11; + h += h >> 17; + break; + case 1: h += *data; + h ^= h << 10; + h += h >> 1; + } + + /* Force "avalanching" of final 127 bits */ + h ^= h << 3; + h += h >> 5; + h ^= h << 4; + h += h >> 17; + h ^= h << 25; + h += h >> 6; + + return h; + + #undef get16bits +} + +/* Jenkins' one-at-a-time hash, the one we used to use. + * http://en.wikipedia.org/wiki/Jenkins_hash_function */ +static uint32_t oneatatime(const unsigned char *key, const size_t ksize) +{ + uint32_t h = 0; + size_t i; + + for (i = 0; i < ksize; i++) { + h += key[i]; + h += (h << 10); + h ^= (h >> 6); + } + h += (h << 3); + h ^= (h >> 11); + h += (h << 15); + return h; +} + +/* FNV 32 bit hash; slower than the rest. + * http://www.isthe.com/chongo/tech/comp/fnv/ */ +static uint32_t fnv_hash(const unsigned char *key, const size_t ksize) +{ + const unsigned char *end = key + ksize; + uint32_t hval; + + hval = 0x811c9dc5; + while (key < end) { + hval += (hval<<1) + (hval<<4) + (hval<<7) + + (hval<<8) + (hval<<24); + hval ^= (uint32_t) *key++; + } + + return hval; +} + +#endif + +#endif + diff --git a/nmdb/log.c b/nmdb/log.c index 0a485b9..3026350 100644 --- a/nmdb/log.c +++ b/nmdb/log.c @@ -72,3 +72,21 @@ void errlog(const char *s) wlog("%s: %s\n", s, strerror(errno)); } +void write_pid() +{ + FILE *f; + + if (settings.pidfile == NULL) + return; + + f = fopen(settings.pidfile, "w"); + if (f == NULL) { + errlog("Can't open pidfile for writing"); + return; + } + + fprintf(f, "%d\n", getpid()); + + fclose(f); +} + diff --git a/nmdb/log.h b/nmdb/log.h index 3e5672b..2a52e5f 100644 --- a/nmdb/log.h +++ b/nmdb/log.h @@ -14,6 +14,9 @@ void wlog(const char *fmt, ...); /* Errno logging, perror()-alike */ void errlog(const char *s); +/* PID file */ +void write_pid(); + #endif diff --git a/nmdb/main.c b/nmdb/main.c index 79d92b4..02df38a 100644 --- a/nmdb/main.c +++ b/nmdb/main.c @@ -13,6 +13,7 @@ #include "net-const.h" #include "log.h" #include "stats.h" +#include "be.h" #define DEFDBNAME "database" @@ -28,6 +29,7 @@ static void help(void) { char h[] = \ "nmdb [options]\n" "\n" + " -b backend backend to use (" DEFAULT_BE_NAME ")\n" " -d dbpath database path ('database' by default)\n" " -l lower TIPC lower port number (10)\n" " -L upper TIPC upper port number (= lower)\n" @@ -39,10 +41,14 @@ static void help(void) { " -S addr SCTP listening address (all local addresses)\n" " -c nobj max. number of objects to be cached, in thousands (128)\n" " -o fname log to the given file (stdout).\n" + " -i pidfile file to write the PID to (none).\n" " -f don't fork and stay in the foreground\n" " -p enable passive mode, for redundancy purposes (read docs.)\n" + " -r read-only mode\n" " -h show this help\n" "\n" + "Available backends: " SUPPORTED_BE "\n" + "\n" "Please report bugs to Alberto Bertogli (albertito@blitiri.com.ar)\n" "\n"; printf("%s", h); @@ -64,17 +70,23 @@ static int load_settings(int argc, char **argv) settings.numobjs = -1; settings.foreground = 0; settings.passive = 0; - settings.logfname = "-"; + settings.read_only = 0; + settings.logfname = NULL; + settings.pidfile = NULL; + settings.backend = DEFAULT_BE; - settings.dbname = malloc(strlen(DEFDBNAME) + 1); - strcpy(settings.dbname, DEFDBNAME); + settings.dbname = strdup(DEFDBNAME); + settings.logfname = strdup("-"); - while ((c = getopt(argc, argv, "d:l:L:t:T:u:U:s:S:c:o:fph?")) != -1) { + while ((c = getopt(argc, argv, + "b:d:l:L:t:T:u:U:s:S:c:o:i:fprh?")) != -1) { switch(c) { + case 'b': + settings.backend = be_type_from_str(optarg); + break; case 'd': free(settings.dbname); - settings.dbname = malloc(strlen(optarg) + 1); - strcpy(settings.dbname, optarg); + settings.dbname = strdup(optarg); break; case 'l': @@ -110,8 +122,13 @@ static int load_settings(int argc, char **argv) break; case 'o': - settings.logfname = malloc(strlen(optarg) + 1); - strcpy(settings.logfname, optarg); + free(settings.logfname); + settings.logfname = strdup(optarg); + break; + + case 'i': + free(settings.pidfile); + settings.pidfile = strdup(optarg); break; case 'f': @@ -120,6 +137,9 @@ static int load_settings(int argc, char **argv) case 'p': settings.passive = 1; break; + case 'r': + settings.read_only = 1; + break; case 'h': case '?': @@ -150,15 +170,31 @@ static int load_settings(int argc, char **argv) if (settings.numobjs == -1) settings.numobjs = 128 * 1024; + if (settings.backend == BE_UNKNOWN) { + printf("Error: unknown backend\n"); + return 0; + } else if (settings.backend == BE_UNSUPPORTED) { + printf("Error: unsupported backend\n"); + return 0; + } + return 1; } +static void free_settings() +{ + free(settings.dbname); + free(settings.logfname); + free(settings.pidfile); +} + + int main(int argc, char **argv) { struct cache *cd; struct queue *q; - db_t *db; + struct db_conn *db; pid_t pid; pthread_t *dbthread; @@ -186,11 +222,13 @@ int main(int argc, char **argv) } op_queue = q; - db = db_open(settings.dbname, 0); + db = db_open(settings.backend, settings.dbname, 0); if (db == NULL) { errlog("Error opening DB"); return 1; } + wlog("Opened database \"%s\" with %s backend\n", settings.dbname, + be_str_from_type(settings.backend)); if (!settings.foreground) { pid = fork(); @@ -208,18 +246,24 @@ int main(int argc, char **argv) wlog("Starting nmdb\n"); + write_pid(); + dbthread = db_loop_start(db); net_loop(); db_loop_stop(dbthread); - db_close(db); + db->close(db); queue_free(q); cache_free(cd); + unlink(settings.pidfile); + + free_settings(); + return 0; } diff --git a/nmdb/net-const.h b/nmdb/net-const.h index 290ab90..3fb80bc 100644 --- a/nmdb/net-const.h +++ b/nmdb/net-const.h @@ -33,6 +33,8 @@ #define REQ_CAS 0x104 #define REQ_INCR 0x105 #define REQ_STATS 0x106 +#define REQ_FIRSTKEY 0x107 +#define REQ_NEXTKEY 0x108 /* Possible request flags (which can be applied to the documented requests) */ #define FLAGS_CACHE_ONLY 1 /* get, set, del, cas, incr */ @@ -53,6 +55,7 @@ #define ERR_UNKREQ 0x104 /* Unknown request */ #define ERR_MEM 0x105 /* Memory allocation error */ #define ERR_DB 0x106 /* Database error */ +#define ERR_RO 0x107 /* Server in read-only mode */ #endif diff --git a/nmdb/net.c b/nmdb/net.c index e1cc3b9..2020e62 100644 --- a/nmdb/net.c +++ b/nmdb/net.c @@ -35,6 +35,16 @@ static void logfd_reopen_sighandler(int fd, short event, void *arg) wlog("Log reopened\n"); } +static void enable_read_only_sighandler(int fd, short event, void *arg) +{ + if (!settings.read_only) { + wlog("Changing to read-only mode\n"); + settings.read_only = 1; + } else { + wlog("Got signal, but already in read-only mode\n"); + } +} + void net_loop(void) { int tipc_fd = -1; @@ -42,7 +52,8 @@ void net_loop(void) int udp_fd = -1; int sctp_fd = -1; struct event tipc_evt, tcp_evt, udp_evt, sctp_evt, - sigterm_evt, sigint_evt, sigusr1_evt, sigusr2_evt; + sigterm_evt, sigint_evt, + sighup_evt, sigusr1_evt, sigusr2_evt; event_init(); @@ -101,7 +112,10 @@ void net_loop(void) signal_add(&sigterm_evt, NULL); signal_set(&sigint_evt, SIGINT, exit_sighandler, &sigint_evt); signal_add(&sigint_evt, NULL); - signal_set(&sigusr1_evt, SIGUSR1, logfd_reopen_sighandler, + signal_set(&sighup_evt, SIGHUP, logfd_reopen_sighandler, + &sighup_evt); + signal_add(&sighup_evt, NULL); + signal_set(&sigusr1_evt, SIGUSR1, enable_read_only_sighandler, &sigusr1_evt); signal_add(&sigusr1_evt, NULL); signal_set(&sigusr2_evt, SIGUSR2, passive_to_active_sighandler, diff --git a/nmdb/nmdb.1 b/nmdb/nmdb.1 index b7fffb2..e6d08a0 100644 --- a/nmdb/nmdb.1 +++ b/nmdb/nmdb.1 @@ -1,8 +1,9 @@ .TH nmdb 1 "11/Sep/2006" .SH NAME nmdb - A multiprotocol network database manager -.SH SYNOPSYS -nmdb [-d dbpath] [-l lower] [-L upper] +.SH SYNOPSIS +nmdb [-b backend] [-d dbpath] + [-l lower] [-L upper] [-t tcpport] [-T tcpaddr] [-u udpport] [-U udpaddr] [-s sctpport] [-S sctpaddr] @@ -27,6 +28,11 @@ For additional documentation, go to the project's website at .SH OPTIONS .TP +.B "-b backend" +Which database library to use as backend. It can be one of tdb, tc, qdbm, bdb, +or null, although not all of them may be available, depending on build-time +options. On doubt, use the default. +.TP .B "-d dbpath" Indicate the path to the database file to use. It will be created if it doesn't exist. If a name is not provided, "database" will be used. @@ -80,14 +86,18 @@ least you have a recent up-to-date database. Be aware that it's not widely tested (although it should work fine), and do .I not start an active and a passive server in the same machine (it misbehaves under -some circumnstances, and doesn't make much sense anyway). +some circumstances, and doesn't make much sense anyway). +.TP +.B "-r" +Read-only mode. The server will refuse any request that would alter the +database and/or cache. Useful for redundancy and replication. .TP .B "-h" Show a brief help. .SH INVOCATION EXAMPLE -To run the server with the database at /var/lib/nmpc-db: -.B "nmpc -d /var/lib/nmpc-db" +To run the server with the database at /var/lib/nmdb-db: +.B "nmdb -d /var/lib/nmdb-db" Be .I very @@ -99,10 +109,7 @@ clients. This behaviour is different from the normal IP networking, where you can't bind a port twice. .SH SEE ALSO -.BR libnmdb (3), -.B TIPC -(http://tipc.sf.net), -.BR qdbm (3). +.BR libnmdb (3). .SH AUTHORS Created by Alberto Bertogli (albertito@blitiri.com.ar). diff --git a/nmdb/parse.c b/nmdb/parse.c index 1df7e1d..fb667fe 100644 --- a/nmdb/parse.c +++ b/nmdb/parse.c @@ -17,6 +17,8 @@ static void parse_set(struct req_info *req); static void parse_del(struct req_info *req); static void parse_cas(struct req_info *req); static void parse_incr(struct req_info *req); +static void parse_firstkey(struct req_info *req); +static void parse_nextkey(struct req_info *req); static void parse_stats(struct req_info *req); @@ -36,39 +38,26 @@ static struct queue_entry *make_queue_long_entry(const struct req_info *req, return NULL; } - kcopy = NULL; + kcopy = vcopy = nvcopy = NULL; + if (key != NULL) { kcopy = malloc(ksize); - if (kcopy == NULL) { - queue_entry_free(e); - return NULL; - } + if (kcopy == NULL) + goto error; memcpy(kcopy, key, ksize); } - vcopy = NULL; if (val != NULL) { vcopy = malloc(vsize); - if (vcopy == NULL) { - queue_entry_free(e); - if (kcopy != NULL) - free(kcopy); - return NULL; - } + if (vcopy == NULL) + goto error; memcpy(vcopy, val, vsize); } - nvcopy = NULL; if (newval != NULL) { nvcopy = malloc(nvsize); - if (nvcopy == NULL) { - queue_entry_free(e); - if (kcopy != NULL) - free(kcopy); - if (vcopy != NULL) - free(vcopy); - return NULL; - } + if (nvcopy == NULL) + goto error; memcpy(nvcopy, newval, nvsize); } @@ -100,6 +89,13 @@ static struct queue_entry *make_queue_long_entry(const struct req_info *req, e->req->psize = 0; return e; + +error: + queue_entry_free(e); + free(kcopy); + free(vcopy); + free(nvcopy); + return NULL; } @@ -207,6 +203,10 @@ int parse_message(struct req_info *req, parse_cas(req); } else if (cmd == REQ_INCR) { parse_incr(req); + } else if (cmd == REQ_FIRSTKEY) { + parse_firstkey(req); + } else if (cmd == REQ_NEXTKEY) { + parse_nextkey(req); } else if (cmd == REQ_STATS) { parse_stats(req); } else { @@ -303,6 +303,11 @@ static void parse_set(struct req_info *req) return; } + if (settings.read_only) { + req->reply_err(req, ERR_RO); + return; + } + FILL_CACHE_FLAG(set); FILL_SYNC_FLAG(); @@ -310,7 +315,7 @@ static void parse_set(struct req_info *req) val = key + ksize; rv = cache_set(cache_table, key, ksize, val, vsize); - if (!rv) { + if (rv != 0) { req->reply_err(req, ERR_MEM); return; } @@ -349,6 +354,11 @@ static void parse_del(struct req_info *req) return; } + if (settings.read_only) { + req->reply_err(req, ERR_RO); + return; + } + FILL_CACHE_FLAG(del); FILL_SYNC_FLAG(); @@ -414,6 +424,11 @@ static void parse_cas(struct req_info *req) return; } + if (settings.read_only) { + req->reply_err(req, ERR_RO); + return; + } + FILL_CACHE_FLAG(cas); key = req->payload + sizeof(uint32_t) * 3; @@ -422,19 +437,24 @@ static void parse_cas(struct req_info *req) rv = cache_cas(cache_table, key, ksize, oldval, ovsize, newval, nvsize); - if (rv == 0) { + if (rv == -1) { /* If the cache doesn't match, there is no need to bother the * DB even if we were asked to impact. */ req->reply_mini(req, REP_NOMATCH); return; + } else if (rv == -3) { + /* If there was an error, don't bother either */ + req->reply_err(req, ERR_MEM); + return; } if (cache_only) { - if (rv == -1) { - req->reply_mini(req, REP_NOTIN); + if (rv == 0) { + req->reply_mini(req, REP_OK); return; } else { - req->reply_mini(req, REP_OK); + /* rv == -2, key not in the cache */ + req->reply_mini(req, REP_NOTIN); return; } } else { @@ -477,6 +497,11 @@ static void parse_incr(struct req_info *req) return; } + if (settings.read_only) { + req->reply_err(req, ERR_RO); + return; + } + FILL_CACHE_FLAG(incr); key = req->payload + sizeof(uint32_t); @@ -517,6 +542,45 @@ static void parse_incr(struct req_info *req) } +static void parse_firstkey(struct req_info *req) +{ + int rv; + + stats.db_firstkey++; + + rv = put_in_queue(req, REQ_FIRSTKEY, 1, NULL, 0, NULL, 0); + if (!rv) { + req->reply_err(req, ERR_MEM); + return; + } +} + +static void parse_nextkey(struct req_info *req) +{ + int rv; + const unsigned char *key; + uint32_t ksize; + + ksize = * (uint32_t *) req->payload; + ksize = ntohl(ksize); + if (req->psize < ksize) { + stats.net_broken_req++; + req->reply_err(req, ERR_BROKEN); + return; + } + + stats.db_nextkey++; + + key = req->payload + sizeof(uint32_t); + + rv = put_in_queue(req, REQ_NEXTKEY, 1, key, ksize, NULL, 0); + if (!rv) { + req->reply_err(req, ERR_MEM); + return; + } +} + + static void parse_stats(struct req_info *req) { int i; @@ -561,6 +625,9 @@ static void parse_stats(struct req_info *req) fcpy(net_broken_req); fcpy(net_unk_req); + fcpy(db_firstkey); + fcpy(db_nextkey); + req->reply_long(req, REP_OK, (unsigned char *) response, sizeof(response)); diff --git a/nmdb/sparse.h b/nmdb/sparse.h index 20adafc..c48b163 100644 --- a/nmdb/sparse.h +++ b/nmdb/sparse.h @@ -5,11 +5,14 @@ #define _SPARSE_H #ifdef __CHECKER__ -# define __acquires(x) __attribute__((exact_context(x,0,1))) -# define __releases(x) __attribute__((exact_context(x,1,0))) -# define __with_lock_acquired(x) __attribute__((exact_context(x,1,1))) -# define __acquire(x) __context__(x,1,0) -# define __release(x) __context__(x,-1,1) +# define __acquires(x) __attribute__((context(x,0,1))) +# define __releases(x) __attribute__((context(x,1,0))) +# define __acquire(x) __context__(x,1) +# define __release(x) __context__(x,-1) +/* __with_lock_acquired() is at the moment just documentation, but we keep it + * in case sparse gets the ability to match exact contexts again */ +//# define __with_lock_acquired(x) __attribute__((exact_context(x,1,1))) +# define __with_lock_acquired(x) #else # define __acquires(x) # define __releases(x) diff --git a/nmdb/stats.c b/nmdb/stats.c index 8cd85ee..fed845b 100644 --- a/nmdb/stats.c +++ b/nmdb/stats.c @@ -26,6 +26,9 @@ void stats_init(struct stats *s) s->net_version_mismatch = 0; s->net_broken_req = 0; s->net_unk_req = 0; + + s->db_firstkey = 0; + s->db_nextkey = 0; } diff --git a/nmdb/stats.h b/nmdb/stats.h index 8d85ff6..92d53d4 100644 --- a/nmdb/stats.h +++ b/nmdb/stats.h @@ -31,9 +31,11 @@ struct stats { unsigned long net_version_mismatch; unsigned long net_broken_req; /* 20 */ unsigned long net_unk_req; + unsigned long db_firstkey; + unsigned long db_nextkey; }; -#define STATS_REPLY_SIZE 21 +#define STATS_REPLY_SIZE 23 void stats_init(struct stats *s); diff --git a/nmdb/tcp.c b/nmdb/tcp.c index b435756..9591145 100644 --- a/nmdb/tcp.c +++ b/nmdb/tcp.c @@ -77,6 +77,7 @@ static void init_req(struct tcp_socket *tcpsock) static void rep_send_error(const struct req_info *req, const unsigned int code) { uint32_t l, r, c; + ssize_t rv; unsigned char minibuf[4 * 4]; if (settings.passive) @@ -92,9 +93,9 @@ static void rep_send_error(const struct req_info *req, const unsigned int code) memcpy(minibuf + 12, &c, 4); /* If this send fails, there's nothing to be done */ - r = send(req->fd, minibuf, 4 * 4, 0); + rv = send(req->fd, minibuf, 4 * 4, 0); - if (r < 0) { + if (rv < 0) { errlog("rep_send_error() failed"); } } diff --git a/tests/coverage/README b/tests/coverage/README new file mode 100644 index 0000000..d6af6e8 --- /dev/null +++ b/tests/coverage/README @@ -0,0 +1,21 @@ + +Here you will find two scripts that can be used to generate code coverage +reports. + + +"coverage" runs the server and all the other tests with different parameters, +trying to maximize code coverage. + +As it does not test for correctness, it is mostly useful to see which code is +not used by the other tests. The output is saved to /tmp/nmdb-lcov.log, to +allow manual verification of the test results. + +It can be combined with gcov, lcov, or other code coverage tools, to generate +friendly reports. + + +"lcov-start" and "lcov-stop" can be used to generate coverage reports with the +lcov tool (http://ltp.sourceforge.net/coverage/lcov.php). The reports are +generated in the "lcov/" subdirectory. + + diff --git a/tests/coverage/coverage b/tests/coverage/coverage new file mode 100755 index 0000000..56c8405 --- /dev/null +++ b/tests/coverage/coverage @@ -0,0 +1,183 @@ +#!/bin/bash + + +if [ ! -x ./nmdb/nmdb ]; then + # attempt go to the git root + CDUP=$(git rev-parse --show-cdup 2>/dev/null) + if [ "$CDUP" != "" ]; then + cd $CDUP + fi + + if [ ! -x ./nmdb/nmdb ]; then + echo "Error: must be run from the project root" + exit 1 + fi +fi + + +DB="/tmp/nmdb-lcov-db" +LOG="/tmp/nmdb-lcov.log" +rm -f $LOG + +function log() { + echo "$@" >> $LOG +} + +function out() { + echo "$@" + log "$@" +} + +function run() { + log "-- running" "$@" + $@ >> $LOG 2>> $LOG + log +} + +function nmdb() { + log "-- nmdb starting:" "$@" + rm -f $DB + ./nmdb/nmdb -f -d $DB "$@" >> $LOG 2>> $LOG & + sleep 0.2 +} + +function kill_nmdb() { + killall nmdb >> $LOG 2>> $LOG + wait `pidof nmdb` + log "-- nmdb stopped" +} + +function nmdb_and_kill() { + log "-- nmdb_and_kill starting:" "$@" + rm -f $DB + ./nmdb/nmdb -f -d $DB "$@" >> $LOG 2>> $LOG & + sleep 0.2 + killall nmdb >> $LOG 2>> $LOG + wait `pidof nmdb` + log "-- nmdb_and_kill stopped" +} + + +out "+ help" + +# this one by hand so it also runs the "default database" path +run ./nmdb/nmdb -h + +# this one by hand for fork mode +out "+ fork mode" +./nmdb/nmdb >> $LOG; sleep 0.3; +killall nmdb >> $LOG 2>> $LOG; sleep 1.2; killall -9 nmdb >> $LOG 2>> $LOG + +out "+ command line" +nmdb_and_kill +nmdb_and_kill -p +nmdb_and_kill -b blah # supposed to fail + +nmdb_and_kill -l 20 +nmdb_and_kill -L 20 +nmdb_and_kill -t 26020 +nmdb_and_kill -u 26020 +nmdb_and_kill -s 26020 +nmdb_and_kill -T localhost # supposed to fail +nmdb_and_kill -U localhost # supposed to fail +nmdb_and_kill -S localhost # supposed to fail +nmdb_and_kill -T 127.0.0.1 +nmdb_and_kill -U 127.0.0.1 +nmdb_and_kill -S 127.0.0.1 +nmdb_and_kill -c 1 +nmdb_and_kill -o /dev/null +nmdb_and_kill -Z # supposed to fail +nmdb_and_kill -o /root/nmdb-coverage-fail # supposed to fail +nmdb_and_kill -d /dev/null # supposed to fail +nmdb_and_kill -i /tmp/nmdb-pid-file.pid +nmdb_and_kill -i /root/nmdb-pid-file.pid # supposed to fail + + +out "+ signals" +nmdb +run killall -HUP nmdb +run killall -USR1 nmdb +run killall -USR1 nmdb # the second time gets a different message +run killall -USR2 nmdb +run killall -TERM nmdb +wait `pidof nmdb` + + +for be in bdb tc qdbm tdb; do + out "+ backend $be" + run rm -f $DB + nmdb -b $be + if ! pidof nmdb > /dev/null; then + out " unsupported" + continue + fi + kill_nmdb + + out " + random" + nmdb -b $be -c 5 + run tests/python/random1.py db 5000 + run tests/python/random1.py cache 5000 + kill_nmdb + + out " + walk" + nmdb -b $be -c 5 + run tests/python/walk.py 5000 + kill_nmdb + + out " + performance" + nmdb -b $be -c 1 + pushd tests/c/ > /dev/null + for p in mult sctp tcp tipc udp; do + for t in cache normal sync; do + run ./1-$p-$t 1200 key val + run ./2-$p-$t 1200 8 8 + run ./3-$p-$t 1200 8 8 + run ./set-$p-$t 1200 8 8 + run ./get-$p-$t 1210 8 + run ./del-$p-$t 1210 8 + run ./incr-$p-$t 1200 10 + done + done + + # stress long tcp requests + run ./2-tcp-normal 1200 30000 30000 + + popd > /dev/null + kill_nmdb +done + +out "+ null backend" +nmdb -b null -c 1 +pushd tests/c/ > /dev/null +for p in mult sctp tcp tipc udp; do + for t in cache normal sync; do + run ./2-$p-$t 1200 8 8 + done +done +popd > /dev/null +kill_nmdb + +out "+ read-only" +nmdb +run tests/c/set-tipc-normal 1200 8 8 +kill_nmdb +nmdb -c 1 -r +pushd tests/c/ > /dev/null +for p in mult sctp tcp tipc udp; do + run ./set-$p-normal 1200 8 8 + run ./get-$p-normal 1210 8 + run ./del-$p-normal 1210 8 + run ./incr-$p-normal 1200 10 +done +popd > /dev/null +kill_nmdb + + +out "+ stats" +nmdb +run utils/nmdb-stats tipc 10 +run utils/nmdb-stats tcp localhost 26010 +run utils/nmdb-stats udp localhost 26010 +run utils/nmdb-stats sctp localhost 26010 +kill_nmdb + diff --git a/tests/coverage/lcov-start b/tests/coverage/lcov-start new file mode 100755 index 0000000..745d22d --- /dev/null +++ b/tests/coverage/lcov-start @@ -0,0 +1,21 @@ +#!/bin/bash + +if [ ! -x ./nmdb/nmdb ]; then + # attempt go to the git root + CDUP=$(git rev-parse --show-cdup 2>/dev/null) + if [ "$CDUP" != "" ]; then + cd $CDUP + fi + + if [ ! -x ./nmdb/nmdb ]; then + echo "Error: must be run from the project root" + exit 1 + fi +fi + +set -e + +LCOV="lcov --directory nmdb/ --base-directory nmdb/" +$LCOV --zerocounters + + diff --git a/tests/coverage/lcov-stop b/tests/coverage/lcov-stop new file mode 100755 index 0000000..da636c8 --- /dev/null +++ b/tests/coverage/lcov-stop @@ -0,0 +1,28 @@ +#!/bin/bash + +if [ ! -x ./nmdb/nmdb ]; then + # attempt go to the git root + CDUP=$(git rev-parse --show-cdup 2>/dev/null) + if [ "$CDUP" != "" ]; then + cd $CDUP + fi + + if [ ! -x ./nmdb/nmdb ]; then + echo "Error: must be run from the project root" + exit 1 + fi +fi + +set -e + +LCOV="lcov --directory nmdb/ --base-directory nmdb/" + +$LCOV -o tests/coverage/test_results.1 \ + --capture -t "Coverage tests" +$LCOV -o tests/coverage/test_results \ + --remove tests/coverage/test_results.1 '/usr/include/*' +rm tests/coverage/test_results.1 +mkdir -p tests/coverage/lcov +genhtml --show-details -o tests/coverage/lcov/ tests/coverage/test_results +rm tests/coverage/test_results + diff --git a/tests/perf/ag.sh b/tests/perf/ag.sh index 0ad9320..c572dc6 100755 --- a/tests/perf/ag.sh +++ b/tests/perf/ag.sh @@ -7,6 +7,11 @@ TIMES=5000 # Times parameter used for the tests set -e +if [ "$1" == "" ]; then + COMPARE="*" +else + COMPARE="$@" +fi # aggregate the results prepending the version, but we need a table because # gnuplot doesn't like strings as values @@ -18,7 +23,7 @@ rm -f ../ag-data/* declare -a COMMITS N=0 -for i in *; do +for i in $COMPARE; do COMMITS[$N]=$i for f in 2-tipc-cache.out 3-tipc-cache.out; do while read L; do diff --git a/tests/python/README b/tests/python/README new file mode 100644 index 0000000..3252f9f --- /dev/null +++ b/tests/python/README @@ -0,0 +1,4 @@ + +The tests in this directory are written to be compatible with both Python 2 +and 3. When making a change, please make sure they continue to be. + diff --git a/tests/python/random1-cache.py b/tests/python/random1-cache.py deleted file mode 100755 index 33ca332..0000000 --- a/tests/python/random1-cache.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python - -import sys -import nmdb -from random import randint, choice - - -class Mismatch (Exception): - pass - - -# network db -ndb = nmdb.Cache() -ndb.add_tipc_server() -ndb.add_tcp_server('localhost') -ndb.add_udp_server('localhost') - -# local db -ldb = {} - -# history of each key -history = {} - -# check decorator -def checked(f): - def newf(k, *args, **kwargs): - try: - return f(k, *args, **kwargs) - except: - if k in history: - print history[k] - else: - print 'No history for key', k - raise - newf.__name__ = f.__name__ - return newf - - -# operations -@checked -def set(k, v): - ndb[k] = v - ldb[k] = v - if k not in history: - history[k] = [] - history[k].append((set, k, v)) - -@checked -def get(k): - try: - n = ndb[k] - except KeyError: - del ldb[k] - del history[k] - return 0 - - l = ldb[k] - if l != n: - raise Mismatch, (n, l) - history[k].append((get, k)) - return True - -@checked -def delete(k): - del ldb[k] - try: - del ndb[k] - except KeyError: - pass - history[k].append((delete, k)) - -def find_missing(): - misses = 0 - for k in ldb.keys(): - if not get(k): - misses += 1 - return misses - -# Use integers because the normal random() generates floating point numbers, -# and they can mess up comparisons because of architecture details. -def getrand(): - return randint(0, 1000000000000000000) - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print 'Use: random1-cache.py number_of_keys [key_prefix]' - sys.exit(1) - - nkeys = int(sys.argv[1]) - if len(sys.argv) > 2: - key_prefix = sys.argv[2] - else: - key_prefix = '' - - # fill all the keys - print 'populate' - for i in xrange(nkeys): - set(key_prefix + str(getrand()), getrand()) - - print 'missing', find_missing() - - lkeys = ldb.keys() - - # operate on them a bit - print 'random operations' - operations = ('set', 'get', 'delete') - for i in xrange(nkeys / 2): - op = choice(operations) - k = choice(lkeys) - if op == 'set': - set(k, getrand()) - elif op == 'get': - get(k) - elif op == 'delete': - delete(k) - lkeys.remove(k) - - print 'missing', find_missing() - - print 'delete' - for k in lkeys: - delete(k) - - print 'missing', find_missing() - - sys.exit(0) - - diff --git a/tests/python/random1.py b/tests/python/random1.py index 172b4b5..c6097ff 100755 --- a/tests/python/random1.py +++ b/tests/python/random1.py @@ -1,98 +1,176 @@ #!/usr/bin/env python +""" +This is a stress test for the nmdb server, library, and Python (2 and 3) +bindings. + +It creates a number of keys, and then operates randomly on them, verifying +that everything is working as expected. + +It can operate with the database or only with the cache. + +You can run it with both Python 2 and 3. +""" + import sys import nmdb from random import randint, choice class Mismatch (Exception): + "Mismatch between local and remote database" pass -# network db -ndb = nmdb.DB() -ndb.add_tipc_server() -ndb.add_tcp_server('localhost') -ndb.add_udp_server('localhost') - -# local db -ldb = {} -# history of each key +# History of each key, global because we use it from several places for +# debugging purposes history = {} -# check decorator def checked(f): - def newf(k, *args, **kwargs): + "Prints the history of the key when an operation fails" + def newf(self, k, *args, **kwargs): try: - return f(k, *args, **kwargs) + return f(self, k, *args, **kwargs) except: if k in history: - print history[k] + print(history[k]) else: - print 'No history for key', k + print('No history for key', k) raise newf.__name__ = f.__name__ return newf -# operations -@checked -def set(k, v): - ndb[k] = v - ldb[k] = v - if k not in history: - history[k] = [] - history[k].append((set, k, v)) - -@checked -def get(k): - n = ndb[k] - l = ldb[k] - if l != n: - raise Mismatch, (n, l) - history[k].append((get, k)) - return n - -@checked -def delete(k): - del ndb[k] - del ldb[k] - history[k].append((delete, k)) - -@checked -def cas(k, ov, nv): - prel = ldb[k] - pren = ndb[k] - n = ndb.cas(k, ov, nv) - if not ldb.has_key(k): - l = 0 - elif ldb[k] == ov: - ldb[k] = nv - l = 2 - else: - l = 1 - if n != l: - print k, ldb[k], ndb[k] - print prel, pren - print history[k] - raise Mismatch, (n, l) - history[k].append((cas, k, ov, nv)) - return n - - -def check(): - for k in ldb.keys(): +class Tester: + "Common code for tester classes" + def __init__(self): + # nmdb connection + self.ndb = self.connect() + self.ndb.add_tipc_server() + self.ndb.add_tcp_server('localhost') + self.ndb.add_udp_server('localhost') + + # local database + self.ldb = {} + + def connect(self): + "Connects to an nmdb instance" + raise NotImplementedError + + @checked + def set(self, k, v): + self.ndb[k] = v + self.ldb[k] = v + if k not in history: + history[k] = [] + history[k].append((self.set, k, v)) + + def check(self): + "Checks all entries in ldb match the ones in ndb" + n = l = None + for k in self.ldb.keys(): + # get() verifies they match so we do not need to care + # about that + self.get(k) + + def get(self, k): + raise NotImplementedError + + def delete(self, k): + raise NotImplementedError + + def cas(self, k, ov, nv): + raise NotImplementedError + + def find_missing(self): + raise NotImplementedError + + +class DBTester (Tester): + "Tester for db mode" + + def connect(self): + return nmdb.DB() + + @checked + def get(self, k): + n = self.ndb[k] + l = self.ldb[k] + if l != n: + raise Mismatch((n, l)) + history[k].append((self.get, k)) + return n + + @checked + def delete(self, k): + del self.ndb[k] + del self.ldb[k] + history[k].append((self.delete, k)) + + @checked + def cas(self, k, ov, nv): + prel = self.ldb[k] + pren = self.ndb[k] + n = self.ndb.cas(k, ov, nv) + if k not in self.ldb: + l = 0 + elif self.ldb[k] == ov: + self.ldb[k] = nv + l = 2 + else: + l = 1 + if n != l: + print(k, self.ldb[k], self.ndb[k]) + print(prel, pren) + print(history[k]) + raise Mismatch((n, l)) + history[k].append((self.cas, k, ov, nv)) + return n + + def find_missing(self): + # we just check we can get them all + for k in self.ldb.keys(): + self.get(k) + return 0 + + +class CacheTester (Tester): + "Tester for cache mode" + + def connect(self): + return nmdb.Cache() + + @checked + def get(self, k): try: - n = ndb[k] - l = ldb[k] - except: - print history[k] - raise Mismatch, (n, l) + n = self.ndb[k] + except KeyError: + del self.ldb[k] + del history[k] + return False + + l = self.ldb[k] + if l != n: + raise Mismatch((n, l)) + history[k].append((self.get, k)) + return True + + @checked + def delete(self, k): + del self.ldb[k] + try: + del self.ndb[k] + except KeyError: + pass + history[k].append((self.delete, k)) - if n != n: - print history[k] - raise Mismatch, (n, l) + def find_missing(self): + misses = 0 + for k in list(self.ldb.keys()): + if not self.get(k): + misses += 1 + return misses # Use integers because the normal random() generates floating point numbers, @@ -101,54 +179,81 @@ def getrand(): return randint(0, 1000000000000000000) -if __name__ == '__main__': - if len(sys.argv) < 2: - print 'Use: random1.py number_of_keys [key_prefix]' +def main(): + # We want to always use a generator range(), which has different names + # in Python 2 and 3, so isolate the code from this hack + if sys.version_info[0] == 2: + gen_range = xrange + else: + gen_range = range + + if len(sys.argv) < 3: + print('Use: random1.py db|cache number_of_keys [key_prefix]') sys.exit(1) - nkeys = int(sys.argv[1]) - if len(sys.argv) > 2: - key_prefix = sys.argv[2] + mode = sys.argv[1] + if mode not in ('db', 'cache'): + print('Error: mode must be either db or cache') + sys.exit(1) + + nkeys = int(sys.argv[2]) + if len(sys.argv) >= 4: + key_prefix = sys.argv[3] else: key_prefix = '' + if mode == 'db': + tester = DBTester() + else: + tester = CacheTester() + # fill all the keys - print 'populate' - for i in xrange(nkeys): - set(key_prefix + str(getrand()), getrand()) + print('populate') + for i in gen_range(nkeys): + tester.set(key_prefix + str(getrand()), getrand()) - lkeys = ldb.keys() + print('missing', tester.find_missing()) + + lkeys = list(tester.ldb.keys()) # operate on them a bit - print 'random operations' - operations = ('set', 'delete', 'cas0', 'cas1') - for i in xrange(nkeys / 2): + print('random operations') + + if mode == 'db': + operations = ('set', 'delete', 'cas0', 'cas1') + else: + operations = ('set', 'get', 'delete') + + for i in gen_range(min(len(lkeys), nkeys // 2)): op = choice(operations) k = choice(lkeys) if op == 'set': - set(k, getrand()) + tester.set(k, getrand()) + elif op == 'get': + tester.get(k) elif op == 'delete': - delete(k) + tester.delete(k) lkeys.remove(k) elif op == 'cas0': # unsucessful cas - cas(k, -1, -1) + tester.cas(k, -1, -1) elif op == 'cas1': # successful cas - cas(k, ldb[k], getrand()) - - print 'check' - check() + tester.cas(k, tester.ldb[k], getrand()) - print 'delete' - for k in lkeys: - delete(k) + print('missing', tester.find_missing()) - print 'check' - check() + print('check') + tester.check() - sys.exit(0) + print('delete') + for k in lkeys: + tester.delete(k) + print('check') + tester.check() +if __name__ == '__main__': + main() diff --git a/tests/python/walk.py b/tests/python/walk.py new file mode 100755 index 0000000..36843cb --- /dev/null +++ b/tests/python/walk.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +""" +This tests adds some random keys and then checks if they all appear when +walking through all the keys in the server. + +You can run it with both Python 2 and 3. +""" + +import sys +import nmdb +from random import randint, choice + + +class Tester: + def __init__(self): + # nmdb connection + self.ndb = nmdb.DB() + self.ndb.add_tipc_server() + + # disable autopickle so we don't crash if there are keys in + # the database that were not stored using it + self.ndb.autopickle = False + + # local database + self.ldb = {} + + def set(self, k, v): + self.ndb[k] = v + self.ldb[k] = v + + def check(self): + keys_not_found = set(self.ldb) + repeated = {} + unknown = 0 + + try: + k = self.ndb.firstkey() + except KeyError: + return None + + while k: + if k in self.ldb: + if k in keys_not_found: + keys_not_found.remove(k) + else: + repeated.setdefault(k, 0) + repeated[k] += 1 + else: + unknown += 1 + + try: + k = self.ndb.nextkey(k) + except KeyError: + break + + return keys_not_found, repeated, unknown + +# Use integers because the normal random() generates floating point numbers, +# and they can mess up comparisons because of architecture details. +def getrand(): + return randint(0, 1000000000000000000) + + +def main(): + # We want to always use a generator range(), which has different names + # in Python 2 and 3, so isolate the code from this hack + if sys.version_info[0] == 2: + gen_range = xrange + else: + gen_range = range + + if len(sys.argv) < 2: + print('Use: random1.py number_of_keys [key_prefix]') + sys.exit(1) + + nkeys = int(sys.argv[1]) + if len(sys.argv) >= 3: + key_prefix = sys.argv[2] + else: + key_prefix = '' + + tester = Tester() + + # fill all the keys + print('populate') + for i in gen_range(nkeys): + k = bytes( (key_prefix + str(getrand())).encode("utf8") ) + tester.set(k, bytes(str(getrand()).encode("ascii")) ) + + print('check') + keys_not_found, repeated, unknown = tester.check() + print(' keys not found: %d' % len(keys_not_found)) + print(' repeated: %d' % len(repeated)) + print(' unknown: %d' % unknown) + + print('delete') + for k in tester.ldb: + del tester.ndb[k] + +if __name__ == '__main__': + main() + diff --git a/tests/python3/README b/tests/python3/README deleted file mode 100644 index 9f857b7..0000000 --- a/tests/python3/README +++ /dev/null @@ -1,4 +0,0 @@ - -These tests are identical to the ones in tests/python, except they've been -modified to work under Python 3 (and obviously use the Python 3 bindings). - diff --git a/tests/python3/random1-cache.py b/tests/python3/random1-cache.py deleted file mode 100755 index 71e8b9f..0000000 --- a/tests/python3/random1-cache.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python - -import sys -import nmdb -from random import randint, choice - - -class Mismatch (Exception): - pass - - -# network db -ndb = nmdb.Cache() -ndb.add_tipc_server() -ndb.add_tcp_server('localhost') -ndb.add_udp_server('localhost') - -# local db -ldb = {} - -# history of each key -history = {} - -# check decorator -def checked(f): - def newf(k, *args, **kwargs): - try: - return f(k, *args, **kwargs) - except: - if k in history: - print(history[k]) - else: - print('No history for key', k) - raise - newf.__name__ = f.__name__ - return newf - - -# operations -@checked -def set(k, v): - ndb[k] = v - ldb[k] = v - if k not in history: - history[k] = [] - history[k].append((set, k, v)) - -@checked -def get(k): - try: - n = ndb[k] - except KeyError: - del ldb[k] - del history[k] - return 0 - - l = ldb[k] - if l != n: - raise Mismatch((n, l)) - history[k].append((get, k)) - return True - -@checked -def delete(k): - del ldb[k] - try: - del ndb[k] - except KeyError: - pass - history[k].append((delete, k)) - -def find_missing(): - misses = 0 - for k in list(ldb.keys()): - if not get(k): - misses += 1 - return misses - -# Use integers because the normal random() generates floating point numbers, -# and they can mess up comparisons because of architecture details. -def getrand(): - return randint(0, 1000000000000000000) - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print('Use: random1-cache.py number_of_keys [key_prefix]') - sys.exit(1) - - nkeys = int(sys.argv[1]) - if len(sys.argv) > 2: - key_prefix = sys.argv[2] - else: - key_prefix = '' - - # fill all the keys - print('populate') - for i in range(nkeys): - set(key_prefix + str(getrand()), getrand()) - - print('missing', find_missing()) - - lkeys = list(ldb.keys()) - - # operate on them a bit - print('random operations') - operations = ('set', 'get', 'delete') - for i in range(nkeys // 2): - op = choice(operations) - k = choice(lkeys) - if op == 'set': - set(k, getrand()) - elif op == 'get': - get(k) - elif op == 'delete': - delete(k) - lkeys.remove(k) - - print('missing', find_missing()) - - print('delete') - for k in lkeys: - delete(k) - - print('missing', find_missing()) - - sys.exit(0) - - diff --git a/tests/python3/random1.py b/tests/python3/random1.py deleted file mode 100755 index ae18bc6..0000000 --- a/tests/python3/random1.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python - -import sys -import nmdb -from random import randint, choice - - -class Mismatch (Exception): - pass - - -# network db -ndb = nmdb.DB() -ndb.add_tipc_server() -ndb.add_tcp_server('localhost') -ndb.add_udp_server('localhost') - -# local db -ldb = {} - -# history of each key -history = {} - -# check decorator -def checked(f): - def newf(k, *args, **kwargs): - try: - return f(k, *args, **kwargs) - except: - if k in history: - print(history[k]) - else: - print('No history for key', k) - raise - newf.__name__ = f.__name__ - return newf - - -# operations -@checked -def set(k, v): - ndb[k] = v - ldb[k] = v - if k not in history: - history[k] = [] - history[k].append((set, k, v)) - -@checked -def get(k): - n = ndb[k] - l = ldb[k] - if l != n: - raise Mismatch((n, l)) - history[k].append((get, k)) - return n - -@checked -def delete(k): - del ndb[k] - del ldb[k] - history[k].append((delete, k)) - -@checked -def cas(k, ov, nv): - prel = ldb[k] - pren = ndb[k] - n = ndb.cas(k, ov, nv) - if k not in ldb: - l = 0 - elif ldb[k] == ov: - ldb[k] = nv - l = 2 - else: - l = 1 - if n != l: - print(k, ldb[k], ndb[k]) - print(prel, pren) - print(history[k]) - raise Mismatch((n, l)) - history[k].append((cas, k, ov, nv)) - return n - - -def check(): - for k in ldb.keys(): - try: - n = ndb[k] - l = ldb[k] - except: - print(history[k]) - raise Mismatch((n, l)) - - if n != n: - print(history[k]) - raise Mismatch((n, l)) - - -# Use integers because the normal random() generates floating point numbers, -# and they can mess up comparisons because of architecture details. -def getrand(): - return randint(0, 1000000000000000000) - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print('Use: random1.py number_of_keys [key_prefix]') - sys.exit(1) - - nkeys = int(sys.argv[1]) - if len(sys.argv) > 2: - key_prefix = sys.argv[2] - else: - key_prefix = '' - - # fill all the keys - print('populate') - for i in range(nkeys): - set(key_prefix + str(getrand()), getrand()) - - lkeys = list(ldb.keys()) - - # operate on them a bit - print('random operations') - operations = ('set', 'delete', 'cas0', 'cas1') - for i in range(nkeys // 2): - op = choice(operations) - k = choice(lkeys) - if op == 'set': - set(k, getrand()) - elif op == 'delete': - delete(k) - lkeys.remove(k) - elif op == 'cas0': - # unsucessful cas - cas(k, -1, -1) - elif op == 'cas1': - # successful cas - cas(k, ldb[k], getrand()) - - print('check') - check() - - print('delete') - for k in lkeys: - delete(k) - - print('check') - check() - - sys.exit(0) - - - - diff --git a/utils/nmdb-stats.c b/utils/nmdb-stats.c index d9f1647..f3cd6f2 100644 --- a/utils/nmdb-stats.c +++ b/utils/nmdb-stats.c @@ -8,6 +8,7 @@ #include /* uint64_t */ #include /* strcmp() */ #include /* atoi() */ +#include /* htonl() and friends */ #include "nmdb.h" @@ -102,12 +103,12 @@ int main(int argc, char **argv) /* The following assumes it can be more than one server. This can * never happen with the current code, but it can be useful as an * example in the future. */ - j = 0; for (i = 0; i < nservers; i++) { printf("stats for server %d:\n", i); j = nstats * i; + /* note they are not necessarily in numerical order */ shst("cache get", 0); shst("cache set", 1); shst("cache del", 2); @@ -119,6 +120,8 @@ int main(int argc, char **argv) shst("db del", 7); shst("db cas", 8); shst("db incr", 9); + shst("db firstkey", 21); + shst("db nextkey", 22); shst("cache hits", 10); shst("cache misses", 11); @@ -136,7 +139,7 @@ int main(int argc, char **argv) shst("unknown requests", 20); /* if there are any fields we don't know, show them anyway */ - for (k = 21; k < nstats; k++) { + for (k = 23; k < nstats; k++) { shst("unknown field", k); }