.gitignore | 16 +++ INSTALL | 68 ++++++++++ LICENSE | 2 + README | 41 ++----- UPGRADING | 13 ++ bindings/bigloo/LICENSE | 34 +++++ bindings/bigloo/Makefile | 29 +++++ bindings/bigloo/nmdb.scm | 155 +++++++++++++++++++++++ bindings/bigloo/test1.scm | 25 ++++ bindings/d/nmdb.d | 54 +++++++- bindings/d/nmdb_ll.d | 6 + bindings/haskell/LICENSE | 34 +++++ bindings/haskell/Nmdb.hs | 174 ++++++++++++++++++++++++++ bindings/haskell/Setup.lhs | 5 + bindings/haskell/nmdb.cabal | 22 ++++ bindings/haskell/test1.hs | 43 +++++++ bindings/newlisp/nmdb.lsp | 31 ++++-- bindings/newlisp/test.lsp | 7 +- bindings/python/nmdb.py | 166 ++++++++++++++++++------- bindings/python/nmdb_ll.c | 63 ++++++++-- bindings/ruby/LICENSE | 34 +++++ bindings/ruby/extconf.rb | 13 ++ bindings/ruby/nmdb.rb | 208 +++++++++++++++++++++++++++++++ bindings/ruby/nmdb_ll.c | 262 +++++++++++++++++++++++++++++++++++++++ bindings/ruby/test1.rb | 18 +++ doc/design.rst | 17 ++- doc/guide.rst | 18 ++- doc/network.rst | 16 ++- libnmdb/Makefile | 37 ++++-- libnmdb/internal.h | 6 +- libnmdb/libnmdb.3 | 52 ++++++-- libnmdb/libnmdb.c | 140 +++++++++++++++++++-- libnmdb/net-const.h | 6 + libnmdb/{nmdb.h => nmdb.skel.h} | 35 +++++- libnmdb/{udp.c => sctp.c} | 57 +++++---- libnmdb/sctp.h | 12 ++ libnmdb/tcp.c | 11 +- libnmdb/tipc.c | 1 + libnmdb/udp.c | 15 ++- nmdb/Makefile | 51 ++++++-- nmdb/be-bdb.c | 99 +++++++++++++++ nmdb/be-null.c | 37 ++++++ nmdb/be-qdbm.c | 2 +- nmdb/be.h | 22 +++- nmdb/cache.c | 77 ++++++++++-- nmdb/cache.h | 3 + nmdb/common.h | 3 + nmdb/{db.c => dbloop.c} | 72 +++++++++-- nmdb/{db.h => dbloop.h} | 4 +- nmdb/log.c | 65 ++++++++++ nmdb/log.h | 18 +++ nmdb/main.c | 41 +++++- nmdb/net-const.h | 6 + nmdb/net.c | 44 +++++-- nmdb/nmdb.1 | 46 ++++--- nmdb/parse.c | 119 ++++++++++++++++-- nmdb/req.h | 8 +- nmdb/sctp-stub.c | 18 +++ nmdb/{udp.c => sctp.c} | 87 ++++++------- nmdb/sctp.h | 10 ++ nmdb/tcp.c | 77 ++++++------ nmdb/tipc.c | 45 ++----- nmdb/udp.c | 45 ++----- tests/c/1.c | 4 +- tests/c/2.c | 4 +- tests/c/3.c | 4 +- tests/c/get.c | 4 +- tests/c/incr.c | 67 ++++++++++ tests/c/make.sh | 11 +- tests/c/prototypes.h | 6 + tests/python/random1-cache.py | 122 ++++++++++++++++++ 71 files changed, 2715 insertions(+), 452 deletions(-) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dc7cf5e --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# git-ls-files --others --exclude-from=.git/info/exclude +# Lines that start with '#' are comments. +# For a project mostly in C, the following would be a good set of +# exclude patterns (uncomment them if you want to use them): +*.[oa] +*.so +.*.swp +*.gcno +*.gcda +# *~ +/nmdb/nmdb +tests/c/*-*-cache +tests/c/*-*-normal +tests/c/*-*-sync +/libnmdb/nmdb.h +/tags diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..687baa8 --- /dev/null +++ b/INSTALL @@ -0,0 +1,68 @@ + +Quick guide for the patience-impaired +------------------------------------- + +At the top level directory, run: + + $ make BACKEND=qdbm ENABLE_TIPC=0 ENABLE_SCTP=0 install + +to build and install the server without TIPC and SCTP support, and to use the +qdbm backend. + + +How to compile and install +-------------------------- + +The only mandatory requisite to build nmdb is libevent +(http://www.monkey.org/~provos/libevent/). + +There are several build-time options, in two groups: supported network +protocols, and backend databases. + +The network protocols can all be enabled at the same time: + * TCP and UDP, the well-known network protocols. + * TIPC (http://tipc.sf.net/), a cluster-oriented network protocol. You will + need a Linux kernel >= 2.6.16 with TIPC support (most distributions + enable it by default). + * SCTP, a network protocol similar to UDP and TCP, offering reliable message + passing over IP, among other very useful things. You will need the + lksctp-tools package. + +The backends, on the other hand, are mutually exclusive and only one can be +selected: + * qdbm (http://qdbm.sf.net/). + * bdb (http://www.oracle.com/database/berkeley-db/). + * null backend, if you don't need a real one. + +By default, all network protocols are enabled, and the qdbm backend is +selected. + +You can change the defaults by passing parameters to make, like this: + + $ make BACKEND=[qbdm|bdb|null] ENABLE_$PROTO=[1|0] + +Where $PROTO can be TCP, UDP, TIPC or SCTP. + +For instance, to build with bdb backend and without TIPC and UDP support, use: + + $ make BACKEND=bdb ENABLE_TIPC=0 + + +Tests +----- + +To run some tests, start the server and then go to the "tests/c/" directory. +Run "make.sh build" and then run any of the generated tests. + + +Bindings +-------- + +To compile the Python bindings, you need to have the library already +installed. Use "make python_install" at the top level directory to build and +install the modules. The module will be named "nmdb". + +The other bindings do not have a properly defined install procedure, and +you'll need knowledge of the language to install them. + + diff --git a/LICENSE b/LICENSE index 3b7cc0c..bde222e 100644 --- a/LICENSE +++ b/LICENSE @@ -12,5 +12,7 @@ As a brief resume, here's how each sub-project is licensed: * bindings/python: BOLA (Public domain) * bindings/d: BOLA (Public domain) * bindings/newlisp: BOLA (Public domain) + * bindings/ruby: BOLA (Public domain) + * bindings/bigloo: BOLA (Public domain) diff --git a/README b/README index bde5926..9702b34 100644 --- a/README +++ b/README @@ -3,8 +3,8 @@ nmdb - A multiprotocol network database manager Alberto Bertogli (albertito@gmail.com) --------------------------------------------------- -nmdb is a network database that can use several protocols to communicate with -it's clients. At the moment, it supports TIPC, TCP and UDP. +nmdb is a network database that can use different protocols to communicate +with its clients. At the moment, it supports TIPC, TCP, UDP and SCTP. It consists of an in-memory cache, that saves (key, value) pairs, and a persistent backend that stores the pairs on disk. @@ -13,9 +13,9 @@ Both work combined, but the use of the persistent backend is optional, so you can use the server only for cache queries, pretty much like memcached. This source distribution is composed of several parts: the server called -"nmdb", the library and bindings for Python, D and NewLISP. Each one has a -separate directory, and is licensed individually. See the LICENSE file for -more information. +"nmdb", the library and bindings for Python, D, NewLISP, Ruby, Bigloo Scheme +and Haskell. Each one has a separate directory, and is licensed individually. +See the LICENSE file for more information. Documentation @@ -29,35 +29,12 @@ For additional documentation and resources, go to the project's website at http://auriga.wearlab.de/~alb/nmdb. -How to compile, test and install --------------------------------- - -General requisites: - * libevent (http://www.monkey.org/~provos/libevent/). - * qdbm (http://qdbm.sf.net/). - -Requisites to enable TIPC: - * Running Linux kernel >= 2.6.16 with TIPC enabled. - * Kernel headers also >= 2.6.16. Alternatively, you can run a TIPC patched - kernel, and have the tipc.h header somewhere in the include path. - -To compile the server and the library, you can just use "make" on the top -level directory. To install them, use "make install". By default, all -protocols are enabled, including TIPC. To disable any of them, run something -like "make ENABLE_TIPC=0". - -To run some tests, start the server and then go to the "tests/c/" directory. -Run "make.sh build" and then run any of the generated tests. - -To compile the Python bindings, you need to have the library already -installed. Use "make python_install" at the top level directory to build and -install the modules. The module will be named "nmdb". - - Where to report bugs -------------------- -Please report any bugs, suggestions, issues or comments to me, Alberto -Bertogli, at albertito@gmail.com. +If you want to report bugs, suggestions, issues, comments, or just get in +touch with nmdb developers and users, join the mailing list at +http://lists.auriga.wearlab.de/cgi-bin/mailman/listinfo/nmdb-devel or just +send an email to nmdb-devel@lists.auriga.wearlab.de. diff --git a/UPGRADING b/UPGRADING new file mode 100644 index 0000000..bd4c765 --- /dev/null +++ b/UPGRADING @@ -0,0 +1,13 @@ + +Here are the notes for upgrading nmdb from one version to another. + +While normally nothing should be done, sometimes things change in incompatible +ways. Here's the listing for the releases, you should always check it before +upgrading. + +0.20 -> 0.21 + * nmdb_get() and nmdb_cache_get() now return -1 instead of 0 when there is a + missing key. This was done to allow setting a value to be "" (no data). All + bindings have been upgraded, so you only need to worry if you used the C + library. + diff --git a/bindings/bigloo/LICENSE b/bindings/bigloo/LICENSE new file mode 100644 index 0000000..f3a9498 --- /dev/null +++ b/bindings/bigloo/LICENSE @@ -0,0 +1,34 @@ + +I don't like licenses, because I don't like having to worry about all this +legal stuff just for a simple piece of software I don't really mind anyone +using. But I also believe that it's important that people share and give back; +so I'm placing this library under the following license, so you feel guilty if +you don't ;) + + +BOLA - Buena Onda License Agreement +----------------------------------- + +This work is provided 'as-is', without any express or implied warranty. In no +event will the authors be held liable for any damages arising from the use of +this work. + +To all effects and purposes, this work is to be considered Public Domain. + + +However, if you want to be "Buena onda", you should: + +1. Not take credit for it, and give proper recognition to the authors. +2. Share your modifications, so everybody benefits from them. +4. Do something nice for the authors. +5. Help someone who needs it: sign up for some volunteer work or help your + neighbour paint the house. +6. Don't waste. Anything, but specially energy that comes from natural + non-renewable resources. Extra points if you discover or invent something + to replace them. +7. Be tolerant. Everything that's good in nature comes from cooperation. + +The order is important, and the further you go the more "Buena onda" you are. +Make the world a better place: be "Buena onda". + + diff --git a/bindings/bigloo/Makefile b/bindings/bigloo/Makefile new file mode 100644 index 0000000..64bd1c9 --- /dev/null +++ b/bindings/bigloo/Makefile @@ -0,0 +1,29 @@ + +BFLAGS += -Wall -O6 +ALL_BFLAGS = $(BFLAGS) + +ifdef DEBUG +ALL_CFLAGS += -g +endif + +ifdef PROFILE +ALL_CFLAGS += -pg +endif + + +default: all + +all: lib test1 + +lib: + bigloo $(ALL_BFLAGS) -c nmdb.scm -o nmdb.o + +test1: lib + bigloo $(ALL_BFLAGS) nmdb.o test1.scm -lnmdb -o test1 + +clean: + rm -f nmdb.o test1.o test1 + +.PHONY: default all clean + + diff --git a/bindings/bigloo/nmdb.scm b/bindings/bigloo/nmdb.scm new file mode 100644 index 0000000..014dbdb --- /dev/null +++ b/bindings/bigloo/nmdb.scm @@ -0,0 +1,155 @@ + +;; Bigloo nmdb bindings + +(module nmdb + + ;; C functions + (extern + (type _nmdb_t (pointer void) "void *") + + (macro _nmdb_init::_nmdb_t () "nmdb_init") + (macro _nmdb_free::int (::_nmdb_t) "nmdb_free") + + (macro _nmdb_add_tipc_server::int (::_nmdb_t ::int) + "nmdb_add_tipc_server") + (macro _nmdb_add_tcp_server::int (::_nmdb_t ::string ::int) + "nmdb_add_tcp_server") + (macro _nmdb_add_udp_server::int (::_nmdb_t ::string ::int) + "nmdb_add_udp_server") + (macro _nmdb_add_sctp_server::int (::_nmdb_t ::string ::int) + "nmdb_add_sctp_server") + + (macro _nmdb_set::int + (::_nmdb_t ::string ::uint ::string ::uint) + "nmdb_set") + (macro _nmdb_set_sync::int + (::_nmdb_t ::string ::uint ::string ::uint) + "nmdb_set_sync") + (macro _nmdb_cache_set::int + (::_nmdb_t ::string ::uint ::string ::uint) + "nmdb_cache_set") + + (macro _nmdb_get::ulong + (::_nmdb_t ::string ::uint ::string ::uint) + "nmdb_get") + (macro _nmdb_cache_get::ulong + (::_nmdb_t ::string ::uint ::string ::uint) + "nmdb_cache_get") + + (macro _nmdb_del::int + (::_nmdb_t ::string ::uint) + "nmdb_del") + (macro _nmdb_del_sync::int + (::_nmdb_t ::string ::uint) + "nmdb_del_sync") + (macro _nmdb_cache_del::int + (::_nmdb_t ::string ::uint) + "nmdb_cache_del") + + (macro _nmdb_cas::int + (::_nmdb_t ::string ::uint ::string ::uint ::string ::uint) + "nmdb_cas") + (macro _nmdb_cache_cas::int + (::_nmdb_t ::string ::uint ::string ::uint ::string ::uint) + "nmdb_cache_cas") + + (macro _nmdb_incr::int + (::_nmdb_t ::string ::uint ::long) + "nmdb_incr") + (macro _nmdb_cache_incr::int + (::_nmdb_t ::string ::uint ::long) + "nmdb_cache_incr") + + ) + + (export + (make-nmdb) + (nmdb-free db) + + (nmdb-add-tipc-server db port) + (nmdb-add-tcp-server db addr port) + (nmdb-add-udp-server db addr port) + (nmdb-add-sctp-server db addr port) + + (nmdb-get db key) + (nmdb-cache-get db key) + + (nmdb-set db key val) + (nmdb-set-sync db key val) + (nmdb-cache-set db key val) + + (nmdb-del db key) + (nmdb-del-sync db key) + (nmdb-cache-del db key) + + (nmdb-cas db key oldval newval) + (nmdb-cache-cas db key oldval newval) + + (nmdb-incr db key increment) + (nmdb-cache-incr db key increment) + ) + + ) + + +;; creator and destructor +(define (make-nmdb) (_nmdb_init)) +(define (nmdb-free db) (_nmdb_free db)) + +;; adding servers +(define (nmdb-add-tipc-server db port) (_nmdb_add_tipc_server db port)) +(define (nmdb-add-tcp-server db addr port) (_nmdb_add_tcp_server db addr port)) +(define (nmdb-add-udp-server db addr port) (_nmdb_add_udp_server db addr port)) +(define (nmdb-add-sctp-server db addr port) (_nmdb_add_sctp_server db addr port)) + +;; get functions +(define (nmdb-generic-get func db key) + (define buflen (* 70 1024)) + (define buf (make-string buflen)) + (define vsize (func db key (string-length key) buf buflen)) + (if (< vsize 0) + vsize + (substring buf 0 vsize) ) + ) + +(define (nmdb-get db key) (nmdb-generic-get _nmdb_get db key)) +(define (nmdb-cache-get db key) (nmdb-generic-get _nmdb_cache_get db key)) + +;; set functions +(define (nmdb-generic-set func db key val) + (func db key (string-length key) val (string-length val)) ) +(define (nmdb-set db key val) + (nmdb-generic-set _nmdb_set db key val)) +(define (nmdb-set-sync db key val) + (nmdb-generic-set _nmdb_set_sync db key val)) +(define (nmdb-cache-set db key val) + (nmdb-generic-set _nmdb_cache_set db key val)) + +;; del functions +(define (nmdb-generic-del func db key) + (func db key (string-length key)) ) +(define (nmdb-del db key) + (nmdb-generic-del _nmdb_del db key)) +(define (nmdb-del-sync db key) + (nmdb-generic-del _nmdb_del_sync db key)) +(define (nmdb-cache-del db key) + (nmdb-generic-del _nmdb_cache_del db key)) + +;; cas functions +(define (nmdb-generic-cas func db key oldval newval) + (func db key (string-length key) + oldval (string-length oldval) + newval (string-length newval) ) ) +(define (nmdb-cas db key oldval newval) + (nmdb-generic-cas _nmdb_cas db key oldval newval)) +(define (nmdb-cache-cas db key oldval newval) + (nmdb-generic-cas _nmdb_cache_cas db key oldval newval)) + +;; incr functions +(define (nmdb-generic-incr func db key increment) + (func db key (string-length key) increment ) ) +(define (nmdb-incr db key increment) + (nmdb-generic-incr _nmdb_incr db key increment)) +(define (nmdb-cache-incr db key increment) + (nmdb-generic-incr _nmdb_cache_incr db key increment)) + diff --git a/bindings/bigloo/test1.scm b/bindings/bigloo/test1.scm new file mode 100644 index 0000000..4262a87 --- /dev/null +++ b/bindings/bigloo/test1.scm @@ -0,0 +1,25 @@ + +(module test1 + (import (nmdb "nmdb.scm")) ) + +(define db (make-nmdb)) +(nmdb-add-tipc-server db -1) + +(print) +(print "db-set D1 V1\t" (nmdb-set db "D1" "D1")) +(print "sync-set S2 V2\t" (nmdb-set-sync db "S2" "V2")) +(print "cache-set C3 C3\t" (nmdb-cache-set db "C3" "C3")) +(print) +(print "db-get D1\t" (nmdb-get db "D1")) +(print "db-get S2\t" (nmdb-get db "S2")) +(print "cache-get C3\t" (nmdb-cache-get db "C3")) +(print) +(print "db-cas D1\t" (nmdb-cas db "D1" "D1" "DX")) +(print "cache-cas C3\t" (nmdb-cache-cas db "C3" "C3" "CX")) +(print) +(print "db-del D1\t" (nmdb-del db "D1")) +(print "sync-del S2\t" (nmdb-del-sync db "S2")) +(print "cache-del C3\t" (nmdb-cache-del db "C3")) + +(nmdb-free db) + diff --git a/bindings/d/nmdb.d b/bindings/d/nmdb.d index e746d86..9c96062 100644 --- a/bindings/d/nmdb.d +++ b/bindings/d/nmdb.d @@ -1,6 +1,6 @@ /* - * nmdb bindings for the Digital Mars D programming language. + * Digital Mars D programming language bindings for the nmdb C library. * Alberto Bertogli (albertito@gmail.com) */ @@ -67,6 +67,14 @@ class DB } } + void add_sctp_server(char[] addr, int port = -1) + { + int r = nmdb_add_sctp_server(db, cast(ubyte *) addr.ptr, port); + if (r == 0) { + throw new Exception("Can't add server"); + } + } + private char[] do_get(char[] key, int mode) { ptrdiff_t size; @@ -75,21 +83,24 @@ class DB if (mode == MODE_NORMAL || mode == MODE_SYNC) { size = nmdb_get(db, k, key.length, - cast(ubyte *) v, v.sizeof); + cast(ubyte *) v, v.length); } else if (mode == MODE_CACHE) { size = nmdb_cache_get(db, k, key.length, - cast(ubyte *) v, v.sizeof); + cast(ubyte *) v, v.length); } else { throw new Exception("Invalid mode"); } - if (size == 0) { + if (size == -1) { throw new KeyNotFound("Key not found: " ~ key); - } else if (size < 0) { + } else if (size <= -2) { throw new Exception("Can't get value"); } - return v[0 .. cast(size_t) size]; + // resize using D's magic + v.length = cast(size_t) size; + + return v; } private void do_set(char[] key, char[] val, int mode) @@ -154,6 +165,21 @@ class DB return res; } + private int do_incr(char[] key, long increment, int mode) + { + ubyte* k = cast(ubyte *) key.ptr; + int res = 0; + + if (mode == MODE_NORMAL || mode == MODE_SYNC) { + res = nmdb_incr(db, k, key.length, increment); + } else if (mode == MODE_CACHE) { + res = nmdb_cache_incr(db, k, key.length, increment); + } else { + throw new Exception("Invalid mode"); + } + return res; + } + char[] get(char[] key) { @@ -229,6 +255,22 @@ class DB } + int incr(char[] key, long increment) + { + return do_incr(key, increment, mode); + } + + int incr_normal(char[] key, long increment) + { + return do_incr(key, increment, MODE_NORMAL); + } + + int cache_incr(char[] key, long increment) + { + return do_incr(key, increment, MODE_CACHE); + } + + char[] opIndex(char[] key) { return get(key); diff --git a/bindings/d/nmdb_ll.d b/bindings/d/nmdb_ll.d index 6fc3852..beb28d3 100644 --- a/bindings/d/nmdb_ll.d +++ b/bindings/d/nmdb_ll.d @@ -40,6 +40,7 @@ extern (C) nmdb_t *nmdb_init(); extern (C) int nmdb_add_tipc_server(nmdb_t *db, int port); extern (C) int nmdb_add_tcp_server(nmdb_t *db, ubyte *addr, int port); extern (C) int nmdb_add_udp_server(nmdb_t *db, ubyte *addr, int port); +extern (C) int nmdb_add_sctp_server(nmdb_t *db, ubyte *addr, int port); extern (C) int nmdb_free(nmdb_t *db); extern (C) ptrdiff_t nmdb_get(nmdb_t *db, ubyte *key, size_t ksize, @@ -63,3 +64,8 @@ extern (C) int nmdb_cas(nmdb_t *db, ubyte *key, size_t ksize, extern (C) int nmdb_cache_cas(nmdb_t *db, ubyte *key, size_t ksize, ubyte *oldval, size_t ovsize, ubyte *newval, size_t nvsize); +extern (C) int nmdb_incr(nmdb_t *db, ubyte *key, size_t ksize, + long increment); +extern (C) int nmdb_cache_incr(nmdb_t *db, ubyte *key, size_t ksize, + long increment); + diff --git a/bindings/haskell/LICENSE b/bindings/haskell/LICENSE new file mode 100644 index 0000000..f3a9498 --- /dev/null +++ b/bindings/haskell/LICENSE @@ -0,0 +1,34 @@ + +I don't like licenses, because I don't like having to worry about all this +legal stuff just for a simple piece of software I don't really mind anyone +using. But I also believe that it's important that people share and give back; +so I'm placing this library under the following license, so you feel guilty if +you don't ;) + + +BOLA - Buena Onda License Agreement +----------------------------------- + +This work is provided 'as-is', without any express or implied warranty. In no +event will the authors be held liable for any damages arising from the use of +this work. + +To all effects and purposes, this work is to be considered Public Domain. + + +However, if you want to be "Buena onda", you should: + +1. Not take credit for it, and give proper recognition to the authors. +2. Share your modifications, so everybody benefits from them. +4. Do something nice for the authors. +5. Help someone who needs it: sign up for some volunteer work or help your + neighbour paint the house. +6. Don't waste. Anything, but specially energy that comes from natural + non-renewable resources. Extra points if you discover or invent something + to replace them. +7. Be tolerant. Everything that's good in nature comes from cooperation. + +The order is important, and the further you go the more "Buena onda" you are. +Make the world a better place: be "Buena onda". + + diff --git a/bindings/haskell/Nmdb.hs b/bindings/haskell/Nmdb.hs new file mode 100644 index 0000000..735385a --- /dev/null +++ b/bindings/haskell/Nmdb.hs @@ -0,0 +1,174 @@ + +-- Haskell bindings for the nmdb C library +-- Alberto Bertogli (albertito@gmail.com) + +module Nmdb ( + NmdbStruct, + nmdbInit, nmdbFree, + nmdbAddTIPCServer, nmdbAddTCPServer, nmdbAddUDPServer, + nmdbAddSCTPServer, + nmdbSet, nmdbSetSync, nmdbCacheSet, + nmdbGet, nmdbCacheGet, + nmdbDel, nmdbDelSync, nmdbCacheDel, + nmdbCAS, nmdbCacheCAS, + nmdbIncr, nmdbCacheIncr, +) where + +import Foreign.Ptr +import Foreign.C.Types +import Foreign.C.String +import Foreign.Marshal.Alloc + + +-- Opaque pointer to nmdb_t +data NmdbStruct = NmdbStruct +type NmdbPtr = Ptr NmdbStruct + + +-- DB creation and destroy +foreign import ccall "nmdb.h nmdb_init" llNmdbInit :: IO NmdbPtr +foreign import ccall "nmdb.h nmdb_free" llNmdbFree :: NmdbPtr -> IO () + +nmdbInit :: IO NmdbPtr +nmdbInit = llNmdbInit + +nmdbFree :: NmdbPtr -> IO () +nmdbFree = llNmdbFree + + +-- Adding servers +foreign import ccall "nmdb.h nmdb_add_tipc_server" llNmdbAddTIPCServer :: + NmdbPtr -> Int -> IO Int +foreign import ccall "nmdb.h nmdb_add_tcp_server" llNmdbAddTCPServer :: + NmdbPtr -> CString -> Int -> IO Int +foreign import ccall "nmdb.h nmdb_add_udp_server" llNmdbAddUDPServer :: + NmdbPtr -> CString -> Int -> IO Int +foreign import ccall "nmdb.h nmdb_add_sctp_server" llNmdbAddSCTPServer :: + NmdbPtr -> CString -> Int -> IO Int + +nmdbAddTIPCServer db port = do + r <- llNmdbAddTIPCServer db port + return r + +nmdbAddTCPServer db host port = do + hstr <- newCString host + r <- llNmdbAddTCPServer db hstr port + free hstr + return r + +nmdbAddUDPServer db host port = do + hstr <- newCString host + r <- llNmdbAddUDPServer db hstr port + free hstr + return r + +nmdbAddSCTPServer db host port = do + hstr <- newCString host + r <- llNmdbAddSCTPServer db hstr port + free hstr + return r + +-- Set functions +foreign import ccall "nmdb.h nmdb_set" llNmdbSet :: + NmdbPtr -> CString -> Int -> CString -> Int -> IO Int +foreign import ccall "nmdb.h nmdb_set_sync" llNmdbSetSync :: + NmdbPtr -> CString -> Int -> CString -> Int -> IO Int +foreign import ccall "nmdb.h nmdb_cache_set" llNmdbCacheSet :: + NmdbPtr -> CString -> Int -> CString -> Int -> IO Int + +nmdbGenericSet llfunc db key val = do + kl <- newCStringLen key + vl <- newCStringLen val + r <- llfunc db (fst kl) (snd kl) (fst vl) (snd vl) + free (fst kl) + free (fst vl) + return r + +nmdbSet = nmdbGenericSet llNmdbSet +nmdbSetSync = nmdbGenericSet llNmdbSetSync +nmdbCacheSet = nmdbGenericSet llNmdbCacheSet + + +-- Get functions +foreign import ccall "nmdb.h nmdb_get" llNmdbGet :: + NmdbPtr -> CString -> Int -> CString -> Int -> IO Int +foreign import ccall "nmdb.h nmdb_cache_get" llNmdbCacheGet :: + NmdbPtr -> CString -> Int -> CString -> Int -> IO Int + +nmdbGenericGet llfunc db key = do + let buflen = 64 * 1024 + buf <- mallocBytes buflen + kl <- newCStringLen key + r <- llfunc db (fst kl) (snd kl) buf buflen + free (fst kl) + if r < 0 + then do + free buf + return Nothing + else do + val <- peekCStringLen (buf, r) + free buf + return $ Just val + +nmdbGet = nmdbGenericGet llNmdbGet +nmdbCacheGet = nmdbGenericGet llNmdbCacheGet + + +-- Del functions +foreign import ccall "nmdb.h nmdb_del" llNmdbDel :: + NmdbPtr -> CString -> Int -> IO Int +foreign import ccall "nmdb.h nmdb_del_sync" llNmdbDelSync :: + NmdbPtr -> CString -> Int -> IO Int +foreign import ccall "nmdb.h nmdb_cache_del" llNmdbCacheDel :: + NmdbPtr -> CString -> Int -> IO Int + +nmdbGenericDel llfunc db key = do + kl <- newCStringLen key + r <- llfunc db (fst kl) (snd kl) + free (fst kl) + return r + +nmdbDel = nmdbGenericDel llNmdbDel +nmdbDelSync = nmdbGenericDel llNmdbDelSync +nmdbCacheDel = nmdbGenericDel llNmdbCacheDel + + +-- CAS functions +foreign import ccall "nmdb.h nmdb_cas" llNmdbCAS :: + NmdbPtr -> CString -> Int -> CString -> Int -> CString -> Int -> IO Int +foreign import ccall "nmdb.h nmdb_cache_cas" llNmdbCacheCAS :: + NmdbPtr -> CString -> Int -> CString -> Int -> CString -> Int -> IO Int + +nmdbGenericCAS llfunc db key oldval newval = do + kl <- newCStringLen key + ovl <- newCStringLen oldval + nvl <- newCStringLen newval + r <- llfunc db + (fst kl) (snd kl) + (fst ovl) (snd ovl) + (fst nvl) (snd nvl) + free (fst kl) + free (fst ovl) + free (fst nvl) + return r + +nmdbCAS = nmdbGenericCAS llNmdbCAS +nmdbCacheCAS = nmdbGenericCAS llNmdbCacheCAS + + +-- Incr functions +foreign import ccall "nmdb.h nmdb_incr" llNmdbIncr :: + NmdbPtr -> CString -> Int -> Int -> IO Int +foreign import ccall "nmdb.h nmdb_cache_incr" llNmdbCacheIncr :: + NmdbPtr -> CString -> Int -> Int -> IO Int + +nmdbGenericIncr llfunc db key increment = do + kl <- newCStringLen key + r <- llfunc db (fst kl) (snd kl) increment + free (fst kl) + return r + +nmdbIncr = nmdbGenericIncr llNmdbIncr +nmdbCacheIncr = nmdbGenericIncr llNmdbCacheIncr + + diff --git a/bindings/haskell/Setup.lhs b/bindings/haskell/Setup.lhs new file mode 100755 index 0000000..8193653 --- /dev/null +++ b/bindings/haskell/Setup.lhs @@ -0,0 +1,5 @@ +#!/usr/bin/env runhaskell + +> import Distribution.Simple +> main = defaultMain + diff --git a/bindings/haskell/nmdb.cabal b/bindings/haskell/nmdb.cabal new file mode 100644 index 0000000..5f1371d --- /dev/null +++ b/bindings/haskell/nmdb.cabal @@ -0,0 +1,22 @@ +Name: Nmdb +Version: 0.1 +License: PublicDomain +License-file: LICENSE +Author: Alberto Bertogli +Homepage: http://auriga.wearlab.de/~alb/nmdb/ +Category: Database +Build-Depends: base +Exposed-modules: Nmdb +Extra-libraries: nmdb +Extensions: ForeignFunctionInterface +Synopsis: nmdb Haskell bindings +Description: + nmdb is a network database (dbm-style) for controlled networks that can use + different protocols to to communicate with its clients. At the moment, it + supports TIPC, TCP and UDP. + It consists of an in-memory cache that saves (key, value) pairs, and a + persistent backend that stores the pairs on disk. Both work combined, but + the use of the backend is optional, so you can use the server only for cache + queries, pretty much like memcached. + This are the Haskell bindings for the C nmdb library. + diff --git a/bindings/haskell/test1.hs b/bindings/haskell/test1.hs new file mode 100644 index 0000000..f824a98 --- /dev/null +++ b/bindings/haskell/test1.hs @@ -0,0 +1,43 @@ + +-- Testing module for nmdb Haskell bindings. +-- Build with ghc --make test1.hs + +module Main where + +import Nmdb + +-- putStrLn + show, all in one +cshow desc f = do + r <- f + putStr desc + putStr " -> " + putStrLn $ show r + +main :: IO () +main = do + db <- nmdbInit + + cshow "Add TIPC" $ nmdbAddTIPCServer db (-1) + cshow "Add TCP" $ nmdbAddTCPServer db "localhost" (-1) + cshow "Add UDP" $ nmdbAddUDPServer db "localhost" (-1) + cshow "Add SCTP" $ nmdbAddSCTPServer db "localhost" (-1) + + cshow "Set 'Hello' 'Bye'" $ nmdbSet db "Hello" "Bye" + + cshow "Get 'Hello'" $ nmdbGet db "Hello" + cshow "Get 'XYZ'" $ nmdbGet db "XYZ" + + cshow "CAS 'Hello' 'Bye' 'Hey'" $ nmdbCAS db "Hello" "Bye" "Hey" + cshow "Get 'Hello'" $ nmdbGet db "Hello" + + cshow "Del 'Hello'" $ nmdbDel db "Hello" + + + cshow "Set 'Hello' '10\\0'" $ nmdbSet db "Hello" "10\0" + cshow "Incr 'Hello' 10" $ nmdbIncr db "Hello" 10 + cshow "Get 'Hello'" $ nmdbGet db "Hello" + + cshow "Free" $ nmdbFree db + + return () + diff --git a/bindings/newlisp/nmdb.lsp b/bindings/newlisp/nmdb.lsp index 187e1dc..640bfe7 100644 --- a/bindings/newlisp/nmdb.lsp +++ b/bindings/newlisp/nmdb.lsp @@ -1,6 +1,6 @@ ; -; nmdb bindings for newlisp (http://www.newlisp.org/) +; newlisp (http://www.newlisp.org/) bindings for nmdb ; Alberto Bertogli (albertito@gmail.com) ; ; Functions: @@ -14,13 +14,13 @@ ; (nmdb:db-get key) -> Gets the value associated to the given key, or -1. ; (nmdb:cache-get key) -> Like dbget but only get from the cache. ; -; (nmdb:db-set key val ) -> Sets the given key to the given value. -; (nmdb:sync-set key val ) -> Like db-set but synchronous. -; (nmdb:cache-set key val ) -> Like db-set but only set to the the cache. +; (nmdb:db-set key val) -> Sets the given key to the given value. +; (nmdb:sync-set key val) -> Like db-set but synchronous. +; (nmdb:cache-set key val) -> Like db-set but only set to the the cache. ; -; (nmdb:db-del key ) -> Removes the given key from the database. -; (nmdb:sync-del key ) -> Like db-del but synchronous. -; (nmdb:cache-del key ) -> Like db-del but only delete from the cache. +; (nmdb:db-del key) -> Removes the given key from the database. +; (nmdb:sync-del key) -> Like db-del but synchronous. +; (nmdb:cache-del key) -> Like db-del but only delete from the cache. ; ; ; Example: @@ -45,6 +45,7 @@ (import libnmdb "nmdb_add_tipc_server") (import libnmdb "nmdb_add_tcp_server") (import libnmdb "nmdb_add_udp_server") +(import libnmdb "nmdb_add_sctp_server") (import libnmdb "nmdb_free") (import libnmdb "nmdb_set") @@ -61,6 +62,9 @@ (import libnmdb "nmdb_cas") (import libnmdb "nmdb_cache_cas") +(import libnmdb "nmdb_incr") +(import libnmdb "nmdb_cache_incr") + ; main functions @@ -78,6 +82,9 @@ (define (add-udp-server addr port) (nmdb_add_udp_server NMDB addr port)) +(define (add-sctp-server addr port) + (nmdb_add_sctp_server NMDB addr port)) + (define (free) (nmdb_free NMDB)) @@ -90,7 +97,7 @@ (val (dup "\000" vallen)) ) (set 'rv (func NMDB key keylen val vallen)) - (if (> rv 0) + (if (>= rv 0) (slice val 0 rv) -1) ) ) @@ -133,6 +140,14 @@ (define (cache-cas key oval nval) (priv-cas nmdb_cache_cas key oval nval)) +; *-incr functions +(define (priv-incr func key increment) + (letn ( (keylen (length key)) ) + (func NMDB key keylen increment) ) ) + +(define (db-incr key increment) (priv-incr nmdb_incr key increment)) +(define (cache-incr key increment) (priv-incr nmdb_cache_incr key increment)) + (context MAIN) diff --git a/bindings/newlisp/test.lsp b/bindings/newlisp/test.lsp index 6a1c784..124567f 100644 --- a/bindings/newlisp/test.lsp +++ b/bindings/newlisp/test.lsp @@ -9,6 +9,7 @@ ;(println "add-tipc-server\t" (nmdb:add-tipc-server 12)) ;(println "add-tcp-server\t" (nmdb:add-tcp-server "127.0.0.1" -1)) ;(println "add-udp-server\t" (nmdb:add-udp-server "127.0.0.1" -1)) +;(println "add-sctp-server\t" (nmdb:add-sctp-server "127.0.0.1" -1)) (println) (println "db-set D1 V1\t" (nmdb:db-set "D1" "D1")) (println "sync-set S2 V2\t" (nmdb:sync-set "S2" "V2")) @@ -24,6 +25,10 @@ (println "db-del D1\t" (nmdb:db-del "D1")) (println "sync-del S2\t" (nmdb:sync-del "S2")) (println "cache-del C3\t" (nmdb:cache-del "C3")) - +(println) +(println "cache-set I 10\t" (nmdb:cache-set "I" "10\000")) +(println "cache-incr I 10\t" (nmdb:cache-incr "I" 10)) +(println "cache-get I\t" (nmdb:cache-get "I")) +(println) (exit) diff --git a/bindings/python/nmdb.py b/bindings/python/nmdb.py index 4d0dccc..38f7366 100644 --- a/bindings/python/nmdb.py +++ b/bindings/python/nmdb.py @@ -48,13 +48,9 @@ class NetworkError (Exception): pass -class _nmdbDict (object): - def __init__(self, db, op_get, op_set, op_delete, op_cas): - self._db = db - self._get = op_get - self._set = op_set - self._delete = op_delete - self._cas = op_cas +class GenericDB (object): + def __init__(self): + self._db = nmdb_ll.new() self.autopickle = True def add_tipc_server(self, port = -1): @@ -78,64 +74,78 @@ class _nmdbDict (object): raise NetworkError return rv - def __getitem__(self, key): + + def generic_get(self, getf, key): "d[k] Returns the value associated with the key k." if self.autopickle: key = str(hash(key)) try: - r = self._get(key) + r = getf(key) except: raise NetworkError - if not r: + if r == -1: + # For key errors, get returns -1 instead of a string + # so we know it's a miss. raise KeyError if self.autopickle: r = cPickle.loads(r) return r - def __setitem__(self, key, val): + def cache_get(self, key): + return self.generic_get(self._db.cache_get, key) + + def normal_get(self, key): + return self.generic_get(self._db.get, key) + + + def generic_set(self, setf, key, val): "d[k] = v Associates the value v to the key k." if self.autopickle: key = str(hash(key)) val = cPickle.dumps(val, protocol = -1) - r = self._set(key, val) + r = setf(key, val) if r <= 0: raise NetworkError return 1 - def __delitem__(self, key): + def cache_set(self, key, val): + return self.generic_set(self._db.cache_set, key, val) + + def normal_set(self, key, val): + return self.generic_set(self._db.set, key, val) + + def set_sync(self, key, val): + return self.generic_set(self._db.set_sync, key, val) + + + def generic_delete(self, delf, key): "del d[k] Deletes the key k." if self.autopickle: key = str(hash(key)) - r = self._delete(key) + r = delf(key) if r < 0: raise NetworkError elif r == 0: raise KeyError return 1 - def __contains__(self, key): - "Returns True if the key is in the database, False otherwise." - if self.autopickle: - key = str(hash(key)) - try: - r = self._get(key) - except KeyError: - return False - if not r: - return False - return True + def cache_delete(self, key): + return self.generic_delete(self._db.cache_delete, key) - def has_key(self, key): - "Returns True if the key is in the database, False otherwise." - return self.__contains__(key) + def normal_delete(self, key): + return self.generic_delete(self._db.delete, key) + + def delete_sync(self, key): + return self.generic_delete(self._db.delete_sync, key) - def cas(self, key, oldval, newval): + + def generic_cas(self, casf, key, oldval, newval): "Perform a compare-and-swap." if self.autopickle: key = str(hash(key)) oldval = cPickle.dumps(oldval, protocol = -1) newval = cPickle.dumps(newval, protocol = -1) - r = self._cas(key, oldval, newval) + r = casf(key, oldval, newval) if r == 2: # success return 2 @@ -148,22 +158,90 @@ class _nmdbDict (object): else: raise NetworkError + def cache_cas(self, key, oldv, newv): + return self.generic_cas(self._db.cache_cas, key, + oldval, newval) -class Cache (_nmdbDict): - def __init__(self): - db = nmdb_ll.connect() - _nmdbDict.__init__(self, db, db.cache_get, db.cache_set, - db.cache_delete, db.cache_cas) + def normal_cas(self, key, oldval, newval): + return self.generic_cas(self._db.cas, key, + oldval, newval) -class DB (_nmdbDict): - def __init__(self): - db = nmdb_ll.connect() - _nmdbDict.__init__(self, db, db.get, db.set, db.delete, db.cas) -class SyncDB (_nmdbDict): - def __init__(self): - db = nmdb_ll.connect() - _nmdbDict.__init__(self, db, db.get, db.set_sync, - db.delete_sync, db.cas) + def generic_incr(self, incrf, key, increment): + """Atomically increment the value associated with the given + key by the given increment.""" + if self.autopickle: + key = str(hash(key)) + r = incrf(key, increment) + if r == 2: + # success + return 2 + elif r == 1: + # no match, because the value didn't have the right + # format + raise TypeError, \ + "The value must be a NULL-terminated string" + elif r == 0: + # not in + raise KeyError + else: + raise NetworkError + + def cache_incr(self, key, increment = 1): + return self.generic_incr(self._db.cache_incr, key, increment) + + def normal_incr(self, key, increment = 1): + return self.generic_incr(self._db.incr, key, increment) + + + # The following functions will assume the existance of self.set, + # self.get, and self.delete, which are supposed to be set by our + # subclasses. + + def __getitem__(self, key): + return self.get(key) + + def __setitem__(self, key, val): + return self.set(key, val) + + def __delitem__(self, key): + return self.delete(key) + + def __contains__(self, key): + "Returns True if the key is in the database, False otherwise." + try: + r = self.get(key) + except KeyError: + return False + if not r: + return False + return True + + def has_key(self, key): + "Returns True if the key is in the database, False otherwise." + return self.__contains__(key) + + + +class Cache (GenericDB): + get = GenericDB.cache_get + set = GenericDB.cache_set + delete = GenericDB.cache_delete + cas = GenericDB.cache_cas + incr = GenericDB.cache_incr + +class DB (GenericDB): + get = GenericDB.normal_get + set = GenericDB.normal_set + delete = GenericDB.normal_delete + cas = GenericDB.normal_cas + incr = GenericDB.normal_incr + +class SyncDB (GenericDB): + get = GenericDB.normal_get + set = GenericDB.set_sync + delete = GenericDB.delete_sync + cas = GenericDB.normal_cas + incr = GenericDB.normal_incr diff --git a/bindings/python/nmdb_ll.c b/bindings/python/nmdb_ll.c index e37b22a..2374118 100644 --- a/bindings/python/nmdb_ll.c +++ b/bindings/python/nmdb_ll.c @@ -129,11 +129,12 @@ static PyObject *db_cache_get(nmdbobject *db, PyObject *args) rv = nmdb_cache_get(db->db, key, ksize, val, vsize); Py_END_ALLOW_THREADS - if (rv < 0) { + if (rv <= -2) { /* FIXME: define a better exception */ r = PyErr_SetFromErrno(PyExc_IOError); - } else if (rv == 0) { - r = PyString_FromStringAndSize("", 0); + } else if (rv == -1) { + /* Miss, handled in the high-level module. */ + r = PyLong_FromLong(-1); } else { r = PyString_FromStringAndSize(val, rv); } @@ -181,6 +182,26 @@ static PyObject *db_cache_cas(nmdbobject *db, PyObject *args) return PyLong_FromLong(rv); } +/* cache increment */ +static PyObject *db_cache_incr(nmdbobject *db, PyObject *args) +{ + unsigned char *key; + int ksize; + int rv; + long long int increment; + + if (!PyArg_ParseTuple(args, "s#L:cache_incr", &key, &ksize, + &increment)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + rv = nmdb_cache_incr(db->db, key, ksize, increment); + Py_END_ALLOW_THREADS + + return PyLong_FromLong(rv); +} + /* db set */ static PyObject *db_set(nmdbobject *db, PyObject *args) @@ -223,11 +244,12 @@ static PyObject *db_get(nmdbobject *db, PyObject *args) rv = nmdb_get(db->db, key, ksize, val, vsize); Py_END_ALLOW_THREADS - if (rv < 0) { + if (rv <= -2) { /* FIXME: define a better exception */ r = PyErr_SetFromErrno(PyExc_IOError); - } else if (rv == 0) { - r = PyString_FromStringAndSize("", 0); + } else if (rv == -1) { + /* Miss, handled in the high-level module. */ + r = PyLong_FromLong(-1); } else { r = PyString_FromStringAndSize(val, rv); } @@ -274,6 +296,25 @@ static PyObject *db_cas(nmdbobject *db, PyObject *args) return PyLong_FromLong(rv); } +/* db increment */ +static PyObject *db_incr(nmdbobject *db, PyObject *args) +{ + unsigned char *key; + int ksize; + int rv; + long long int increment; + + if (!PyArg_ParseTuple(args, "s#L:incr", &key, &ksize, &increment)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + rv = nmdb_incr(db->db, key, ksize, increment); + Py_END_ALLOW_THREADS + + return PyLong_FromLong(rv); +} + /* db set sync */ static PyObject *db_set_sync(nmdbobject *db, PyObject *args) @@ -327,10 +368,12 @@ static PyMethodDef nmdb_methods[] = { { "cache_get", (PyCFunction) db_cache_get, METH_VARARGS, NULL }, { "cache_delete", (PyCFunction) db_cache_delete, METH_VARARGS, NULL }, { "cache_cas", (PyCFunction) db_cache_cas, METH_VARARGS, NULL }, + { "cache_incr", (PyCFunction) db_cache_incr, METH_VARARGS, NULL }, { "set", (PyCFunction) db_set, METH_VARARGS, NULL }, { "get", (PyCFunction) db_get, METH_VARARGS, NULL }, { "delete", (PyCFunction) db_delete, METH_VARARGS, NULL }, { "cas", (PyCFunction) db_cas, METH_VARARGS, NULL }, + { "incr", (PyCFunction) db_incr, METH_VARARGS, NULL }, { "set_sync", (PyCFunction) db_set_sync, METH_VARARGS, NULL }, { "delete_sync", (PyCFunction) db_delete_sync, METH_VARARGS, NULL }, @@ -358,8 +401,8 @@ static PyTypeObject nmdbType = { * The module */ -/* connect, returns an nmdb object */ -static PyObject *db_connect(PyObject *self, PyObject *args) +/* new, returns an nmdb object */ +static PyObject *db_new(PyObject *self, PyObject *args) { nmdbobject *db; @@ -367,7 +410,7 @@ static PyObject *db_connect(PyObject *self, PyObject *args) if (db == NULL) return NULL; - if (!PyArg_ParseTuple(args, ":connect")) { + if (!PyArg_ParseTuple(args, ":new")) { return NULL; } @@ -386,7 +429,7 @@ static PyObject *db_connect(PyObject *self, PyObject *args) } static PyMethodDef nmdb_functions[] = { - { "connect", (PyCFunction) db_connect, METH_VARARGS, NULL }, + { "new", (PyCFunction) db_new, METH_VARARGS, NULL }, { NULL } }; diff --git a/bindings/ruby/LICENSE b/bindings/ruby/LICENSE new file mode 100644 index 0000000..f3a9498 --- /dev/null +++ b/bindings/ruby/LICENSE @@ -0,0 +1,34 @@ + +I don't like licenses, because I don't like having to worry about all this +legal stuff just for a simple piece of software I don't really mind anyone +using. But I also believe that it's important that people share and give back; +so I'm placing this library under the following license, so you feel guilty if +you don't ;) + + +BOLA - Buena Onda License Agreement +----------------------------------- + +This work is provided 'as-is', without any express or implied warranty. In no +event will the authors be held liable for any damages arising from the use of +this work. + +To all effects and purposes, this work is to be considered Public Domain. + + +However, if you want to be "Buena onda", you should: + +1. Not take credit for it, and give proper recognition to the authors. +2. Share your modifications, so everybody benefits from them. +4. Do something nice for the authors. +5. Help someone who needs it: sign up for some volunteer work or help your + neighbour paint the house. +6. Don't waste. Anything, but specially energy that comes from natural + non-renewable resources. Extra points if you discover or invent something + to replace them. +7. Be tolerant. Everything that's good in nature comes from cooperation. + +The order is important, and the further you go the more "Buena onda" you are. +Make the world a better place: be "Buena onda". + + diff --git a/bindings/ruby/extconf.rb b/bindings/ruby/extconf.rb new file mode 100644 index 0000000..2da19fb --- /dev/null +++ b/bindings/ruby/extconf.rb @@ -0,0 +1,13 @@ +#!/usr/bin/env ruby + +require 'mkmf' + +if have_library("nmdb", "nmdb_init") and have_header("nmdb.h") +then + dir_config("libnmdb") + + create_makefile("nmdb_ll") +else + puts "Can't find libnmdb" +end + diff --git a/bindings/ruby/nmdb.rb b/bindings/ruby/nmdb.rb new file mode 100644 index 0000000..09672af --- /dev/null +++ b/bindings/ruby/nmdb.rb @@ -0,0 +1,208 @@ + +module Nmdb + +require "nmdb_ll" + + +class NetworkException < StandardError +end + + +class GenericDB + attr_accessor :automarshal + attr_accessor :default + + def initialize(default = nil) + @db = Nmdb_ll::DB.new + @automarshal = true + @default = default + end + + def add_tipc_server(port = -1) + return @db.add_tipc_server(port) + end + + def add_tcp_server(host, port = 26010) + return @db.add_tcp_server(host, port) + end + + def add_udp_server(host, port = 26010) + return @db.add_udp_server(host, port) + end + + def add_sctp_server(host, port = 26010) + return @db.add_sctp_server(host, port) + end + + + def normal_set(key, val) + if @automarshal then + key = Marshal.dump(key) + val = Marshal.dump(val) + end + return @db.set(key, val) + end + + def set_sync(key, val) + if @automarshal then + key = Marshal.dump(key) + val = Marshal.dump(val) + end + return @db.set_sync(key, val) + end + + def cache_set(key, val) + if @automarshal then + key = Marshal.dump(key) + val = Marshal.dump(val) + end + return @db.cache_set(key, val) + end + + + def generic_get(gfunc, key) + if @automarshal then + key = Marshal.dump(key) + end + r = gfunc.call(key) + if r.class == String then + if @automarshal then + r = Marshal.load(r) + end + return r + elsif r == -1 then + # key not in the database + return @default + elsif r <= 2 then + raise NetworkException + else + # we should never get here + raise Exception + end + end + + def normal_get(key) + return generic_get(@db.method(:get), key) + end + + def cache_get(key) + return generic_get(@db.method(:cache_get), key) + end + + + def normal_delete(key) + if @automarshal then + key = Marshal.dump(key) + end + return @db.del(key) + end + + def cache_delete(key) + if @automarshal then + key = Marshal.dump(key) + end + return @db.cache_del(key) + end + + def delete_sync(key) + if @automarshal then + key = Marshal.dump(key) + end + return @db.del_sync(key) + end + + + def generic_cas(gfunc, key, old, new) + if @automarshal then + key = Marshal.dump(key) + old = Marshal.dump(old) + new = Marshal.dump(new) + end + r = gfunc.call(key, old, new) + if r == 0 then + # key not in the database + return nil + elsif r < 0 then + raise NetworkException + else + return r + end + end + + def normal_cas(key, old, new) + return generic_cas(@db.method(:cas), key, old, new) + end + + def cache_cas(key, old, new) + return generic_cas(@db.method(:cache_cas), key, old, new) + end + + + def generic_incr(gfunc, key, increment) + if @automarshal then + key = Marshal.dump(key) + end + r = gfunc.call(key, increment) + if r == 0 then + # key not in the database + return nil + elsif r == 1 or r < 0 then + raise NetworkException + else + return r + end + end + + def normal_incr(key, increment) + return generic_incr(@db.method(:incr), key, increment) + end + + def cache_incr(key, increment) + return generic_incr(@db.method(:cache_incr), key, increment) + end + + + # The following functions asume we have set(), get(), delete() and + # cas(), which are supposed to be implemented by our subclasses + + def include?(key) + # we assume we have get() implemented + return get(key) == @default + end + + def []=(key, val) + return set(key, val) + end + + def [](key) + return get(key) + end +end + + +class DB < GenericDB + alias set normal_set + alias get normal_get + alias delete normal_delete + alias cas normal_cas + alias incr normal_incr +end + +class Cache < GenericDB + alias set cache_set + alias get cache_get + alias delete cache_delete + alias cas cache_cas + alias incr cache_cas +end + +class Sync < GenericDB + alias set set_sync + alias get normal_get + alias delete delete_sync + alias cas normal_cas + alias incr normal_cas +end + +end + diff --git a/bindings/ruby/nmdb_ll.c b/bindings/ruby/nmdb_ll.c new file mode 100644 index 0000000..1ef13cc --- /dev/null +++ b/bindings/ruby/nmdb_ll.c @@ -0,0 +1,262 @@ + +#include "nmdb.h" +#include + +static VALUE rb_mnmdb_ll; +static VALUE rb_cDB; + + +/* Functions for ruby memory management. + * The "mm" prefix is used by us (not enforced by Ruby) to distinguish it from + * the rest. */ + +static void mm_db_mark(nmdb_t *db) +{ +} + +static void mm_db_free(nmdb_t *db) +{ + nmdb_free(db); +} + +static VALUE mm_db_allocate(VALUE class) +{ + nmdb_t *db = nmdb_init(); + + return Data_Wrap_Struct(class, mm_db_mark, mm_db_free, db); +} + + +/* + * DB methods + */ + +VALUE m_add_tipc_server(VALUE self, VALUE port) +{ + nmdb_t *db; + Data_Get_Struct(self, nmdb_t, db); + + int rv = nmdb_add_tipc_server(db, NUM2INT(port)); + return INT2NUM(rv); +} + +VALUE m_add_tcp_server(VALUE self, VALUE hostname, VALUE port) +{ + nmdb_t *db; + Data_Get_Struct(self, nmdb_t, db); + + int rv = nmdb_add_tcp_server(db, StringValuePtr(hostname), + NUM2INT(port)); + return INT2NUM(rv); +} + +VALUE m_add_udp_server(VALUE self, VALUE hostname, VALUE port) +{ + nmdb_t *db; + Data_Get_Struct(self, nmdb_t, db); + + int rv = nmdb_add_udp_server(db, StringValuePtr(hostname), + NUM2INT(port)); + return INT2NUM(rv); +} + +VALUE m_add_sctp_server(VALUE self, VALUE hostname, VALUE port) +{ + nmdb_t *db; + Data_Get_Struct(self, nmdb_t, db); + + int rv = nmdb_add_sctp_server(db, StringValuePtr(hostname), + NUM2INT(port)); + return INT2NUM(rv); +} + + +/* Set functions */ +typedef int (*setf_t) (nmdb_t *db, + const unsigned char *k, size_t ks, + const unsigned char *v, size_t vs); +static VALUE generic_set(VALUE self, VALUE key, VALUE val, setf_t set_func) +{ + int rv; + unsigned char *k, *v; + size_t ksize, vsize; + nmdb_t *db; + Data_Get_Struct(self, nmdb_t, db); + + k = rb_str2cstr(key, &ksize); + v = rb_str2cstr(val, &vsize); + + rv = set_func(db, k, ksize, v, vsize); + return INT2NUM(rv); +} + +static VALUE m_set(VALUE self, VALUE key, VALUE val) { + return generic_set(self, key, val, nmdb_set); +} + +static VALUE m_set_sync(VALUE self, VALUE key, VALUE val) { + return generic_set(self, key, val, nmdb_set_sync); +} + +static VALUE m_cache_set(VALUE self, VALUE key, VALUE val) { + return generic_set(self, key, val, nmdb_cache_set); +} + + +/* Get functions */ +typedef ssize_t (*getf_t) (nmdb_t *db, + const unsigned char *k, size_t ks, + unsigned char *v, size_t vs); +VALUE generic_get(VALUE self, VALUE key, getf_t get_func) +{ + ssize_t rv; + unsigned char *k, *v; + size_t ksize, vsize; + VALUE retval; + nmdb_t *db; + Data_Get_Struct(self, nmdb_t, db); + + k = rb_str2cstr(key, &ksize); + + v = ALLOC_N(char, (68 * 1024)); + + rv = get_func(db, k, ksize, v, (68 * 1024)); + if (rv >= 0) + retval = rb_str_new(v, rv); + else + /* The high level module knows and checks for this, there's no + * need to make the C code more complex by raising an + * exception. */ + retval = INT2NUM(rv); + + free(v); + return retval; +} + +VALUE m_get(VALUE self, VALUE key) { + return generic_get(self, key, nmdb_get); +} + +VALUE m_cache_get(VALUE self, VALUE key) { + return generic_get(self, key, nmdb_cache_get); +} + + +/* Del functions */ +typedef int (*delf_t) (nmdb_t *db, const unsigned char *k, size_t ks); +VALUE generic_del(VALUE self, VALUE key, delf_t del_func) +{ + ssize_t rv; + unsigned char *k; + size_t ksize; + nmdb_t *db; + Data_Get_Struct(self, nmdb_t, db); + + k = rb_str2cstr(key, &ksize); + + rv = del_func(db, k, ksize); + return INT2NUM(rv); +} + +VALUE m_del(VALUE self, VALUE key) { + return generic_del(self, key, nmdb_del); +} + +VALUE m_del_sync(VALUE self, VALUE key) { + return generic_del(self, key, nmdb_del_sync); +} + +VALUE m_cache_del(VALUE self, VALUE key) { + return generic_del(self, key, nmdb_cache_del); +} + + +/* CAS functions */ +typedef int (*casf_t) (nmdb_t *db, + const unsigned char *k, size_t ks, + const unsigned char *ov, size_t ovs, + const unsigned char *nv, size_t nvs); +static VALUE generic_cas(VALUE self, VALUE key, VALUE oldval, VALUE newval, + casf_t cas_func) +{ + int rv; + unsigned char *k, *ov, *nv; + size_t ksize, ovsize, nvsize; + nmdb_t *db; + Data_Get_Struct(self, nmdb_t, db); + + k = rb_str2cstr(key, &ksize); + ov = rb_str2cstr(oldval, &ovsize); + nv = rb_str2cstr(newval, &nvsize); + + rv = cas_func(db, k, ksize, ov, ovsize, nv, nvsize); + return INT2NUM(rv); +} + +static VALUE m_cas(VALUE self, VALUE key, VALUE oldval, VALUE newval) { + return generic_cas(self, key, oldval, newval, nmdb_cas); +} + +static VALUE m_cache_cas(VALUE self, VALUE key, VALUE oldval, VALUE newval) { + return generic_cas(self, key, oldval, newval, nmdb_cache_cas); +} + + +/* Del functions */ +typedef int (*incrf_t) (nmdb_t *db, const unsigned char *k, size_t ks, + int64_t increment); +VALUE generic_incr(VALUE self, VALUE key, VALUE increment, incrf_t incr_func) +{ + ssize_t rv; + unsigned char *k; + size_t ksize; + int64_t cincr; + nmdb_t *db; + Data_Get_Struct(self, nmdb_t, db); + + k = rb_str2cstr(key, &ksize); + cincr = rb_num2ll(increment); + + rv = incr_func(db, k, ksize, cincr); + return INT2NUM(rv); +} + +VALUE m_incr(VALUE self, VALUE key, VALUE increment) { + return generic_incr(self, key, increment, nmdb_incr); +} + +VALUE m_cache_incr(VALUE self, VALUE key, VALUE increment) { + return generic_incr(self, key, increment, nmdb_cache_incr); +} + + +/* Module initialization */ +void Init_nmdb_ll() +{ + rb_mnmdb_ll = rb_define_module("Nmdb_ll"); + rb_cDB = rb_define_class_under(rb_mnmdb_ll, "DB", rb_cObject); + rb_define_alloc_func(rb_cDB, mm_db_allocate); + + rb_define_method(rb_cDB, "add_tipc_server", m_add_tipc_server, 1); + rb_define_method(rb_cDB, "add_tcp_server", m_add_tcp_server, 2); + rb_define_method(rb_cDB, "add_udp_server", m_add_udp_server, 2); + rb_define_method(rb_cDB, "add_sctp_server", m_add_sctp_server, 2); + + rb_define_method(rb_cDB, "set", m_set, 2); + rb_define_method(rb_cDB, "set_sync", m_set_sync, 2); + rb_define_method(rb_cDB, "cache_set", m_cache_set, 2); + + rb_define_method(rb_cDB, "get", m_get, 1); + rb_define_method(rb_cDB, "cache_get", m_cache_get, 1); + + rb_define_method(rb_cDB, "del", m_del, 1); + rb_define_method(rb_cDB, "del_sync", m_del_sync, 1); + rb_define_method(rb_cDB, "cache_del", m_cache_del, 1); + + rb_define_method(rb_cDB, "cas", m_cas, 3); + rb_define_method(rb_cDB, "cache_cas", m_cache_cas, 3); + + rb_define_method(rb_cDB, "incr", m_incr, 2); + rb_define_method(rb_cDB, "cache_incr", m_cache_incr, 2); +} + diff --git a/bindings/ruby/test1.rb b/bindings/ruby/test1.rb new file mode 100644 index 0000000..b5a0641 --- /dev/null +++ b/bindings/ruby/test1.rb @@ -0,0 +1,18 @@ +#!/usr/bin/env ruby + +require 'nmdb' + +db = Nmdb::DB.new +db.add_tipc_server() +db.add_udp_server('localhost') +db.add_tcp_server('localhost') +db.add_sctp_server('localhost') + +puts 'set 1 <- 2'; db[1] = 2 +puts 'get 1: ' + db[1].to_s + +db.automarshal = false +puts 'set I <- 10\0'; db['I'] = "10\0" +puts 'incr I 10: ' + db.incr('I', 10).to_s +puts 'get I: ' + db['I'].to_s + diff --git a/doc/design.rst b/doc/design.rst index 105370c..ddb60b9 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -21,8 +21,8 @@ Network interface ================= The server communicates with its clients using messages, which can be -delivered through TIPC_, TCP or UDP. Messages are limited by design to 64k so -they stay inside within TIPC_'s limits. +delivered through TIPC_, TCP, UDP or SCTP. Messages are limited by design to +64k so they stay inside within TIPC_'s limits. TIPC_ is completely connectionless, and uses the reliable datagram layer provided by TIPC_. The network protocol is specified in another document, and @@ -41,6 +41,13 @@ get *key* Retrieves the value for the given key. If the key is in the cache, it returns immediately. If not, it performs a query in the database. +cas *key* *oldvalue* *newvalue* + Do a compare-and-swap, using *oldvalue* to compare with the value stored in + the database, and replacing it with *newvalue* if they match. + +incr *key* *increment* + Increments the value associated to the given key by the given increment. + set_async *key* *value* Stores the *(key, value)* pair in the database. It does the set in the cache, queues the operation for the database, and returns. @@ -66,8 +73,10 @@ cache_del *key* Like *del*, but only affects the cache and not the database. cache_cas *key* *oldvalue* *newvalue* - Do a compare-and-swap, using *oldvalue* to compare with the value stored in - the database, and replacing it with *newvalue* if they match. + Like *cas*, but only affects the cache and not the database. + +cache_incr *key* *increment* + Like *incr*, but only affects the cache and not the database. As you can see, it's possible to operate exclusively with the cache, ignoring the database completely. This is very similar to what memcached_ does. Note diff --git a/doc/guide.rst b/doc/guide.rst index 4431722..ed8fe17 100644 --- a/doc/guide.rst +++ b/doc/guide.rst @@ -8,10 +8,11 @@ nmdb User Guide Introduction ============ -nmdb_ is a simple and fast cache and database for TIPC_, TCP and UDP clusters. -It allows applications in the cluster to use a centralized, shared cache and +nmdb_ is a simple and fast cache and database for controlled networks. +It allows applications in the network to use a centralized, shared cache and database in a very easy way. It stores *(key, value)* pairs, with each key -having only one associated value. +having only one associated value. At the moment, it supports the TIPC_, TCP, +UDP and SCTP protocols. This document explains how to setup nmdb and a simple guide to writing clients. It also includes a "quick start" section for the anxious. @@ -87,7 +88,7 @@ TIPC setup If you want to use the server and the clients in different machines using TIPC, you need to setup your TIPC network. If you just want to run everything in one machine, you already have a TIPC network set up, or you only want to -use TCP or UDP connections, you can skip this section. +use TCP, UDP or SCTP connections, you can skip this section. Before we begin, all the machines should already be connected in an Ethernet LAN, and have the tipc-config application that should come with your Linux @@ -406,10 +407,10 @@ The best place to go from here is to your text editor, to start writing some simple clients to play with. If you are in doubt about something, you can consult the manpages or the -documentation inside the *doc/* directory. And if you can't find an answer to -your question there, you can ask me, Alberto Bertogli, at -*albertito@gmail.com*. +documentation inside the *doc/* directory. +To get in touch with nmdb developers and users, you can join the `mailing +list`_, or just send an email to nmdb-devel@lists.auriga.wearlab.de. .. _nmdb: http://auriga.wearlab.de/~alb/nmdb/ @@ -419,4 +420,7 @@ your question there, you can ask me, Alberto Bertogli, at .. _QDBM: http://qdbm.sf.net .. _`Linux kernel`: http://kernel.org .. _tetrations: http://en.wikipedia.org/wiki/Tetration +.. _`mailing list`: + http://lists.auriga.wearlab.de/cgi-bin/mailman/listinfo/nmdb-devel + diff --git a/doc/network.rst b/doc/network.rst index 7df5cae..c1e9909 100644 --- a/doc/network.rst +++ b/doc/network.rst @@ -7,10 +7,11 @@ nmdb_ Network Protocol **NOTE:** All integers are in network byte order. The nmdb network protocol relies on a message passing underlying transport -protocol. It normally uses TIPC, but can use UDP, or TCP with a messaging -layer too. This document describes the protocol in a transport-independent -way, assuming the transport protocol can send and receive messages reliably -and preserve message boundaries. No ordering guarantees are required. +protocol. It can be used on top of TIPC, UDP, TCP (with a thin messaging +layer) or SCTP. This document describes the protocol in a +transport-independent way, assuming the transport protocol can send and +receive messages reliably and preserve message boundaries. No ordering +guarantees are required. Requests @@ -58,7 +59,9 @@ REQ_DEL_SYNC 0x106 REQ_SET_ASYNC 0x107 REQ_DEL_ASYNC 0x108 REQ_CACHE_CAS 0x109 -REQ__CAS 0x110 +REQ_CAS 0x110 +REQ_CACHE_INCR 0x111 +REQ_INCR 0x112 ============== ====== @@ -78,6 +81,9 @@ REQ_DEL_* and REQ_CACHE_DEL REQ_CAS and REQ_CACHE_CAS First the key size, then the old value size, then the new value size, and then the key, the old value and the new value. +REQ_INCR and REQ_CACHE_INCR + First the key size (32 bits), then the key, and then the increment as a + signed network byte order 64 bit integer. Replies diff --git a/libnmdb/Makefile b/libnmdb/Makefile index 9206f10..e0e0cfe 100644 --- a/libnmdb/Makefile +++ b/libnmdb/Makefile @@ -2,12 +2,14 @@ ENABLE_TIPC = 1 ENABLE_TCP = 1 ENABLE_UDP = 1 +ENABLE_SCTP = 1 -CFLAGS += -std=c99 -Wall -O3 +CFLAGS += -std=c99 -pedantic -Wall -O3 ALL_CFLAGS = -D_XOPEN_SOURCE=500 -fPIC $(CFLAGS) ALL_CFLAGS += -DENABLE_TIPC=$(ENABLE_TIPC) \ -DENABLE_TCP=$(ENABLE_TCP) \ - -DENABLE_UDP=$(ENABLE_UDP) + -DENABLE_UDP=$(ENABLE_UDP) \ + -DENABLE_SCTP=$(ENABLE_SCTP) ifdef DEBUG ALL_CFLAGS += -g @@ -17,51 +19,60 @@ ifdef PROFILE ALL_CFLAGS += -g -pg -fprofile-arcs -ftest-coverage endif -ifdef STRICT -ALL_CFLAGS += -ansi -pedantic -endif # prefix for installing the binaries PREFIX=/usr/local -OBJS = libnmdb.o tcp.o tipc.o udp.o +OBJS = libnmdb.o tcp.o tipc.o udp.o sctp.o default: all all: libs +nmdb.h: nmdb.skel.h + @echo "generating nmdb.h" + @cat nmdb.skel.h | \ + sed 's/++CONFIG_ENABLE_TIPC++/$(ENABLE_TIPC)/g' | \ + sed 's/++CONFIG_ENABLE_TCP++/$(ENABLE_TCP)/g' | \ + sed 's/++CONFIG_ENABLE_UDP++/$(ENABLE_UDP)/g' | \ + sed 's/++CONFIG_ENABLE_SCTP++/$(ENABLE_SCTP)/g' \ + > nmdb.h libs: libnmdb.so libnmdb.a -libnmdb.so: $(OBJS) +libnmdb.so: nmdb.h $(OBJS) $(CC) $(ALL_CFLAGS) -shared -fPIC $(OBJS) -o libnmdb.so -libnmdb.a: $(OBJS) +libnmdb.a: nmdb.h $(OBJS) $(AR) cr libnmdb.a $(OBJS) -install: libs +install-lib: libs install -d $(PREFIX)/lib install -m 0755 libnmdb.so $(PREFIX)/lib install -m 0755 libnmdb.a $(PREFIX)/lib install -d $(PREFIX)/include install -m 0644 nmdb.h $(PREFIX)/include - install -d $(PREFIX)/man/man3 - install -m 0644 libnmdb.3 $(PREFIX)/man/man3/ @echo @echo "Please run ldconfig to update your library cache" @echo +install-man: + install -d $(PREFIX)/man/man3 + install -m 0644 libnmdb.3 $(PREFIX)/man/man3/ + +install: install-lib install-man + .c.o: $(CC) $(ALL_CFLAGS) -c $< -o $@ clean: - rm -f $(OBJS) libnmdb.so libnmdb.a + rm -f nmdb.h $(OBJS) libnmdb.so libnmdb.a rm -f *.bb *.bbg *.da *.gcov *.gcda *.gcno gmon.out -.PHONY: default all libs install clean +.PHONY: default all libs install-lib install-man install clean diff --git a/libnmdb/internal.h b/libnmdb/internal.h index 4c2f93c..ce52cee 100644 --- a/libnmdb/internal.h +++ b/libnmdb/internal.h @@ -2,11 +2,12 @@ #ifndef _INTERNAL_H #define _INTERNAL_H -/* Different connection types. Used internally to differentiate between TIPC - * and TCP connections in struct nmdb_srv. */ +/* Different connection types. Used internally to differentiate between the + * different protocols in struct nmdb_srv. */ #define TIPC_CONN 1 #define TCP_CONN 2 #define UDP_CONN 3 +#define SCTP_CONN 4 /* The ID code for requests is hardcoded for now, until asynchronous requests * * are implemented. */ @@ -17,6 +18,7 @@ #define TIPC_MSG_OFFSET 0 #define TCP_MSG_OFFSET 4 #define UDP_MSG_OFFSET 0 +#define SCTP_MSG_OFFSET 0 /* Functions used internally but shared among the different files. */ int compare_servers(const void *s1, const void *s2); diff --git a/libnmdb/libnmdb.3 b/libnmdb/libnmdb.3 index 72cb400..6d23c50 100644 --- a/libnmdb/libnmdb.3 +++ b/libnmdb/libnmdb.3 @@ -9,6 +9,7 @@ libnmdb - Library for interacting with a nmdb server .BI "int nmdb_add_tipc_server(nmdb_t *" db ", int " port ");" .BI "int nmdb_add_tcp_server(nmdb_t *" db ", const char * " addr ", int " port ");" .BI "int nmdb_add_udp_server(nmdb_t *" db ", const char * " addr ", int " port ");" +.BI "int nmdb_add_sctp_server(nmdb_t *" db ", const char * " addr ", int " port ");" .BI "int nmdb_free(nmdb_t *" db ");" .sp .BI "int nmdb_set(nmdb_t *" db "," @@ -21,10 +22,10 @@ libnmdb - Library for interacting with a nmdb server .BI " const unsigned char *" key ", size_t " ksize "," .BI " const unsigned char *" val ", size_t " vsize ");" .sp -.BI "int nmdb_get(nmdb_t *" db "," +.BI "ssize_t nmdb_get(nmdb_t *" db "," .BI " const unsigned char *" key ", size_t " ksize "," .BI " unsigned char *" val ", size_t " vsize ");" -.BI "int nmdb_cache_get(nmdb_t *" db "," +.BI "ssize_t nmdb_cache_get(nmdb_t *" db "," .BI " const unsigned char *" key ", size_t " ksize "," .BI " unsigned char *" val ", size_t " vsize ");" .sp @@ -38,11 +39,18 @@ libnmdb - Library for interacting with a nmdb server .BI "int nmdb_cas(nmdb_t *" db "," .BI " const unsigned char *" key " , size_t " ksize "," .BI " const unsigned char *" oldval ", size_t " ovsize "," -.BI " const unsigned char *" newval ", size_t " nvsize ")" +.BI " const unsigned char *" newval ", size_t " nvsize ");" .BI "int nmdb_cache_cas(nmdb_t *" db "," .BI " const unsigned char *" key " , size_t " ksize "," .BI " const unsigned char *" oldval ", size_t " ovsize "," -.BI " const unsigned char *" newval ", size_t " nvsize ")" +.BI " const unsigned char *" newval ", size_t " nvsize ");" +.sp +.BI "int nmdb_incr(nmdb_t *" db "," +.BI " const unsigned char *" key " , size_t " ksize "," +.BI " int64_t " increment ");" +.BI "int nmdb_cache_incr(nmdb_t *" db "," +.BI " const unsigned char *" key " , size_t " ksize "," +.BI " int64_t " increment ");" .fi .SH DESCRIPTION @@ -62,9 +70,10 @@ You can add any number of servers, and each time a request is made, one will be selected. Be aware that you should add all the servers before start using the database. For .BR nmdb_add_tipc_server() , -.B nmdb_add_tcp_server() +.BR nmdb_add_tcp_server() , +.B nmdb_add_udp_server() and -.BR nmdb_add_udp_server() , +.BR nmdb_add_sctp_server() , if you pass -1 as the port, it will select the default one. They return 1 on success or 0 on error (or if the specified protocol was not compiled in). @@ -86,17 +95,19 @@ of the keys and values apply: keys can't exceed 64Kb, values can't exceed 64Kb, and the size of a key + the size of it's associated value can't exceed 64Kb. -There are four kinds of operations: +There are five kinds of operations: .IR set , which sets pairs; .IR get , which gets pairs; .IR del , which removes pairs; -and .IR cas , -which compares-and-sets values. The normal set and del operations return as -soon as they've been queued on the server for asynchronous completion. Note +which compares-and-sets values; +and +.IR incr , +which atomically increments a value. The normal set and del operations return +as soon as they've been queued on the server for asynchronous completion. Note that in this case no message is sent to the client when the operation completes. @@ -125,8 +136,8 @@ greater than 64kb in size to make room for the largest possible value. It will return the size of the retrieved key (which will be put in the buffer pointed at by .IR val ), -0 if the requested key was not in the database (or cache, if the cache variant -is used), or < 0 on failure. +-1 if the requested key was not in the database (or cache, if the cache +variant is used), or -2 on failure. .BR nmdb_del () is used to remove a given key (and it's associated value). The normal variant @@ -138,7 +149,15 @@ was not in the database/cache, or < 0 on failure. is used to compare-and-swap a key's value. It takes an old value to compare with the one in the database, and if they match, it sets the key to the given new value. Returns 2 if the swap was performed, 1 if the values didn't match, -0 if the key was not on in the database/cache, and < 0 on failure. +0 if the key was not on in the database/cache, or < 0 on failure. + +.BR nmdb_incr () +is used to atomically increment (or decrement) a key's value. It takes a 64 +bit number to add to the value (it can be negative), and returns 2 if the +increment was performed, 1 if the value was not in the appropriate format, 0 +if the key was not in the database/cache, or < 0 on failure. +The value MUST be a NULL-terminated string with only a number in base 10 in it +(i.e. it must be parseable by strtoll(3)). .SH SEE ALSO @@ -147,3 +166,10 @@ new value. Returns 2 if the swap was performed, 1 if the values didn't match, (http://tipc.sf.net). .SH AUTHORS Created by Alberto Bertogli (albertito@gmail.com). + +.SH CONTACT + +To get in touch with developers and users, join the mailing list at +http://lists.auriga.wearlab.de/cgi-bin/mailman/listinfo/nmdb-devel or just +send an email to nmdb-devel@lists.auriga.wearlab.de. + diff --git a/libnmdb/libnmdb.c b/libnmdb/libnmdb.c index f43dd10..eb2a1d7 100644 --- a/libnmdb/libnmdb.c +++ b/libnmdb/libnmdb.c @@ -12,6 +12,7 @@ #include "tipc.h" #include "tcp.h" #include "udp.h" +#include "sctp.h" #include "internal.h" @@ -39,18 +40,20 @@ int compare_servers(const void *s1, const void *s2) return 1; } #endif -#if ENABLE_TCP || ENABLE_UDP - if (srv1->type == TCP_CONN || srv1->type == UDP_CONN) { +#if ENABLE_TCP || ENABLE_UDP || ENABLE_SCTP + if (srv1->type == TCP_CONN + || srv1->type == UDP_CONN + || srv1->type == SCTP_CONN) { in_addr_t a1, a2; - a1 = srv1->info.tcp.srvsa.sin_addr.s_addr; - a2 = srv2->info.tcp.srvsa.sin_addr.s_addr; + a1 = srv1->info.in.srvsa.sin_addr.s_addr; + a2 = srv2->info.in.srvsa.sin_addr.s_addr; if (a1 < a2) { return -1; } else if (a1 == a2) { in_port_t p1, p2; - p1 = srv1->info.tcp.srvsa.sin_port; - p2 = srv2->info.tcp.srvsa.sin_port; + p1 = srv1->info.in.srvsa.sin_port; + p2 = srv2->info.in.srvsa.sin_port; if (p1 < p2) return -1; @@ -159,6 +162,8 @@ static int srv_send(struct nmdb_srv *srv, return tcp_srv_send(srv, buf, bsize); case UDP_CONN: return udp_srv_send(srv, buf, bsize); + case SCTP_CONN: + return sctp_srv_send(srv, buf, bsize); default: return 0; } @@ -178,6 +183,8 @@ static uint32_t get_rep(struct nmdb_srv *srv, return tcp_get_rep(srv, buf, bsize, payload, psize); case UDP_CONN: return udp_get_rep(srv, buf, bsize, payload, psize); + case SCTP_CONN: + return sctp_get_rep(srv, buf, bsize, payload, psize); default: return 0; } @@ -195,6 +202,8 @@ static int srv_get_msg_offset(struct nmdb_srv *srv) return TCP_MSG_OFFSET; case UDP_CONN: return UDP_MSG_OFFSET; + case SCTP_CONN: + return SCTP_MSG_OFFSET; default: return 0; } @@ -278,21 +287,21 @@ static ssize_t do_get(nmdb_t *db, t = srv_send(srv, buf, moff + reqsize); if (t <= 0) { - rv = -1; + rv = -2; goto exit; } reply = get_rep(srv, buf, bsize, &p, &psize); if (reply == REP_CACHE_MISS || reply == REP_NOTIN) { - rv = 0; + rv = -1; goto exit; } else if (reply == REP_ERR) { - rv = -1; + rv = -2; goto exit; } else if (reply != REP_OK && reply != REP_CACHE_HIT) { /* invalid response */ - rv = -1; + rv = -2; goto exit; } @@ -300,7 +309,9 @@ static ssize_t do_get(nmdb_t *db, rv = * (uint32_t *) p; rv = ntohl(rv); if (rv > (psize - 4) || rv > vsize) { - rv = 0; + /* the value is too big for the packet size, or it is too big + * to fit in the buffer we were given */ + rv = -2; goto exit; } memcpy(val, p + 4, rv); @@ -575,3 +586,110 @@ int nmdb_cache_cas(nmdb_t *db, const unsigned char *key, size_t ksize, return do_cas(db, key, ksize, oldval, ovsize, newval, nvsize, 0); } + +/* ntohll() is not standard, so we define it using an UGLY trick because there + * is no standard way to check for endianness at runtime! (same problem as the + * one in nmdb/parse.c) */ +static uint64_t htonll(uint64_t x) +{ + static int endianness = 0; + + /* determine the endianness by checking how htonl() behaves; use -1 + * for little endian and 1 for big endian */ + if (endianness == 0) { + if (htonl(1) == 1) + endianness = 1; + else + endianness = -1; + } + + if (endianness == 1) { + /* big endian */ + return x; + } + + /* little endian */ + return ( htonl( (x >> 32) & 0xFFFFFFFF ) | \ + ( (uint64_t) htonl(x & 0xFFFFFFFF) ) << 32 ); +} + +static int do_incr(nmdb_t *db, const unsigned char *key, size_t ksize, + int64_t increment, int impact_db) +{ + int moff; + ssize_t rv, t; + unsigned char *buf, *p; + size_t bsize; + uint32_t request, reply; + struct nmdb_srv *srv; + + if (impact_db) + request = REQ_INCR; + else + request = REQ_CACHE_INCR; + + srv = select_srv(db, key, ksize); + moff = srv_get_msg_offset(srv); + + /* Use the same buffer for the request and the reply. + * Request: 4 bytes ver+id, 4 bytes request code, 4 bytes ksize, + * ksize bytes key, 8 bytes increment. + * Reply: 4 bytes id, 4 bytes reply code. + */ + bsize = moff + 4 + 4 + 4 + ksize + 8; + buf = malloc(bsize); + if (buf == NULL) + return -1; + + increment = htonll(increment); + + p = buf + moff; + + * (uint32_t *) p = htonl( (PROTO_VER << 28) | ID_CODE ); + * ((uint32_t *) p + 1) = htonl(request); + * ((uint32_t *) p + 2) = htonl(ksize); + memcpy(p + 3 * 4, key, ksize); + memcpy(p + 3 * 4 + ksize, &increment, sizeof(int64_t)); + + t = srv_send(srv, buf, bsize); + if (t <= 0) { + rv = -1; + goto exit; + } + + reply = get_rep(srv, buf, bsize, NULL, NULL); + + switch (reply) { + case REP_OK: + rv = 2; + break; + case REP_NOMATCH: + rv = 1; + break; + case REP_NOTIN: + rv = 0; + break; + default: + rv = -1; + break; + } + +exit: + free(buf); + return rv; + +} + +int nmdb_incr(nmdb_t *db, const unsigned char *key, size_t ksize, + int64_t increment) +{ + return do_incr(db, key, ksize, increment, 1); +} + +int nmdb_cache_incr(nmdb_t *db, const unsigned char *key, size_t ksize, + int64_t increment) +{ + return do_incr(db, key, ksize, increment, 0); +} + + diff --git a/libnmdb/net-const.h b/libnmdb/net-const.h index bdaa29d..b9e875a 100644 --- a/libnmdb/net-const.h +++ b/libnmdb/net-const.h @@ -19,6 +19,10 @@ #define UDP_SERVER_ADDR "0.0.0.0" #define UDP_SERVER_PORT 26010 +/* SCTP default listen address and port. */ +#define SCTP_SERVER_ADDR "0.0.0.0" +#define SCTP_SERVER_PORT 26010 + /* Protocol version, for checking in the network header. */ #define PROTO_VER 1 @@ -33,6 +37,8 @@ #define REQ_DEL_ASYNC 0x108 #define REQ_CACHE_CAS 0x109 #define REQ_CAS 0x110 +#define REQ_CACHE_INCR 0x111 +#define REQ_INCR 0x112 /* Network replies (different namespace from requests) */ #define REP_ERR 0x800 diff --git a/libnmdb/nmdb.h b/libnmdb/nmdb.skel.h similarity index 70% rename from libnmdb/nmdb.h rename to libnmdb/nmdb.skel.h index 9184e8c..4c8b915 100644 --- a/libnmdb/nmdb.h +++ b/libnmdb/nmdb.skel.h @@ -1,30 +1,48 @@ +/* Header for the libnmdb library. */ + #ifndef _NMDB_H #define _NMDB_H +/* Defined to 0 or 1 at libnmdb build time according the build configuration, + * not to be used externally. */ +#define _ENABLE_TIPC ++CONFIG_ENABLE_TIPC++ +#define _ENABLE_TCP ++CONFIG_ENABLE_TCP++ +#define _ENABLE_UDP ++CONFIG_ENABLE_UDP++ +#define _ENABLE_SCTP ++CONFIG_ENABLE_SCTP++ + + #include /* socket defines */ #include /* socklen_t */ + +#if _ENABLE_TIPC #include /* struct sockaddr_tipc */ -#include /* struct sockaddr_in */ +#endif +#if (_ENABLE_TCP || _ENABLE_UDP || _ENABLE_SCTP) +#include /* struct sockaddr_in */ +#endif struct nmdb_srv { int fd; int type; union { + +#if _ENABLE_TIPC struct { unsigned int port; struct sockaddr_tipc srvsa; socklen_t srvlen; } tipc; +#endif + +#if (_ENABLE_TCP || _ENABLE_UDP || _ENABLE_SCTP) struct { struct sockaddr_in srvsa; socklen_t srvlen; - } tcp; - struct { - struct sockaddr_in srvsa; - socklen_t srvlen; - } udp; + } in; +#endif + } info; }; @@ -37,6 +55,7 @@ nmdb_t *nmdb_init(); int nmdb_add_tipc_server(nmdb_t *db, int port); int nmdb_add_tcp_server(nmdb_t *db, const char *addr, int port); int nmdb_add_udp_server(nmdb_t *db, const char *addr, int port); +int nmdb_add_sctp_server(nmdb_t *db, const char *addr, int port); int nmdb_free(nmdb_t *db); ssize_t nmdb_get(nmdb_t *db, const unsigned char *key, size_t ksize, @@ -62,6 +81,10 @@ int nmdb_cache_cas(nmdb_t *db, const unsigned char *key, size_t ksize, const unsigned char *oldval, size_t ovsize, const unsigned char *newval, size_t nvsize); +int nmdb_incr(nmdb_t *db, const unsigned char *key, size_t ksize, + int64_t increment); +int nmdb_cache_incr(nmdb_t *db, const unsigned char *key, size_t ksize, + int64_t increment); #endif diff --git a/libnmdb/udp.c b/libnmdb/sctp.c similarity index 60% copy from libnmdb/udp.c copy to libnmdb/sctp.c index 5ece6e8..1f1c1ab 100644 --- a/libnmdb/udp.c +++ b/libnmdb/sctp.c @@ -1,5 +1,5 @@ -#if ENABLE_UDP +#if ENABLE_SCTP #include /* socket defines */ #include /* socket functions */ @@ -9,7 +9,7 @@ #include /* memcpy() */ #include /* close() */ -#include /* UDP stuff */ +#include /* SCTP stuff */ #include /* gethostbyname() */ #include "nmdb.h" @@ -18,15 +18,23 @@ /* Used internally to really add the server once we have an IP address. */ -static int add_udp_server_addr(nmdb_t *db, in_addr_t *inetaddr, int port) +static int add_sctp_server_addr(nmdb_t *db, in_addr_t *inetaddr, int port) { - int fd; + int fd, rv; struct nmdb_srv *newsrv, *newarray; - fd = socket(AF_INET, SOCK_DGRAM, 0); + fd = socket(AF_INET, SOCK_SEQPACKET, IPPROTO_SCTP); if (fd < 0) return 0; + /* Disable Nagle algorithm because we often send small + * packets. Huge gain in performance. */ + rv = 1; + if (setsockopt(fd, IPPROTO_SCTP, SCTP_NODELAY, &rv, sizeof(rv)) < 0 ) { + close(fd); + return 0; + } + newarray = realloc(db->servers, sizeof(struct nmdb_srv) * (db->nservers + 1)); if (newarray == NULL) { @@ -38,17 +46,17 @@ static int add_udp_server_addr(nmdb_t *db, in_addr_t *inetaddr, int port) db->nservers++; if (port < 0) - port = UDP_SERVER_PORT; + port = SCTP_SERVER_PORT; newsrv = &(db->servers[db->nservers - 1]); newsrv->fd = fd; - newsrv->info.udp.srvsa.sin_family = AF_INET; - newsrv->info.udp.srvsa.sin_port = htons(port); - newsrv->info.udp.srvsa.sin_addr.s_addr = *inetaddr; - newsrv->info.udp.srvlen = sizeof(struct sockaddr_in); + newsrv->info.in.srvsa.sin_family = AF_INET; + newsrv->info.in.srvsa.sin_port = htons(port); + newsrv->info.in.srvsa.sin_addr.s_addr = *inetaddr; + newsrv->info.in.srvlen = sizeof(struct sockaddr_in); - newsrv->type = UDP_CONN; + newsrv->type = SCTP_CONN; /* keep the list sorted by port, so we can do a reliable selection */ qsort(db->servers, db->nservers, sizeof(struct nmdb_srv), @@ -57,14 +65,14 @@ static int add_udp_server_addr(nmdb_t *db, in_addr_t *inetaddr, int port) return 1; } -/* Same as nmdb_add_tcp_server() but for UDP. */ -int nmdb_add_udp_server(nmdb_t *db, const char *addr, int port) +/* Same as nmdb_add_tcp_server() but for SCTP. */ +int nmdb_add_sctp_server(nmdb_t *db, const char *addr, int port) { int rv; struct hostent *he; struct in_addr ia; - /* We try to resolve and then pass it to add_udp_server_addr(). */ + /* We try to resolve and then pass it to add_sctp_server_addr(). */ rv = inet_pton(AF_INET, addr, &ia); if (rv <= 0) { he = gethostbyname(addr); @@ -74,23 +82,23 @@ int nmdb_add_udp_server(nmdb_t *db, const char *addr, int port) ia.s_addr = *( (in_addr_t *) (he->h_addr_list[0]) ); } - return add_udp_server_addr(db, &(ia.s_addr), port); + return add_sctp_server_addr(db, &(ia.s_addr), port); } -int udp_srv_send(struct nmdb_srv *srv, +int sctp_srv_send(struct nmdb_srv *srv, const unsigned char *buf, size_t bsize) { ssize_t rv; rv = sendto(srv->fd, buf, bsize, 0, - (struct sockaddr *) &(srv->info.udp.srvsa), - srv->info.udp.srvlen); + (struct sockaddr *) &(srv->info.in.srvsa), + srv->info.in.srvlen); if (rv <= 0) return 0; return 1; } /* Used internally to get and parse replies from the server. */ -uint32_t udp_get_rep(struct nmdb_srv *srv, +uint32_t sctp_get_rep(struct nmdb_srv *srv, unsigned char *buf, size_t bsize, unsigned char **payload, size_t *psize) { @@ -119,27 +127,28 @@ uint32_t udp_get_rep(struct nmdb_srv *srv, } #else -/* Stubs to use when UDP is not enabled. */ +/* Stubs to use when SCTP is not enabled. */ +#include #include "nmdb.h" -int nmdb_add_udp_server(nmdb_t *db, int port) +int nmdb_add_sctp_server(nmdb_t *db, const char *addr, int port) { return 0; } -int udp_srv_send(struct nmdb_srv *srv, +int sctp_srv_send(struct nmdb_srv *srv, const unsigned char *buf, size_t bsize) { return 0; } -uint32_t udp_get_rep(struct nmdb_srv *srv, +uint32_t sctp_get_rep(struct nmdb_srv *srv, unsigned char *buf, size_t bsize, unsigned char **payload, size_t *psize) { return -1; } -#endif /* ENABLE_UDP */ +#endif /* ENABLE_SCTP */ diff --git a/libnmdb/sctp.h b/libnmdb/sctp.h new file mode 100644 index 0000000..56b0add --- /dev/null +++ b/libnmdb/sctp.h @@ -0,0 +1,12 @@ + +#ifndef _SCTP_H +#define _SCTP_H + +int sctp_srv_send(struct nmdb_srv *srv, + const unsigned char *buf, size_t bsize); +uint32_t sctp_get_rep(struct nmdb_srv *srv, + unsigned char *buf, size_t bsize, + unsigned char **payload, size_t *psize); + +#endif + diff --git a/libnmdb/tcp.c b/libnmdb/tcp.c index 6e701ed..0bee671 100644 --- a/libnmdb/tcp.c +++ b/libnmdb/tcp.c @@ -43,12 +43,12 @@ static int add_tcp_server_addr(nmdb_t *db, in_addr_t *inetaddr, int port) newsrv = &(db->servers[db->nservers - 1]); newsrv->fd = fd; - newsrv->info.tcp.srvsa.sin_family = AF_INET; - newsrv->info.tcp.srvsa.sin_port = htons(port); - newsrv->info.tcp.srvsa.sin_addr.s_addr = *inetaddr; + newsrv->info.in.srvsa.sin_family = AF_INET; + newsrv->info.in.srvsa.sin_port = htons(port); + newsrv->info.in.srvsa.sin_addr.s_addr = *inetaddr; - rv = connect(fd, (struct sockaddr *) &(newsrv->info.tcp.srvsa), - sizeof(newsrv->info.tcp.srvsa)); + rv = connect(fd, (struct sockaddr *) &(newsrv->info.in.srvsa), + sizeof(newsrv->info.in.srvsa)); if (rv < 0) goto error_exit; @@ -185,6 +185,7 @@ uint32_t tcp_get_rep(struct nmdb_srv *srv, #else /* Stubs to use when TCP is not enabled. */ +#include #include "nmdb.h" int nmdb_add_tcp_server(nmdb_t *db, const char *addr, int port) diff --git a/libnmdb/tipc.c b/libnmdb/tipc.c index cea13ff..6482352 100644 --- a/libnmdb/tipc.c +++ b/libnmdb/tipc.c @@ -105,6 +105,7 @@ uint32_t tipc_get_rep(struct nmdb_srv *srv, #else /* Stubs to use when TIPC is not enabled. */ +#include #include "nmdb.h" int nmdb_add_tipc_server(nmdb_t *db, int port) diff --git a/libnmdb/udp.c b/libnmdb/udp.c index 5ece6e8..55902d2 100644 --- a/libnmdb/udp.c +++ b/libnmdb/udp.c @@ -43,10 +43,10 @@ static int add_udp_server_addr(nmdb_t *db, in_addr_t *inetaddr, int port) newsrv = &(db->servers[db->nservers - 1]); newsrv->fd = fd; - newsrv->info.udp.srvsa.sin_family = AF_INET; - newsrv->info.udp.srvsa.sin_port = htons(port); - newsrv->info.udp.srvsa.sin_addr.s_addr = *inetaddr; - newsrv->info.udp.srvlen = sizeof(struct sockaddr_in); + newsrv->info.in.srvsa.sin_family = AF_INET; + newsrv->info.in.srvsa.sin_port = htons(port); + newsrv->info.in.srvsa.sin_addr.s_addr = *inetaddr; + newsrv->info.in.srvlen = sizeof(struct sockaddr_in); newsrv->type = UDP_CONN; @@ -82,8 +82,8 @@ int udp_srv_send(struct nmdb_srv *srv, { ssize_t rv; rv = sendto(srv->fd, buf, bsize, 0, - (struct sockaddr *) &(srv->info.udp.srvsa), - srv->info.udp.srvlen); + (struct sockaddr *) &(srv->info.in.srvsa), + srv->info.in.srvlen); if (rv <= 0) return 0; return 1; @@ -121,9 +121,10 @@ uint32_t udp_get_rep(struct nmdb_srv *srv, #else /* Stubs to use when UDP is not enabled. */ +#include #include "nmdb.h" -int nmdb_add_udp_server(nmdb_t *db, int port) +int nmdb_add_udp_server(nmdb_t *db, const char *addr, int port) { return 0; } diff --git a/nmdb/Makefile b/nmdb/Makefile index 5c58c5a..c3e103a 100644 --- a/nmdb/Makefile +++ b/nmdb/Makefile @@ -1,13 +1,19 @@ +# Protocols to enable ENABLE_TIPC = 1 ENABLE_TCP = 1 ENABLE_UDP = 1 +ENABLE_SCTP = 1 -CFLAGS += -std=c99 -Wall -O3 +# Backend to use, can be qdbm, bdb, or null +BACKEND = qdbm + +CFLAGS += -std=c99 -pedantic -Wall -O3 ALL_CFLAGS = -D_XOPEN_SOURCE=500 $(CFLAGS) ALL_CFLAGS += -DENABLE_TIPC=$(ENABLE_TIPC) \ -DENABLE_TCP=$(ENABLE_TCP) \ - -DENABLE_UDP=$(ENABLE_UDP) + -DENABLE_UDP=$(ENABLE_UDP) \ + -DENABLE_SCTP=$(ENABLE_SCTP) ifdef DEBUG ALL_CFLAGS += -g @@ -17,15 +23,14 @@ ifdef PROFILE ALL_CFLAGS += -g -pg -fprofile-arcs -ftest-coverage endif -ifdef STRICT -ALL_CFLAGS += -ansi -pedantic -endif # prefix for installing the binaries PREFIX=/usr/local -OBJS = be-qdbm.o cache.o db.o queue.o net.o parse.o main.o +OBJS = cache.o dbloop.o queue.o log.o net.o parse.o main.o +LIBS = -levent -lpthread -lrt + ifeq ($(ENABLE_TIPC), 1) OBJS += tipc.o @@ -45,27 +50,55 @@ else OBJS += udp-stub.o endif +ifeq ($(ENABLE_SCTP), 1) + OBJS += sctp.o +else + OBJS += sctp-stub.o +endif + +# Use series of ifeq-endif instead of else-ifeq because otherwise the nesting +# is a mess. Using "else ifeq ..." in the same line is only supported from +# gmake 3.81, which is too new. +ifeq ($(BACKEND), qdbm) + OBJS += be-qdbm.o + ALL_CFLAGS += -DBACKEND_QDBM + LIBS += -lqdbm +endif +ifeq ($(BACKEND), bdb) + OBJS += be-bdb.o + ALL_CFLAGS += -DBACKEND_BDB + LIBS += -ldb +endif +ifeq ($(BACKEND), null) + OBJS += be-null.o + ALL_CFLAGS += -DBACKEND_NULL +endif + default: all all: nmdb nmdb: $(OBJS) - $(CC) $(ALL_CFLAGS) $(OBJS) -levent -lpthread -lrt -lqdbm -o nmdb + $(CC) $(ALL_CFLAGS) $(OBJS) $(LIBS) -o nmdb .c.o: $(CC) $(ALL_CFLAGS) -c $< -o $@ -install: all +install-bin: nmdb install -d $(PREFIX)/bin install -m 0755 nmdb $(PREFIX)/bin + +install-man: install -d $(PREFIX)/man/man1 install -m 0644 nmdb.1 $(PREFIX)/man/man1/ +install: install-bin install-man + clean: rm -f $(OBJS) nmdb rm -f *.bb *.bbg *.da *.gcov *.gcda *.gcno gmon.out -.PHONY: default all clean +.PHONY: default all clean install-bin install-man install diff --git a/nmdb/be-bdb.c b/nmdb/be-bdb.c new file mode 100644 index 0000000..4a930f8 --- /dev/null +++ b/nmdb/be-bdb.c @@ -0,0 +1,99 @@ + +#include /* memset() */ +#include "be.h" + + +db_t *db_open(const char *name, int flags) +{ + int rv; + db_t *db; + + rv = db_create(&db, NULL, 0); + if (rv != 0) + return NULL; + + rv = db->open(db, NULL, name, NULL, DB_HASH, DB_CREATE, 0); + if (rv != 0) { + db->close(db, 0); + return NULL; + } + + return db; +} + + +int db_close(db_t *db) +{ + int rv; + + rv = db->close(db, 0); + if (rv != 0) + return 0; + return 1; +} + + +int db_set(db_t *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t vsize) +{ + int rv; + DBT k, v; + + memset(&k, 0, sizeof(DBT)); + memset(&v, 0, sizeof(DBT)); + + /* we can't maintain "const"ness here because bdb's prototypes; the + * same applies to get and del */ + k.data = key; + k.size = ksize; + v.data = val; + v.size = vsize; + + rv = db->put(db, NULL, &k, &v, 0); + if (rv != 0) + return 0; + return 1; +} + + +int db_get(db_t *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t *vsize) +{ + int rv; + DBT k, v; + + memset(&k, 0, sizeof(DBT)); + memset(&v, 0, sizeof(DBT)); + + k.data = key; + k.size = ksize; + v.data = val; + v.ulen = *vsize; + v.flags = DB_DBT_USERMEM; /* we supplied the memory */ + + rv = db->get(db, NULL, &k, &v, 0); + if (rv != 0) { + return 0; + } else { + *vsize = v.size; + return 1; + } +} + +int db_del(db_t *db, const unsigned char *key, size_t ksize) +{ + int rv; + DBT k, v; + + memset(&k, 0, sizeof(DBT)); + memset(&v, 0, sizeof(DBT)); + + k.data = key; + k.size = ksize; + + rv = db->del(db, NULL, &k, 0); + if (rv != 0) + return 0; + return 1; +} + diff --git a/nmdb/be-null.c b/nmdb/be-null.c new file mode 100644 index 0000000..c358fa0 --- /dev/null +++ b/nmdb/be-null.c @@ -0,0 +1,37 @@ + +#include /* size_t */ +#include "be.h" + + +db_t *db_open(const char *name, int flags) +{ + /* Use a dumb not-null pointer because it is never looked at outside + * the functions defined here */ + return (db_t *) 1; +} + + +int db_close(db_t *db) +{ + return 1; +} + + +int db_set(db_t *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t vsize) +{ + return 1; +} + + +int db_get(db_t *db, const unsigned char *key, size_t ksize, + unsigned char *val, size_t *vsize) +{ + return 0; +} + +int db_del(db_t *db, const unsigned char *key, size_t ksize) +{ + return 0; +} + diff --git a/nmdb/be-qdbm.c b/nmdb/be-qdbm.c index d247d1d..3c98f9b 100644 --- a/nmdb/be-qdbm.c +++ b/nmdb/be-qdbm.c @@ -9,7 +9,7 @@ db_t *db_open(const char *name, int flags) { int f; - f = DP_OREADER | DP_OWRITER | DP_ONOLCK; + f = DP_OREADER | DP_OWRITER | DP_ONOLCK | DP_OCREAT; return dpopen(name, f, 0); } diff --git a/nmdb/be.h b/nmdb/be.h index 246f3de..fd8e384 100644 --- a/nmdb/be.h +++ b/nmdb/be.h @@ -2,10 +2,24 @@ #ifndef _BE_H #define _BE_H -/* The following should be specific to the db backend we use. As we only - * handle qdbm for now, there's no need to play with #ifdefs. */ -#include -typedef DEPOT db_t; +/* Depending on our backend, we define db_t to be different things so the + * generic code doesn't have to care about which backend we're using. */ +#if defined BACKEND_QDBM + #include + typedef DEPOT db_t; +#elif defined BACKEND_BDB + /* typedefs to work around db.h include bug */ + typedef unsigned int u_int; + typedef unsigned long u_long; + #include + typedef DB db_t; +#elif defined BACKEND_NULL + typedef int db_t; +#else + #error "Unknown backend" + /* Define it anyway, so this is the only warning/error the user sees. */ + typedef int db_t; +#endif db_t *db_open(const char *name, int flags); diff --git a/nmdb/cache.c b/nmdb/cache.c index 59538b3..5f8ae14 100644 --- a/nmdb/cache.c +++ b/nmdb/cache.c @@ -6,11 +6,13 @@ */ #include /* for size_t */ -#include /* for uint32_t */ +#include /* for [u]int*_t */ #include /* for malloc() */ #include /* for memcpy()/memcmp() */ +#include /* snprintf() */ #include "cache.h" + struct cache *cache_create(size_t numobjs, unsigned int flags) { size_t i; @@ -27,6 +29,7 @@ struct cache *cache_create(size_t numobjs, unsigned int flags) * an arbitrary number. It's long enough to make LRU useful, and small * enough to make lookups fast. */ cd->chainlen = 4; + cd->numobjs = numobjs; cd->hashlen = numobjs / cd->chainlen; cd->table = (struct cache_chain *) @@ -103,12 +106,9 @@ static uint32_t hash(const unsigned char *key, const size_t ksize) } -/* - * Looks given key up in the chain. - * Returns NULL if not found, or a pointer to the cache entry if it's found. - * The chain can be empty. - * Used in cache_get() and cache_set(). - */ +/* Looks given key up in the chain. Returns NULL if not found, or a pointer to + * the cache entry if it's found. The chain can be empty. Used in cache_get() + * and cache_set(). */ static struct cache_entry *find_in_chain(struct cache_chain *c, const unsigned char *key, size_t ksize) { @@ -135,10 +135,8 @@ static struct cache_entry *find_in_chain(struct cache_chain *c, } -/* - * Gets the matching value for the given key. - * Returns 0 if no match was found, or 1 otherwise. - */ +/* Gets the matching value for the given key. Returns 0 if no match was + * found, or 1 otherwise. */ int cache_get(struct cache *cd, const unsigned char *key, size_t ksize, unsigned char **val, size_t *vsize) { @@ -356,3 +354,60 @@ exit: return rv; } + +/* Increment the value associated with the given key by the given increment. + * The increment is a signed 64 bit value, and the value size must be >= 8 + * bytes. + * Returns: + * 1 if the increment succeeded. + * -1 if the value was not in the cache. + * -2 if the value was not null terminated. + * -3 if there was a memory error. + */ +int cache_incr(struct cache *cd, const unsigned char *key, size_t ksize, + int64_t increment) +{ + uint32_t h = 0; + unsigned char *val, *newval; + int64_t intval; + size_t vsize; + struct cache_chain *c; + struct cache_entry *e; + + h = hash(key, ksize) % cd->hashlen; + c = cd->table + h; + + e = find_in_chain(c, key, ksize); + + if (e == NULL) + return -1; + + val = e->val; + vsize = e->vsize; + + /* the value must be a NULL terminated string, otherwise strtoll might + * cause a segmentation fault */ + if (val && val[vsize - 1] != '\0') + return -2; + + intval = strtoll((char *) val, NULL, 10); + intval = intval + increment; + + /* The max value for an unsigned long long is 18446744073709551615, + * and strlen('18446744073709551615') = 20, so if the value is smaller + * than 24 (just in case) we create a new buffer. */ + if (vsize < 24) { + newval = malloc(24); + if (newval == NULL) + return -3; + free(val); + e->val = val = newval; + e->vsize = vsize = 24; + } + + snprintf((char *) val, vsize, "%23lld", (long long int) intval); + + return 1; +} + + diff --git a/nmdb/cache.h b/nmdb/cache.h index 66c705d..c4d3d7f 100644 --- a/nmdb/cache.h +++ b/nmdb/cache.h @@ -5,6 +5,7 @@ /* Generic cache layer. See cache.c for more information. */ #include /* for size_t */ +#include /* for int64_t */ struct cache { @@ -47,6 +48,8 @@ int cache_del(struct cache *cd, const unsigned char *key, size_t ksize); int cache_cas(struct cache *cd, const unsigned char *key, size_t ksize, const unsigned char *oldval, size_t ovsize, const unsigned char *newval, size_t nvsize); +int cache_incr(struct cache *cd, const unsigned char *key, size_t ksize, + int64_t increment); #endif diff --git a/nmdb/common.h b/nmdb/common.h index 2d6cf56..1a872c7 100644 --- a/nmdb/common.h +++ b/nmdb/common.h @@ -20,10 +20,13 @@ struct settings { int tcp_port; char *udp_addr; int udp_port; + char *sctp_addr; + int sctp_port; int numobjs; int foreground; int passive; char *dbname; + char *logfname; }; extern struct settings settings; diff --git a/nmdb/db.c b/nmdb/dbloop.c similarity index 68% rename from nmdb/db.c rename to nmdb/dbloop.c index defc881..998720d 100644 --- a/nmdb/db.c +++ b/nmdb/dbloop.c @@ -2,15 +2,17 @@ #include /* threading functions */ #include /* nanosleep() */ #include /* ETIMEDOUT */ -#include /* perror() */ #include /* memcmp() */ +#include /* malloc()/free() */ +#include /* snprintf() */ #include "common.h" -#include "db.h" +#include "dbloop.h" #include "be.h" #include "queue.h" #include "net-const.h" #include "req.h" +#include "log.h" static void *db_loop(void *arg); @@ -70,7 +72,7 @@ static void *db_loop(void *arg) } if (rv != 0 && rv != ETIMEDOUT) { - perror("Error in queue_timedwait()"); + errlog("Error in queue_timedwait()"); continue; } @@ -104,7 +106,7 @@ static void process_op(db_t *db, struct queue_entry *e) e->req->reply_err(e->req, ERR_DB); return; } - e->req->reply_set(e->req, REP_OK); + e->req->reply_mini(e->req, REP_OK); } else if (e->operation == REQ_SET_ASYNC) { db_set(db, e->key, e->ksize, e->val, e->vsize); @@ -120,20 +122,20 @@ static void process_op(db_t *db, struct queue_entry *e) } rv = db_get(db, e->key, e->ksize, val, &vsize); if (rv == 0) { - e->req->reply_get(e->req, REP_NOTIN, NULL, 0); + e->req->reply_mini(e->req, REP_NOTIN); free(val); return; } - e->req->reply_get(e->req, REP_OK, val, vsize); + e->req->reply_long(e->req, REP_OK, val, vsize); free(val); } else if (e->operation == REQ_DEL_SYNC) { rv = db_del(db, e->key, e->ksize); if (rv == 0) { - e->req->reply_del(e->req, REP_NOTIN); + e->req->reply_mini(e->req, REP_NOTIN); return; } - e->req->reply_del(e->req, REP_OK); + e->req->reply_mini(e->req, REP_OK); } else if (e->operation == REQ_DEL_ASYNC) { db_del(db, e->key, e->ksize); @@ -150,7 +152,7 @@ static void process_op(db_t *db, struct queue_entry *e) } rv = db_get(db, e->key, e->ksize, dbval, &dbvsize); if (rv == 0) { - e->req->reply_get(e->req, REP_NOTIN, NULL, 0); + e->req->reply_mini(e->req, REP_NOTIN); free(dbval); return; } @@ -164,16 +166,62 @@ static void process_op(db_t *db, struct queue_entry *e) return; } - e->req->reply_cas(e->req, REP_OK); + e->req->reply_mini(e->req, REP_OK); free(dbval); return; } - e->req->reply_cas(e->req, REP_NOMATCH); + e->req->reply_mini(e->req, REP_NOMATCH); + free(dbval); + + } else if (e->operation == REQ_INCR) { + unsigned char *dbval; + size_t dbvsize = 64 * 1024; + int64_t intval; + + dbval = malloc(dbvsize); + if (dbval == NULL) { + e->req->reply_err(e->req, ERR_MEM); + return; + } + rv = db_get(db, e->key, e->ksize, dbval, &dbvsize); + if (rv == 0) { + e->req->reply_mini(e->req, REP_NOTIN); + free(dbval); + return; + } + + /* val must be NULL terminated; see cache_incr() */ + if (dbval && dbval[dbvsize - 1] != '\0') { + e->req->reply_mini(e->req, REP_NOMATCH); + free(dbval); + return; + } + + intval = strtoll((char *) dbval, NULL, 10); + intval = intval + * (int64_t *) e->val; + + if (dbvsize < 24) { + /* We know dbval is long enough because we've + * allocated it, so we only change dbvsize */ + dbvsize = 24; + } + + snprintf((char *) dbval, dbvsize, "%23lld", + (long long int) intval); + + rv = db_set(db, e->key, e->ksize, dbval, dbvsize); + if (!rv) { + e->req->reply_err(e->req, ERR_DB); + return; + } + + e->req->reply_mini(e->req, REP_OK); + free(dbval); } else { - printf("Unknown op 0x%x\n", e->operation); + wlog("Unknown op 0x%x\n", e->operation); } } diff --git a/nmdb/db.h b/nmdb/dbloop.h similarity index 81% rename from nmdb/db.h rename to nmdb/dbloop.h index c764d6a..a4a9d63 100644 --- a/nmdb/db.h +++ b/nmdb/dbloop.h @@ -1,6 +1,6 @@ -#ifndef _DB_H -#define _DB_H +#ifndef _DBLOOP_H +#define _DBLOOP_H #include /* for pthread_t */ #include "be.h" /* for db_t */ diff --git a/nmdb/log.c b/nmdb/log.c new file mode 100644 index 0000000..89d107e --- /dev/null +++ b/nmdb/log.c @@ -0,0 +1,65 @@ + +#include /* vsprintf() */ +#include +#include /* open() */ +#include +#include +#include /* write() */ +#include /* strcmp(), strerror() */ +#include /* errno */ + +#include "log.h" +#include "common.h" + + +/* Logging file descriptor, -1 if logging is disabled */ +int logfd = -1; + + +int log_init(void) +{ + if (settings.logfname == NULL) { + logfd = -1; + return 1; + } + + if (strcmp(settings.logfname, "-") == 0) { + logfd = 1; + } else { + logfd = open(settings.logfname, O_WRONLY | O_APPEND | O_CREAT); + if (logfd < 0) + return 0; + } + + return 1; +} + +void wlog(const char *fmt, ...) +{ + int r, tr; + va_list ap; + char str[MAX_LOG_STR]; + char timestr[MAX_LOG_STR]; + time_t t; + struct tm *tmp; + + if (logfd == -1) + return; + + t = time(NULL); + tmp = localtime(&t); + tr = strftime(timestr, MAX_LOG_STR, "%F %H:%M:%S ", tmp); + + va_start(ap, fmt); + r = vsnprintf(str, MAX_LOG_STR, fmt, ap); + va_end(ap); + + write(logfd, timestr, tr); + write(logfd, str, r); +} + +void errlog(const char *s) +{ + wlog("%s: %s\n", s, strerror(errno)); +} + diff --git a/nmdb/log.h b/nmdb/log.h new file mode 100644 index 0000000..b187cc0 --- /dev/null +++ b/nmdb/log.h @@ -0,0 +1,18 @@ + +#ifndef _LOG_H +#define _LOG_H + +/* Maximum string to log */ +#define MAX_LOG_STR 512 + +int log_init(void); + +/* Normal logging, printf()-alike */ +void wlog(const char *fmt, ...); + +/* Errno logging, perror()-alike */ +void errlog(const char *s); + +#endif + + diff --git a/nmdb/main.c b/nmdb/main.c index 4e75eb7..5bb3cad 100644 --- a/nmdb/main.c +++ b/nmdb/main.c @@ -8,9 +8,10 @@ #include "cache.h" #include "net.h" -#include "db.h" +#include "dbloop.h" #include "common.h" #include "net-const.h" +#include "log.h" #define DEFDBNAME "database" @@ -32,7 +33,10 @@ static void help(void) { " -T addr TCP listening address (all local addresses)\n" " -u port UDP listening port (26010)\n" " -U addr UDP listening address (all local addresses)\n" + " -s port SCTP listening port (26010)\n" + " -S addr SCTP listening address (all local addresses)\n" " -c nobj max. number of objects to be cached, in thousands (128)\n" + " -o fname log to the file 'fname'.\n" " -f don't fork and stay in the foreground\n" " -p enable passive mode, for redundancy purposes (read docs.)\n" " -h show this help\n" @@ -53,14 +57,17 @@ static int load_settings(int argc, char **argv) settings.tcp_port = -1; settings.udp_addr = NULL; settings.udp_port = -1; + settings.sctp_addr = NULL; + settings.sctp_port = -1; settings.numobjs = -1; settings.foreground = 0; settings.passive = 0; + settings.logfname = "-"; settings.dbname = malloc(strlen(DEFDBNAME) + 1); strcpy(settings.dbname, DEFDBNAME); - while ((c = getopt(argc, argv, "d:l:L:a:P:c:fph?")) != -1) { + while ((c = getopt(argc, argv, "d:l:L:t:T:u:U:c:o:fph?")) != -1) { switch(c) { case 'd': free(settings.dbname); @@ -89,9 +96,22 @@ static int load_settings(int argc, char **argv) settings.udp_addr = optarg; break; + case 's': + settings.sctp_port = atoi(optarg); + break; + case 'S': + settings.sctp_addr = optarg; + break; + case 'c': settings.numobjs = atoi(optarg) * 1024; break; + + case 'o': + settings.logfname = malloc(strlen(optarg) + 1); + strcpy(settings.logfname, optarg); + break; + case 'f': settings.foreground = 1; break; @@ -121,6 +141,10 @@ static int load_settings(int argc, char **argv) settings.udp_addr = UDP_SERVER_ADDR; if (settings.udp_port == -1) settings.udp_port = UDP_SERVER_PORT; + if (settings.sctp_addr == NULL) + settings.sctp_addr = SCTP_SERVER_ADDR; + if (settings.sctp_port == -1) + settings.sctp_port = SCTP_SERVER_PORT; if (settings.numobjs == -1) settings.numobjs = 128 * 1024; @@ -148,25 +172,30 @@ int main(int argc, char **argv) if (!load_settings(argc, argv)) return 1; + if (!log_init()) { + perror("Error opening log file"); + return 1; + } + init_stats(); cd = cache_create(settings.numobjs, 0); if (cd == NULL) { - perror("Error creating cache"); + errlog("Error creating cache"); return 1; } cache_table = cd; q = queue_create(); if (q == NULL) { - perror("Error creating queue"); + errlog("Error creating queue"); return 1; } op_queue = q; db = db_open(settings.dbname, 0); if (db == NULL) { - perror("Error opening DB"); + errlog("Error opening DB"); return 1; } @@ -176,7 +205,7 @@ int main(int argc, char **argv) /* parent exits */ return 0; } else if (pid < 0) { - perror("Error in fork()"); + errlog("Error in fork()"); return 1; } diff --git a/nmdb/net-const.h b/nmdb/net-const.h index bdaa29d..b9e875a 100644 --- a/nmdb/net-const.h +++ b/nmdb/net-const.h @@ -19,6 +19,10 @@ #define UDP_SERVER_ADDR "0.0.0.0" #define UDP_SERVER_PORT 26010 +/* SCTP default listen address and port. */ +#define SCTP_SERVER_ADDR "0.0.0.0" +#define SCTP_SERVER_PORT 26010 + /* Protocol version, for checking in the network header. */ #define PROTO_VER 1 @@ -33,6 +37,8 @@ #define REQ_DEL_ASYNC 0x108 #define REQ_CACHE_CAS 0x109 #define REQ_CAS 0x110 +#define REQ_CACHE_INCR 0x111 +#define REQ_INCR 0x112 /* Network replies (different namespace from requests) */ #define REP_ERR 0x800 diff --git a/nmdb/net.c b/nmdb/net.c index a376c17..fc01220 100644 --- a/nmdb/net.c +++ b/nmdb/net.c @@ -1,6 +1,5 @@ #include /* signal constants */ -#include /* perror() */ #include /* exit() */ /* Workaround for libevent 1.1a: the header assumes u_char is typedef'ed to an @@ -13,18 +12,20 @@ typedef unsigned char u_char; #include "tipc.h" #include "tcp.h" #include "udp.h" +#include "sctp.h" #include "net.h" +#include "log.h" static void exit_sighandler(int fd, short event, void *arg) { - printf("Got signal! Puf!\n"); + wlog("Got signal! Puf!\n"); event_loopexit(NULL); } static void passive_to_active_sighandler(int fd, short event, void *arg) { - printf("Passive toggle!\n"); + wlog("Passive toggle!\n"); settings.passive = !settings.passive; } @@ -33,7 +34,8 @@ void net_loop(void) int tipc_fd = -1; int tcp_fd = -1; int udp_fd = -1; - struct event tipc_evt, tcp_evt, udp_evt, + int sctp_fd = -1; + struct event tipc_evt, tcp_evt, udp_evt, sctp_evt, sigterm_evt, sigint_evt, sigusr2_evt; event_init(); @@ -44,7 +46,7 @@ void net_loop(void) if (ENABLE_TIPC) { tipc_fd = tipc_init(); if (tipc_fd < 0) { - perror("Error initializing TIPC"); + errlog("Error initializing TIPC"); exit(1); } @@ -56,19 +58,19 @@ void net_loop(void) if (ENABLE_TCP) { tcp_fd = tcp_init(); if (tcp_fd < 0) { - perror("Error initializing TCP"); + errlog("Error initializing TCP"); exit(1); } - event_set(&tcp_evt, tcp_fd, EV_READ | EV_PERSIST, tcp_newconnection, - &tcp_evt); + event_set(&tcp_evt, tcp_fd, EV_READ | EV_PERSIST, + tcp_newconnection, &tcp_evt); event_add(&tcp_evt, NULL); } if (ENABLE_UDP) { udp_fd = udp_init(); if (udp_fd < 0) { - perror("Error initializing UDP"); + errlog("Error initializing UDP"); exit(1); } @@ -77,6 +79,18 @@ void net_loop(void) event_add(&udp_evt, NULL); } + if (ENABLE_SCTP) { + sctp_fd = sctp_init(); + if (sctp_fd < 0) { + errlog("Error initializing SCTP"); + exit(1); + } + + event_set(&sctp_evt, sctp_fd, EV_READ | EV_PERSIST, sctp_recv, + &sctp_evt); + event_add(&sctp_evt, NULL); + } + signal_set(&sigterm_evt, SIGTERM, exit_sighandler, &sigterm_evt); signal_add(&sigterm_evt, NULL); signal_set(&sigint_evt, SIGINT, exit_sighandler, &sigint_evt); @@ -87,8 +101,15 @@ void net_loop(void) event_dispatch(); - event_del(&tipc_evt); - event_del(&tcp_evt); + if (ENABLE_TIPC) + event_del(&tipc_evt); + if (ENABLE_TCP) + event_del(&tcp_evt); + if (ENABLE_UDP) + event_del(&udp_evt); + if (ENABLE_SCTP) + event_del(&sctp_evt); + signal_del(&sigterm_evt); signal_del(&sigint_evt); signal_del(&sigusr2_evt); @@ -96,6 +117,7 @@ void net_loop(void) tipc_close(tipc_fd); tcp_close(tcp_fd); udp_close(udp_fd); + sctp_close(sctp_fd); } diff --git a/nmdb/nmdb.1 b/nmdb/nmdb.1 index 36d4418..49c5a77 100644 --- a/nmdb/nmdb.1 +++ b/nmdb/nmdb.1 @@ -2,25 +2,21 @@ .SH NAME nmdb - A multiprotocol network database manager .SH SYNOPSYS -nmdb [-d dbpath] [-l lower] [-L upper] [-t tcpport] [-T tcpaddr] -[-u udpport] [-U udpaddr] [-c nobj] [-f] [-p] [-h] +nmdb [-d dbpath] [-l lower] [-L upper] + [-t tcpport] [-T tcpaddr] + [-u udpport] [-U udpaddr] + [-s sctpport] [-S sctpaddr] + [-c nobj] [-f] [-p] [-h] + .SH DESCRIPTION -nmdb is a network database that can use several protocols to communicate with -it's clients. At the moment, it supports TIPC, TCP and UDP. +nmdb is a network database that can use different protocols to communicate +with its clients. At the moment, it supports TIPC, TCP, UDP and SCTP. It can also be used as a generic caching system (pretty much like memcached), because it has a very fast cache that can be used without impacting on the database. -Before using it, you need to create the backing database. As nmdb uses -.BR qdbm (3) -as a backend, you need to do this using the -.I dpmgr -command. See -.B "INVOCATION EXAMPLE" -below for more important information. - The database is accessed with the .BR libnmdb (3) library. Consult its manual page for programming information. Python bindings @@ -32,11 +28,8 @@ For additional documentation, go to the project's website at .SH OPTIONS .TP .B "-d dbpath" -Indicate the path to the database file to use. It must have been created with -.RB ' "dpmgr create" -.IR "dbpath" ' -and have read and write permissions. If a name is not provided, "database" -will be used. +Indicate the path to the database file to use. It will be created if it +doesn't exist. If a name is not provided, "database" will be used. .TP .B "-l lower" Lower TIPC port number to bind to. Defaults to 10. It's useful if you want to @@ -59,6 +52,12 @@ UDP listening port. Defaults to 26010. .B "-U udpaddr" IP listening address for UDP. Defaults to all local addresses. .TP +.B "-s sctpport" +SCTP listening port. Defaults to 26010. +.TP +.B "-S sctpaddr" +IP listening address for SCTP. Defaults to all local addresses. +.TP .B "-c nobj" Sets the maximum number of objects the cache will held, in thousands. Note that the size of the memory used by the cache layer depends on the size of the @@ -83,10 +82,7 @@ some circumnstances, and doesn't make much sense anyway). Show a brief help. .SH INVOCATION EXAMPLE -To create the database: -.B "dpmgr create /var/lib/nmpc-db" - -To run the server with the said database: +To run the server with the database at /var/lib/nmpc-db: .B "nmpc -d /var/lib/nmpc-db" Be @@ -103,5 +99,13 @@ can't bind a port twice. .B TIPC (http://tipc.sf.net), .BR qdbm (3). + .SH AUTHORS Created by Alberto Bertogli (albertito@gmail.com). + +.SH CONTACT + +To get in touch with developers and users, join the mailing list at +http://lists.auriga.wearlab.de/cgi-bin/mailman/listinfo/nmdb-devel or just +send an email to nmdb-devel@lists.auriga.wearlab.de. + diff --git a/nmdb/parse.c b/nmdb/parse.c index 1acd07f..368e67e 100644 --- a/nmdb/parse.c +++ b/nmdb/parse.c @@ -16,6 +16,7 @@ static void parse_get(struct req_info *req, int impact_db); static void parse_set(struct req_info *req, int impact_db, int async); static void parse_del(struct req_info *req, int impact_db, int async); static void parse_cas(struct req_info *req, int impact_db); +static void parse_incr(struct req_info *req, int impact_db); /* Create a queue entry structure based on the parameters passed. Memory @@ -178,6 +179,10 @@ int parse_message(struct req_info *req, parse_cas(req, 0); else if (cmd == REQ_CAS) parse_cas(req, 1); + else if (cmd == REQ_CACHE_INCR) + parse_incr(req, 0); + else if (cmd == REQ_INCR) + parse_incr(req, 1); else { stats.net_unk_req++; req->reply_err(req, ERR_UNKREQ); @@ -208,7 +213,7 @@ static void parse_get(struct req_info *req, int impact_db) hit = cache_get(cache_table, key, ksize, &val, &vsize); if (!hit && !impact_db) { - req->mini_reply(req, REP_CACHE_MISS); + req->reply_mini(req, REP_CACHE_MISS); return; } else if (!hit && impact_db) { struct queue_entry *e; @@ -223,7 +228,7 @@ static void parse_get(struct req_info *req, int impact_db) queue_signal(op_queue); return; } else { - req->reply_get(req, REP_CACHE_HIT, val, vsize); + req->reply_long(req, REP_CACHE_HIT, val, vsize); return; } } @@ -287,7 +292,7 @@ static void parse_set(struct req_info *req, int impact_db, int async) queue_unlock(op_queue); if (async) { - req->mini_reply(req, REP_OK); + req->reply_mini(req, REP_OK); } else { /* Signal the DB thread it has work only if it's a * synchronous operation, asynchronous don't mind @@ -298,7 +303,7 @@ static void parse_set(struct req_info *req, int impact_db, int async) } return; } else { - req->mini_reply(req, REP_OK); + req->reply_mini(req, REP_OK); } return; @@ -324,9 +329,9 @@ static void parse_del(struct req_info *req, int impact_db, int async) hit = cache_del(cache_table, key, ksize); if (!impact_db && hit) { - req->mini_reply(req, REP_OK); + req->reply_mini(req, REP_OK); } else if (!impact_db && !hit) { - req->mini_reply(req, REP_NOTIN); + req->reply_mini(req, REP_NOTIN); } else if (impact_db) { struct queue_entry *e; uint32_t request; @@ -345,7 +350,7 @@ static void parse_del(struct req_info *req, int impact_db, int async) queue_unlock(op_queue); if (async) { - req->mini_reply(req, REP_OK); + req->reply_mini(req, REP_OK); } else { /* See comment on parse_set(). */ queue_signal(op_queue); @@ -403,16 +408,16 @@ static void parse_cas(struct req_info *req, int impact_db) if (rv == 0) { /* If the cache doesn't match, there is no need to bother the * DB even if we were asked to impact. */ - req->mini_reply(req, REP_NOMATCH); + req->reply_mini(req, REP_NOMATCH); return; } if (!impact_db) { if (rv == -1) { - req->mini_reply(req, REP_NOTIN); + req->reply_mini(req, REP_NOTIN); return; } else { - req->mini_reply(req, REP_OK); + req->reply_mini(req, REP_OK); return; } } else { @@ -437,3 +442,97 @@ static void parse_cas(struct req_info *req, int impact_db) } +/* ntohll() is not standard, so we define it using an UGLY trick because there + * is no standard way to check for endianness at runtime! */ +static uint64_t ntohll(uint64_t x) +{ + static int endianness = 0; + + /* determine the endianness by checking how htonl() behaves; use -1 + * for little endian and 1 for big endian */ + if (endianness == 0) { + if (htonl(1) == 1) + endianness = 1; + else + endianness = -1; + } + + if (endianness == 1) { + /* big endian */ + return x; + } + + /* little endian */ + return ( ntohl( (x >> 32) & 0xFFFFFFFF ) | \ + ( (uint64_t) ntohl(x & 0xFFFFFFFF) ) << 32 ); +} + + +static void parse_incr(struct req_info *req, int impact_db) +{ + int cres; + const unsigned char *key; + uint32_t ksize; + int64_t increment; + const int max = 65536; + + /* Request format: + * 4 ksize + * ksize key + * 8 increment (big endian int64_t) + */ + ksize = * (uint32_t *) req->payload; + ksize = ntohl(ksize); + + /* Sanity check on sizes: + * - ksize + 8 must be < req->psize + * - ksize + 8 must be < 2^16 = 64k + */ + if ( (req->psize < ksize + 8) || ((ksize + 8) > max)) { + stats.net_broken_req++; + req->reply_err(req, ERR_BROKEN); + return; + } + + key = req->payload + sizeof(uint32_t); + increment = ntohll( * (int64_t *) (key + ksize) ); + + cres = cache_incr(cache_table, key, ksize, increment); + if (cres == -3) { + req->reply_err(req, ERR_MEM); + return; + } else if (cres == -2) { + /* the value was not NULL terminated */ + req->reply_mini(req, REP_NOMATCH); + return; + } + + if (impact_db) { + struct queue_entry *e; + + /* at this point, the cache_incr() was either successful or a + * miss, but we don't really care */ + + e = make_queue_entry(req, REQ_INCR, key, ksize, + (unsigned char *) &increment, + sizeof(increment)); + if (e == NULL) { + req->reply_err(req, ERR_MEM); + return; + } + queue_lock(op_queue); + queue_put(op_queue, e); + queue_unlock(op_queue); + + queue_signal(op_queue); + } else { + if (cres == -1) + req->reply_mini(req, REP_NOTIN); + else + req->reply_mini(req, REP_OK); + } + + return; +} + + diff --git a/nmdb/req.h b/nmdb/req.h index 454f0f8..4637307 100644 --- a/nmdb/req.h +++ b/nmdb/req.h @@ -11,6 +11,7 @@ #define REQTYPE_TIPC 1 #define REQTYPE_TCP 2 #define REQTYPE_UDP 3 +#define REQTYPE_SCTP 4 struct req_info { @@ -28,13 +29,10 @@ struct req_info { size_t psize; /* operations */ - void (*mini_reply)(struct req_info *req, uint32_t reply); + void (*reply_mini)(struct req_info *req, uint32_t reply); void (*reply_err)(struct req_info *req, uint32_t reply); - void (*reply_get)(struct req_info *req, uint32_t reply, + void (*reply_long)(struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize); - void (*reply_set)(struct req_info *req, uint32_t reply); - void (*reply_del)(struct req_info *req, uint32_t reply); - void (*reply_cas)(struct req_info *req, uint32_t reply); }; #endif diff --git a/nmdb/sctp-stub.c b/nmdb/sctp-stub.c new file mode 100644 index 0000000..96221f5 --- /dev/null +++ b/nmdb/sctp-stub.c @@ -0,0 +1,18 @@ + +/* SCTP stub file, used when SCTP is not compiled in. */ + +int sctp_init(void) +{ + return -1; +} + +void sctp_close(int fd) +{ + return; +} + +void sctp_recv(int fd, short event, void *arg) +{ + return; +} + diff --git a/nmdb/udp.c b/nmdb/sctp.c similarity index 68% copy from nmdb/udp.c copy to nmdb/sctp.c index 29cb789..42fdb4c 100644 --- a/nmdb/udp.c +++ b/nmdb/sctp.c @@ -2,28 +2,25 @@ #include /* socket defines */ #include /* socket functions */ #include /* malloc() */ -#include /* perror() */ #include /* uint32_t and friends */ #include /* htonls() and friends */ #include /* INET stuff */ -#include /* UDP stuff */ +#include /* SCTP stuff */ #include /* memcpy() */ #include /* close() */ -#include "udp.h" +#include "sctp.h" #include "common.h" #include "net-const.h" #include "req.h" #include "parse.h" +#include "log.h" -static void udp_mini_reply(struct req_info *req, uint32_t reply); -static void udp_reply_err(struct req_info *req, uint32_t reply); -static void udp_reply_get(struct req_info *req, uint32_t reply, +static void sctp_reply_mini(struct req_info *req, uint32_t reply); +static void sctp_reply_err(struct req_info *req, uint32_t reply); +static void sctp_reply_long(struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize); -static void udp_reply_set(struct req_info *req, uint32_t reply); -static void udp_reply_del(struct req_info *req, uint32_t reply); -static void udp_reply_cas(struct req_info *req, uint32_t reply); /* @@ -49,7 +46,7 @@ static void rep_send_error(const struct req_info *req, const unsigned int code) r = sendto(req->fd, minibuf, 3 * 4, 0, req->clisa, req->clilen); if (r < 0) { - perror("rep_send_error() failed"); + errlog("rep_send_error() failed"); } } @@ -72,7 +69,7 @@ static int rep_send(const struct req_info *req, const unsigned char *buf, /* Send small replies, consisting in only a value. */ -static void udp_mini_reply(struct req_info *req, uint32_t reply) +static void sctp_reply_mini(struct req_info *req, uint32_t reply) { /* We use a mini buffer to speedup the small replies, to avoid the * malloc() overhead. */ @@ -89,20 +86,20 @@ static void udp_mini_reply(struct req_info *req, uint32_t reply) } -/* The udp_reply_* functions are used by the db code to send the network +/* The sctp_reply_* functions are used by the db code to send the network * replies. */ -void udp_reply_err(struct req_info *req, uint32_t reply) +void sctp_reply_err(struct req_info *req, uint32_t reply) { rep_send_error(req, reply); } -void udp_reply_get(struct req_info *req, uint32_t reply, +void sctp_reply_long(struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize) { if (val == NULL) { /* miss */ - udp_mini_reply(req, reply); + sctp_reply_mini(req, reply); } else { unsigned char *buf; size_t bsize; @@ -134,60 +131,61 @@ void udp_reply_get(struct req_info *req, uint32_t reply, } -void udp_reply_set(struct req_info *req, uint32_t reply) -{ - udp_mini_reply(req, reply); -} - - -void udp_reply_del(struct req_info *req, uint32_t reply) -{ - udp_mini_reply(req, reply); -} - -void udp_reply_cas(struct req_info *req, uint32_t reply) -{ - udp_mini_reply(req, reply); -} - - /* * Main functions for receiving and parsing */ -int udp_init(void) +int sctp_init(void) { int fd, rv; struct sockaddr_in srvsa; struct in_addr ia; - rv = inet_pton(AF_INET, settings.udp_addr, &ia); + rv = inet_pton(AF_INET, settings.sctp_addr, &ia); if (rv <= 0) return -1; srvsa.sin_family = AF_INET; srvsa.sin_addr.s_addr = ia.s_addr; - srvsa.sin_port = htons(settings.udp_port); + srvsa.sin_port = htons(settings.sctp_port); - fd = socket(AF_INET, SOCK_DGRAM, 0); + fd = socket(AF_INET, SOCK_SEQPACKET, IPPROTO_SCTP); if (fd < 0) return -1; rv = 1; if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &rv, sizeof(rv)) < 0 ) { + errlog("Error in setsockopt():"); close(fd); return -1; } rv = bind(fd, (struct sockaddr *) &srvsa, sizeof(srvsa)); - if (rv < 0) + if (rv < 0) { + close(fd); return -1; + } + + rv = listen(fd, 1024); + if (rv < 0) { + close(fd); + return -1; + } + + /* Disable nagle algorithm, as we often handle small amounts of data + * it can make I/O quite slow. */ + rv = 1; + if (setsockopt(fd, IPPROTO_SCTP, SCTP_NODELAY, &rv, sizeof(rv)) < 0 ) { + close(fd); + return -1; + } + return fd; } -void udp_close(int fd) +void sctp_close(int fd) { close(fd); } @@ -199,7 +197,7 @@ void udp_close(int fd) static unsigned char static_buf[SBSIZE]; /* Called by libevent for each receive event */ -void udp_recv(int fd, short event, void *arg) +void sctp_recv(int fd, short event, void *arg) { int rv; struct req_info req; @@ -220,15 +218,12 @@ void udp_recv(int fd, short event, void *arg) } req.fd = fd; - req.type = REQTYPE_UDP; + req.type = REQTYPE_SCTP; req.clisa = (struct sockaddr *) &clisa; req.clilen = clilen; - req.mini_reply = udp_mini_reply; - req.reply_err = udp_reply_err; - req.reply_get = udp_reply_get; - req.reply_set = udp_reply_set; - req.reply_del = udp_reply_del; - req.reply_cas = udp_reply_cas; + req.reply_mini = sctp_reply_mini; + req.reply_err = sctp_reply_err; + req.reply_long = sctp_reply_long; /* parse the message */ parse_message(&req, static_buf, rv); diff --git a/nmdb/sctp.h b/nmdb/sctp.h new file mode 100644 index 0000000..710fb18 --- /dev/null +++ b/nmdb/sctp.h @@ -0,0 +1,10 @@ + +#ifndef _SCTP_H +#define _SCTP_H + +int sctp_init(void); +void sctp_close(int fd); +void sctp_recv(int fd, short event, void *arg); + +#endif + diff --git a/nmdb/tcp.c b/nmdb/tcp.c index 7c86a99..dc98c26 100644 --- a/nmdb/tcp.c +++ b/nmdb/tcp.c @@ -2,7 +2,6 @@ #include /* socket defines */ #include /* socket functions */ #include /* malloc() */ -#include /* perror() */ #include /* uint32_t and friends */ #include /* htonls() and friends */ #include /* INET stuff */ @@ -23,6 +22,7 @@ typedef unsigned char u_char; #include "net-const.h" #include "req.h" #include "parse.h" +#include "log.h" /* TCP socket structure. Used mainly to hold buffers from incomplete @@ -44,13 +44,10 @@ static void tcp_recv(int fd, short event, void *arg); static void process_buf(struct tcp_socket *tcpsock, unsigned char *buf, size_t len); -static void tcp_mini_reply(struct req_info *req, uint32_t reply); +static void tcp_reply_mini(struct req_info *req, uint32_t reply); static void tcp_reply_err(struct req_info *req, uint32_t reply); -static void tcp_reply_get(struct req_info *req, uint32_t reply, +static void tcp_reply_long(struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize); -static void tcp_reply_set(struct req_info *req, uint32_t reply); -static void tcp_reply_del(struct req_info *req, uint32_t reply); -static void tcp_reply_cas(struct req_info *req, uint32_t reply); /* @@ -72,12 +69,9 @@ static void init_req(struct tcp_socket *tcpsock) tcpsock->req.type = REQTYPE_TCP; tcpsock->req.clisa = (struct sockaddr *) &tcpsock->clisa; tcpsock->req.clilen = tcpsock->clilen; - tcpsock->req.mini_reply = tcp_mini_reply; + tcpsock->req.reply_mini = tcp_reply_mini; tcpsock->req.reply_err = tcp_reply_err; - tcpsock->req.reply_get = tcp_reply_get; - tcpsock->req.reply_set = tcp_reply_set; - tcpsock->req.reply_del = tcp_reply_del; - tcpsock->req.reply_cas = tcp_reply_cas; + tcpsock->req.reply_long = tcp_reply_long; } static void rep_send_error(const struct req_info *req, const unsigned int code) @@ -101,7 +95,7 @@ static void rep_send_error(const struct req_info *req, const unsigned int code) r = send(req->fd, minibuf, 4 * 4, 0); if (r < 0) { - perror("rep_send_error() failed"); + errlog("rep_send_error() failed"); } } @@ -109,22 +103,48 @@ static void rep_send_error(const struct req_info *req, const unsigned int code) static int rep_send(const struct req_info *req, const unsigned char *buf, const size_t size) { - int rv; + ssize_t rv, c; if (settings.passive) return 1; - rv = send(req->fd, buf, size, 0); - if (rv < 0) { - rep_send_error(req, ERR_SEND); - return 0; + c = 0; + while (c < size) { + rv = send(req->fd, buf + c, size - c, 0); + + if (rv == size) { + return 1; + } else if (rv < 0) { + if (errno != EAGAIN || errno != EWOULDBLOCK) { + rep_send_error(req, ERR_SEND); + return 0; + } else { + /* With big packets, the receiver window might + * get exhausted and send() would block, but + * as the fd is set in non-blocking mode, it + * returns EAGAIN. This makes us to retry when + * send() fails in this way. + * + * The proper way to fix this would be to add + * an event so we get notified when the fd is + * available for writing, and retry the send; + * but this is complex so leave it for when + * it's really needed. */ + continue; + } + } else if (rv == 0) { + return 1; + } + + c += rv; } + return 1; } /* Send small replies, consisting in only a value. */ -void tcp_mini_reply(struct req_info *req, uint32_t reply) +void tcp_reply_mini(struct req_info *req, uint32_t reply) { /* We use a mini buffer to speedup the small replies, to avoid the * malloc() overhead. */ @@ -149,12 +169,12 @@ void tcp_reply_err(struct req_info *req, uint32_t reply) rep_send_error(req, reply); } -void tcp_reply_get(struct req_info *req, uint32_t reply, +void tcp_reply_long(struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize) { if (val == NULL) { /* miss */ - tcp_mini_reply(req, reply); + tcp_reply_mini(req, reply); } else { unsigned char *buf; size_t bsize; @@ -190,23 +210,6 @@ void tcp_reply_get(struct req_info *req, uint32_t reply, } -void tcp_reply_set(struct req_info *req, uint32_t reply) -{ - tcp_mini_reply(req, reply); -} - - -void tcp_reply_del(struct req_info *req, uint32_t reply) -{ - tcp_mini_reply(req, reply); -} - -void tcp_reply_cas(struct req_info *req, uint32_t reply) -{ - tcp_mini_reply(req, reply); -} - - /* * Main functions for receiving and parsing */ diff --git a/nmdb/tipc.c b/nmdb/tipc.c index eb73bc4..658a056 100644 --- a/nmdb/tipc.c +++ b/nmdb/tipc.c @@ -3,7 +3,6 @@ #include /* socket functions */ #include /* malloc() */ #include /* tipc stuff */ -#include /* perror() */ #include /* uint32_t and friends */ #include /* htonls() and friends */ #include /* memcpy() */ @@ -14,15 +13,13 @@ #include "net-const.h" #include "req.h" #include "parse.h" +#include "log.h" -static void tipc_mini_reply(struct req_info *req, uint32_t reply); +static void tipc_reply_mini(struct req_info *req, uint32_t reply); static void tipc_reply_err(struct req_info *req, uint32_t reply); -static void tipc_reply_get(struct req_info *req, uint32_t reply, +static void tipc_reply_long(struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize); -static void tipc_reply_set(struct req_info *req, uint32_t reply); -static void tipc_reply_del(struct req_info *req, uint32_t reply); -static void tipc_reply_cas(struct req_info *req, uint32_t reply); /* @@ -48,7 +45,7 @@ static void rep_send_error(const struct req_info *req, const unsigned int code) r = sendto(req->fd, minibuf, 3 * 4, 0, req->clisa, req->clilen); if (r < 0) { - perror("rep_send_error() failed"); + errlog("rep_send_error() failed"); } } @@ -71,7 +68,7 @@ static int rep_send(const struct req_info *req, const unsigned char *buf, /* Send small replies, consisting in only a value. */ -static void tipc_mini_reply(struct req_info *req, uint32_t reply) +static void tipc_reply_mini(struct req_info *req, uint32_t reply) { /* We use a mini buffer to speedup the small replies, to avoid the * malloc() overhead. */ @@ -96,12 +93,12 @@ void tipc_reply_err(struct req_info *req, uint32_t reply) rep_send_error(req, reply); } -void tipc_reply_get(struct req_info *req, uint32_t reply, +void tipc_reply_long(struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize) { if (val == NULL) { /* miss */ - tipc_mini_reply(req, reply); + tipc_reply_mini(req, reply); } else { unsigned char *buf; size_t bsize; @@ -133,23 +130,6 @@ void tipc_reply_get(struct req_info *req, uint32_t reply, } -void tipc_reply_set(struct req_info *req, uint32_t reply) -{ - tipc_mini_reply(req, reply); -} - - -void tipc_reply_del(struct req_info *req, uint32_t reply) -{ - tipc_mini_reply(req, reply); -} - -void tipc_reply_cas(struct req_info *req, uint32_t reply) -{ - tipc_mini_reply(req, reply); -} - - /* * Main functions for receiving and parsing */ @@ -175,8 +155,10 @@ int tipc_init(void) return -1; rv = bind(fd, (struct sockaddr *) &srvsa, sizeof(srvsa)); - if (rv < 0) + if (rv < 0) { + close(fd); return -1; + } return fd; } @@ -223,12 +205,9 @@ void tipc_recv(int fd, short event, void *arg) req.type = REQTYPE_TIPC; req.clisa = (struct sockaddr *) &clisa; req.clilen = clilen; - req.mini_reply = tipc_mini_reply; + req.reply_mini = tipc_reply_mini; req.reply_err = tipc_reply_err; - req.reply_get = tipc_reply_get; - req.reply_set = tipc_reply_set; - req.reply_del = tipc_reply_del; - req.reply_cas = tipc_reply_cas; + req.reply_long = tipc_reply_long; /* parse the message */ parse_message(&req, static_buf, rv); diff --git a/nmdb/udp.c b/nmdb/udp.c index 29cb789..c9c51ea 100644 --- a/nmdb/udp.c +++ b/nmdb/udp.c @@ -2,7 +2,6 @@ #include /* socket defines */ #include /* socket functions */ #include /* malloc() */ -#include /* perror() */ #include /* uint32_t and friends */ #include /* htonls() and friends */ #include /* INET stuff */ @@ -15,15 +14,13 @@ #include "net-const.h" #include "req.h" #include "parse.h" +#include "log.h" -static void udp_mini_reply(struct req_info *req, uint32_t reply); +static void udp_reply_mini(struct req_info *req, uint32_t reply); static void udp_reply_err(struct req_info *req, uint32_t reply); -static void udp_reply_get(struct req_info *req, uint32_t reply, +static void udp_reply_long(struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize); -static void udp_reply_set(struct req_info *req, uint32_t reply); -static void udp_reply_del(struct req_info *req, uint32_t reply); -static void udp_reply_cas(struct req_info *req, uint32_t reply); /* @@ -49,7 +46,7 @@ static void rep_send_error(const struct req_info *req, const unsigned int code) r = sendto(req->fd, minibuf, 3 * 4, 0, req->clisa, req->clilen); if (r < 0) { - perror("rep_send_error() failed"); + errlog("rep_send_error() failed"); } } @@ -72,7 +69,7 @@ static int rep_send(const struct req_info *req, const unsigned char *buf, /* Send small replies, consisting in only a value. */ -static void udp_mini_reply(struct req_info *req, uint32_t reply) +static void udp_reply_mini(struct req_info *req, uint32_t reply) { /* We use a mini buffer to speedup the small replies, to avoid the * malloc() overhead. */ @@ -97,12 +94,12 @@ void udp_reply_err(struct req_info *req, uint32_t reply) rep_send_error(req, reply); } -void udp_reply_get(struct req_info *req, uint32_t reply, +void udp_reply_long(struct req_info *req, uint32_t reply, unsigned char *val, size_t vsize) { if (val == NULL) { /* miss */ - udp_mini_reply(req, reply); + udp_reply_mini(req, reply); } else { unsigned char *buf; size_t bsize; @@ -134,23 +131,6 @@ void udp_reply_get(struct req_info *req, uint32_t reply, } -void udp_reply_set(struct req_info *req, uint32_t reply) -{ - udp_mini_reply(req, reply); -} - - -void udp_reply_del(struct req_info *req, uint32_t reply) -{ - udp_mini_reply(req, reply); -} - -void udp_reply_cas(struct req_info *req, uint32_t reply) -{ - udp_mini_reply(req, reply); -} - - /* * Main functions for receiving and parsing */ @@ -180,8 +160,10 @@ int udp_init(void) } rv = bind(fd, (struct sockaddr *) &srvsa, sizeof(srvsa)); - if (rv < 0) + if (rv < 0) { + close(fd); return -1; + } return fd; } @@ -223,12 +205,9 @@ void udp_recv(int fd, short event, void *arg) req.type = REQTYPE_UDP; req.clisa = (struct sockaddr *) &clisa; req.clilen = clilen; - req.mini_reply = udp_mini_reply; + req.reply_mini = udp_reply_mini; req.reply_err = udp_reply_err; - req.reply_get = udp_reply_get; - req.reply_set = udp_reply_set; - req.reply_del = udp_reply_del; - req.reply_cas = udp_reply_cas; + req.reply_long = udp_reply_long; /* parse the message */ parse_message(&req, static_buf, rv); diff --git a/tests/c/1.c b/tests/c/1.c index 518313f..6e1658e 100644 --- a/tests/c/1.c +++ b/tests/c/1.c @@ -54,10 +54,10 @@ int main(int argc, char **argv) timer_start(); for (i = 0; i < times; i++) { r = NGET(db, key, ksize, gval, vsize); - if (r < 0) { + if (r <= -2) { perror("Get"); return 1; - } else if (r == 0) { + } else if (r == -1) { misses++; } } diff --git a/tests/c/2.c b/tests/c/2.c index 79a0de1..5df1d85 100644 --- a/tests/c/2.c +++ b/tests/c/2.c @@ -72,10 +72,10 @@ int main(int argc, char **argv) for (i = 0; i < times; i++) { * (int *) key = i; r = NGET(db, key, ksize, val, vsize); - if (r < 0) { + if (r <= -2) { perror("Get"); return 1; - } else if (r == 0) { + } else if (r == -1) { misses++; } } diff --git a/tests/c/3.c b/tests/c/3.c index b629254..955fbd5 100644 --- a/tests/c/3.c +++ b/tests/c/3.c @@ -66,10 +66,10 @@ int main(int argc, char **argv) * (int *) key = i; r = NGET(db, key, ksize, val, bsize); - if (r < 0) { + if (r <= -2) { perror("Get"); return 1; - } else if (r == 0) { + } else if (r == -1) { misses++; } diff --git a/tests/c/get.c b/tests/c/get.c index 829e0d3..c57efd6 100644 --- a/tests/c/get.c +++ b/tests/c/get.c @@ -57,10 +57,10 @@ int main(int argc, char **argv) for (i = 0; i < times; i++) { * (int *) key = i; r = NGET(db, key, ksize, val, vsize); - if (r < 0) { + if (r <= -2) { perror("Get"); return 1; - } else if (r == 0) { + } else if (r == -1) { misses++; } } diff --git a/tests/c/incr.c b/tests/c/incr.c new file mode 100644 index 0000000..a0604d1 --- /dev/null +++ b/tests/c/incr.c @@ -0,0 +1,67 @@ + +#include +#include +#include +#include +#include + +#include +#include "timer.h" +#include "prototypes.h" + + +int main(int argc, char **argv) +{ + int i, r, times; + unsigned long s_elapsed; + char *key = "k"; + char *initval = "0"; + size_t ksize; + long long int increment; + nmdb_t *db; + + if (argc != 3) { + printf("Usage: incr-* TIMES INCREMENT\n"); + return 1; + } + + times = atoi(argv[1]); + if (times < 1) { + printf("Error: TIMES must be >= 1\n"); + return 1; + } + + increment = strtoll(argv[2], NULL, 10); + + db = nmdb_init(); + if (db == NULL) { + perror("nmdb_init() failed"); + return 1; + } + + NADDSRV(db); + + ksize = strlen(key) + 1; + + /* initial set */ + NSET(db, (unsigned char *) key, ksize, + (unsigned char *) initval, strlen(initval) + 1); + + timer_start(); + for (i = 0; i < times; i++) { + r = NINCR(db, (unsigned char *) key, ksize, increment); + if (r != 2) { + printf("result: %d\n", r); + perror("Incr"); + return 1; + } + } + s_elapsed = timer_stop(); + + printf("%lu\n", s_elapsed); + + nmdb_free(db); + + return 0; +} + diff --git a/tests/c/make.sh b/tests/c/make.sh index a38525c..3b07957 100755 --- a/tests/c/make.sh +++ b/tests/c/make.sh @@ -25,22 +25,23 @@ case "$1" in "clean" ) CLEAN=1 ;; - "help" | "--help" | "-h" | "" ) + "help" | "--help" | "-h" | "" | *) echo $USAGE exit 1 ;; esac; -for p in TIPC TCP UDP MULT; do +for p in TIPC TCP UDP SCTP MULT; do for v in NORMAL CACHE SYNC; do OP=`echo $p-$v | tr '[A-Z]' '[a-z]'` TF="-DUSE_$p=1 -DUSE_$v=1" - echo " * $OP" - for t in 1 2 3 "set" "get" "del"; do + echo " * $OP:" + for t in 1 2 3 "set" "get" "del" "incr"; do + echo " * $t" if [ "$CLEAN" == 1 ]; then - rm -vf $t-$OP + rm -f $t-$OP else cc -lnmdb $ALLCF $TF -o $t-$OP $t.c fi diff --git a/tests/c/prototypes.h b/tests/c/prototypes.h index e0efcc2..e127493 100644 --- a/tests/c/prototypes.h +++ b/tests/c/prototypes.h @@ -8,16 +8,19 @@ #define NSET(...) nmdb_set(__VA_ARGS__) #define NDEL(...) nmdb_del(__VA_ARGS__) #define NCAS(...) nmdb_cas(__VA_ARGS__) + #define NINCR(...) nmdb_incr(__VA_ARGS__) #elif USE_CACHE #define NGET(...) nmdb_cache_get(__VA_ARGS__) #define NSET(...) nmdb_cache_set(__VA_ARGS__) #define NDEL(...) nmdb_cache_del(__VA_ARGS__) #define NCAS(...) nmdb_cache_cas(__VA_ARGS__) + #define NINCR(...) nmdb_cache_incr(__VA_ARGS__) #elif USE_SYNC #define NGET(...) nmdb_get(__VA_ARGS__) #define NSET(...) nmdb_set_sync(__VA_ARGS__) #define NDEL(...) nmdb_del_sync(__VA_ARGS__) #define NCAS(...) nmdb_cas(__VA_ARGS__) + #define NINCR(...) nmdb_incr(__VA_ARGS__) #endif @@ -25,6 +28,8 @@ #define NADDSRV(db) nmdb_add_tcp_server(db, "localhost", -1) #elif USE_UDP #define NADDSRV(db) nmdb_add_udp_server(db, "localhost", -1) +#elif USE_SCTP + #define NADDSRV(db) nmdb_add_sctp_server(db, "localhost", -1) #elif USE_TIPC #define NADDSRV(db) nmdb_add_tipc_server(db, -1) #elif USE_MULT @@ -33,6 +38,7 @@ nmdb_add_tipc_server(db, -1); \ nmdb_add_tcp_server(db, "localhost", -1); \ nmdb_add_udp_server(db, "localhost", -1); \ + nmdb_add_sctp_server(db, "localhost", -1); \ } while (0) #endif diff --git a/tests/python/random1-cache.py b/tests/python/random1-cache.py new file mode 100755 index 0000000..3dca13c --- /dev/null +++ b/tests/python/random1-cache.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python + +import sys +import nmdb +from random import randint, choice + + +class Mismatch (Exception): + pass + + +# network db +ndb = nmdb.Cache() +ndb.add_tipc_server() +ndb.add_tcp_server('localhost') +ndb.add_udp_server('localhost') + +# local db +ldb = {} + +# history of each key +history = {} + +# check decorator +def checked(f): + def newf(k, *args, **kwargs): + try: + return f(k, *args, **kwargs) + except: + print history[k] + raise + newf.__name__ = f.__name__ + return newf + + +# operations +@checked +def set(k, v): + ndb[k] = v + ldb[k] = v + if k not in history: + history[k] = [] + history[k].append((set, k, v)) + +@checked +def get(k): + try: + n = ndb[k] + except KeyError: + del ldb[k] + del history[k] + return 0 + + l = ldb[k] + if l != n: + raise Mismatch, (n, l) + history[k].append((get, k)) + return True + +@checked +def delete(k): + del ldb[k] + try: + del ndb[k] + except KeyError: + pass + history[k].append((delete, k)) + +def find_missing(): + misses = 0 + for k in ldb.keys(): + if not get(k): + misses += 1 + return misses + +# Use integers because the normal random() generates floating point numbers, +# and they can mess up comparisons because of architecture details. +def getrand(): + return randint(0, 1000000000000000000) + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print 'Use: random1-cache.py number_of_keys' + sys.exit(1) + + nkeys = int(sys.argv[1]) + + # fill all the keys + print 'populate' + for i in xrange(nkeys): + set(getrand(), getrand()) + + print 'missing', find_missing() + + lkeys = ldb.keys() + + # operate on them a bit + print 'random operations' + operations = ('set', 'get', 'delete') + for i in xrange(nkeys / 2): + op = choice(operations) + k = choice(lkeys) + if op == 'set': + set(k, getrand()) + elif op == 'get': + get(k) + elif op == 'delete': + delete(k) + lkeys.remove(k) + + print 'missing', find_missing() + + print 'delete' + for k in lkeys: + delete(k) + + print 'missing', find_missing() + + sys.exit(0) + +