INSTALL | 11 ++++ Makefile | 20 +++++++- bindings/python/libjio.c | 110 +++++++++++++++++++++++++++++++------------ bindings/python/setup.py | 4 +- libjio/Makefile | 4 +- libjio/libjio.3 | 22 ++++---- libjio/libjio.h | 2 +- libjio/trans.c | 14 ++++-- samples/Makefile | 2 +- samples/jio1.c | 3 + samples/jio2.c | 3 + tests/behaviour/t_normal.py | 56 +++++++++++++++++++++- tests/stress/jiostress | 21 ++++++-- tests/util/README | 6 ++ tests/util/quick-test-run | 72 ++++++++++++++++++++++++++++ tests/util/wrap-python | 59 +++++++++++++++++++++++ 16 files changed, 346 insertions(+), 63 deletions(-) diff --git a/INSTALL b/INSTALL index 13e479c..08ac159 100644 --- a/INSTALL +++ b/INSTALL @@ -37,3 +37,14 @@ them, you should have libjio already installed. - To build the Python 3 bindings, run "make python3". To install them, run "make python3_install". + +Tests +----- + +Several tests can be found in the "tests/" directory. For practical purposes, +there are two make targets that run a reasonable set of tests against the +built version of the library: + + - To run the standard tests, run "make tests". + - To run the tests with fault injection support, run "make FI=1 tests-fi". + diff --git a/Makefile b/Makefile index c5c046d..5e6a3f8 100644 --- a/Makefile +++ b/Makefile @@ -11,14 +11,19 @@ install: $(MAKE) -C libjio/ install +PY_DEBUG= +ifdef DEBUG + PY_DEBUG=--debug +endif + python2: libjio - cd bindings/python && python setup.py build + cd bindings/python && python setup.py build $(PY_DEBUG) python2_install: python2 cd bindings/python && python setup.py install python3: libjio - cd bindings/python && python3 setup.py build + cd bindings/python && python3 setup.py build $(PY_DEBUG) python3_install: python3 cd bindings/python && python3 setup.py install @@ -30,6 +35,15 @@ preload: preload_install: preload $(MAKE) -C bindings/preload/ install +tests: all python2 python3 + tests/util/quick-test-run normal + +tests-fi: all python2 python3 + @if [ "$(FI)" != "1" ]; then \ + echo "Error: $@ has to be run using: make FI=1 $@"; \ + exit 1; \ + fi + tests/util/quick-test-run fiu clean: $(MAKE) -C libjio/ clean @@ -40,5 +54,5 @@ clean: .PHONY: default all libjio install \ python2 python2_install python3 python3_install \ preload preload_install \ - clean + tests tests-fi clean diff --git a/bindings/python/libjio.c b/bindings/python/libjio.c index b75ba98..f033dfb 100644 --- a/bindings/python/libjio.c +++ b/bindings/python/libjio.c @@ -72,6 +72,26 @@ static void jf_dealloc(jfile_object *fp) PyObject_Del(fp); } + +/* In order to support older Python versions, we use this function to convert + * from ssize_t to a Python long. */ +PyObject *Our_PyLong_FromSsize_t(ssize_t v) +{ + /* Use PyLong_FromSsize_t() if available (Python 3 and >= 2.6), + * otherwise just fall back to the function that fits the size. */ +#if PY_MAJOR_VERSION >= 3 || (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION >= 6) + return PyLong_FromSsize_t(v); +#else + switch (sizeof(ssize_t)) { + case sizeof(long long): + return PyLong_FromLongLong(v); + default: + return PyLong_FromLong(v); + } +#endif +} + + /* fileno */ PyDoc_STRVAR(jf_fileno__doc, "fileno()\n\ @@ -96,14 +116,18 @@ It's a wrapper to jread().\n"); static PyObject *jf_read(jfile_object *fp, PyObject *args) { - long rv; - long len; + ssize_t rv, len; unsigned char *buf; PyObject *r; - if (!PyArg_ParseTuple(args, "i:read", &len)) + if (!PyArg_ParseTuple(args, "n:read", &len)) return NULL; + if (len < 0) { + PyErr_SetString(PyExc_TypeError, "len must be >= 0"); + return NULL; + } + buf = malloc(len); if (buf == NULL) return PyErr_NoMemory(); @@ -136,14 +160,23 @@ It's a wrapper to jpread().\n"); static PyObject *jf_pread(jfile_object *fp, PyObject *args) { - long rv; - long len; + ssize_t rv, len; long long offset; unsigned char *buf; PyObject *r; - if (!PyArg_ParseTuple(args, "iL:pread", &len, &offset)) + if (!PyArg_ParseTuple(args, "nL:pread", &len, &offset)) + return NULL; + + if (len < 0) { + PyErr_SetString(PyExc_TypeError, "len must be >= 0"); + return NULL; + } + + if (offset < 0) { + PyErr_SetString(PyExc_TypeError, "offset must be >= 0"); return NULL; + } buf = malloc(len); if (buf == NULL) @@ -183,7 +216,7 @@ It's a wrapper to jreadv().\n"); #if PY_MAJOR_VERSION >= 3 || (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION >= 6) static PyObject *jf_readv(jfile_object *fp, PyObject *args) { - long rv; + ssize_t rv; PyObject *buffers, *buf; Py_buffer *views = NULL; ssize_t len, pos = 0; @@ -240,7 +273,7 @@ static PyObject *jf_readv(jfile_object *fp, PyObject *args) if (rv < 0) return PyErr_SetFromErrno(PyExc_IOError); - return PyLong_FromLong(rv); + return Our_PyLong_FromSsize_t(rv); error: /* We might get here with pos between 0 and len, so we must release @@ -277,7 +310,7 @@ It's a wrapper to jwrite().\n"); static PyObject *jf_write(jfile_object *fp, PyObject *args) { - long rv; + ssize_t rv; unsigned char *buf; ssize_t len; @@ -291,7 +324,7 @@ static PyObject *jf_write(jfile_object *fp, PyObject *args) if (rv < 0) return PyErr_SetFromErrno(PyExc_IOError); - return PyLong_FromLong(rv); + return Our_PyLong_FromSsize_t(rv); } /* pwrite */ @@ -304,7 +337,7 @@ It's a wrapper to jpwrite().\n"); static PyObject *jf_pwrite(jfile_object *fp, PyObject *args) { - long rv; + ssize_t rv; unsigned char *buf; long long offset; ssize_t len; @@ -312,6 +345,11 @@ static PyObject *jf_pwrite(jfile_object *fp, PyObject *args) if (!PyArg_ParseTuple(args, "s#L:pwrite", &buf, &len, &offset)) return NULL; + if (offset < 0) { + PyErr_SetString(PyExc_TypeError, "offset must be >= 0"); + return NULL; + } + Py_BEGIN_ALLOW_THREADS rv = jpwrite(fp->fs, buf, len, offset); Py_END_ALLOW_THREADS @@ -319,7 +357,7 @@ static PyObject *jf_pwrite(jfile_object *fp, PyObject *args) if (rv < 0) return PyErr_SetFromErrno(PyExc_IOError); - return PyLong_FromLong(rv); + return Our_PyLong_FromSsize_t(rv); } /* writev */ @@ -333,7 +371,7 @@ It's a wrapper to jwritev().\n"); static PyObject *jf_writev(jfile_object *fp, PyObject *args) { - long rv; + ssize_t rv; PyObject *buffers, *buf; ssize_t len, pos; struct iovec *iov; @@ -373,12 +411,12 @@ static PyObject *jf_writev(jfile_object *fp, PyObject *args) if (rv < 0) return PyErr_SetFromErrno(PyExc_IOError); - return PyLong_FromLong(rv); + return Our_PyLong_FromSsize_t(rv); } /* truncate */ PyDoc_STRVAR(jf_truncate__doc, -"truncate(lenght)\n\ +"truncate(length)\n\ \n\ Truncate the file to the given size.\n\ It's a wrapper to jtruncate().\n"); @@ -386,13 +424,13 @@ It's a wrapper to jtruncate().\n"); static PyObject *jf_truncate(jfile_object *fp, PyObject *args) { int rv; - long long lenght; + long long length; - if (!PyArg_ParseTuple(args, "L:truncate", &lenght)) + if (!PyArg_ParseTuple(args, "L:truncate", &length)) return NULL; Py_BEGIN_ALLOW_THREADS - rv = jtruncate(fp->fs, lenght); + rv = jtruncate(fp->fs, length); Py_END_ALLOW_THREADS if (rv != 0) @@ -444,7 +482,7 @@ It's a wrapper to jsync().\n"); static PyObject *jf_jsync(jfile_object *fp, PyObject *args) { - long rv; + int rv; if (!PyArg_ParseTuple(args, ":jsync")) return NULL; @@ -469,7 +507,7 @@ It's a wrapper to jmove_journal().\n"); static PyObject *jf_jmove_journal(jfile_object *fp, PyObject *args) { - long rv; + int rv; char *newpath; if (!PyArg_ParseTuple(args, "s:jmove_journal", &newpath)) @@ -543,7 +581,7 @@ It's a wrapper to jtrans_new().\n"); static PyObject *jf_new_trans(jfile_object *fp, PyObject *args) { - jtrans_object *tp; + jtrans_object *tp = NULL; unsigned int flags = 0; if (!PyArg_ParseTuple(args, "|I:new_trans", &flags)) @@ -658,7 +696,7 @@ It's a wrapper to jtrans_add_w().\n"); static PyObject *jt_add_w(jtrans_object *tp, PyObject *args) { - long rv; + int rv; ssize_t len; long long offset; unsigned char *buf; @@ -666,6 +704,11 @@ static PyObject *jt_add_w(jtrans_object *tp, PyObject *args) if (!PyArg_ParseTuple(args, "s#L:add_w", &buf, &len, &offset)) return NULL; + if (offset < 0) { + PyErr_SetString(PyExc_TypeError, "offset must be >= 0"); + return NULL; + } + rv = jtrans_add_w(tp->ts, buf, len, offset); if (rv < 0) return PyErr_SetFromErrno(PyExc_IOError); @@ -690,9 +733,9 @@ Only available in Python >= 2.6.\n"); #if PY_MAJOR_VERSION >= 3 || (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION >= 6) static PyObject *jt_add_r(jtrans_object *tp, PyObject *args) { - long rv; + int rv; PyObject *py_buf; - unsigned long long offset; + long long offset; Py_buffer *view = NULL, **new_views; if (!PyArg_ParseTuple(args, "OL:add_r", &py_buf, &offset)) @@ -704,6 +747,11 @@ static PyObject *jt_add_r(jtrans_object *tp, PyObject *args) return NULL; } + if (offset < 0) { + PyErr_SetString(PyExc_TypeError, "offset must be >= 0"); + return NULL; + } + view = malloc(sizeof(Py_buffer)); if (view == NULL) return PyErr_NoMemory(); @@ -723,7 +771,7 @@ static PyObject *jt_add_r(jtrans_object *tp, PyObject *args) return PyErr_SetFromErrno(PyExc_IOError); } - new_views = realloc(tp->views, sizeof(Py_buffer *) * tp->nviews + 1); + new_views = realloc(tp->views, sizeof(Py_buffer *) * (tp->nviews + 1)); if (new_views == NULL) { PyBuffer_Release(view); free(view); @@ -758,7 +806,7 @@ It's a wrapper to jtrans_commit().\n"); static PyObject *jt_commit(jtrans_object *tp, PyObject *args) { - long rv; + ssize_t rv; if (!PyArg_ParseTuple(args, ":commit")) return NULL; @@ -770,7 +818,7 @@ static PyObject *jt_commit(jtrans_object *tp, PyObject *args) if (rv < 0) return PyErr_SetFromErrno(PyExc_IOError); - return PyLong_FromLong(rv); + return Our_PyLong_FromSsize_t(rv); } /* rollback */ @@ -782,7 +830,7 @@ It's a wrapper to jtrans_rollback().\n"); static PyObject *jt_rollback(jtrans_object *tp, PyObject *args) { - long rv; + ssize_t rv; if (!PyArg_ParseTuple(args, ":rollback")) return NULL; @@ -794,7 +842,7 @@ static PyObject *jt_rollback(jtrans_object *tp, PyObject *args) if (rv < 0) return PyErr_SetFromErrno(PyExc_IOError); - return PyLong_FromLong(rv); + return Our_PyLong_FromSsize_t(rv); } /* method table */ @@ -854,7 +902,7 @@ static PyObject *jf_open(PyObject *self, PyObject *args) int flags = O_RDONLY; int mode = 0600; unsigned int jflags = 0; - jfile_object *fp; + jfile_object *fp = NULL; flags = O_RDWR; mode = 0600; @@ -900,7 +948,7 @@ It's a wrapper to jfsck().\n"); static PyObject *jf_jfsck(PyObject *self, PyObject *args, PyObject *kw) { int rv; - unsigned int flags; + unsigned int flags = 0; char *name, *jdir = NULL; struct jfsck_result res; PyObject *dict; diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 78ba462..b77fc89 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -15,12 +15,12 @@ libjio = Extension("libjio", # these two allow us to build without having libjio installed, # assuming we're in the libjio source tree include_dirs = ['../../libjio/'], - library_dirs=['../../libjio/'] + library_dirs=['../../libjio/build/'] ) setup( name = 'libjio', - version = '1.01', + version = '1.02', description = "A library for journaled, transactional I/O", author = "Alberto Bertogli", author_email = "albertito@blitiri.com.ar", diff --git a/libjio/Makefile b/libjio/Makefile index c17eaae..e6c8b42 100644 --- a/libjio/Makefile +++ b/libjio/Makefile @@ -60,7 +60,7 @@ endif # library version, used for soname and generated documentation -LIB_VER=1.01 +LIB_VER=1.02 LIB_SO_VER=1 @@ -99,6 +99,7 @@ $O/libjio.so: $O/build-flags $(OBJS) $(LIBS) $(OBJS) -o $O/libjio.so.$(LIB_VER) @echo " LN libjio.so.$(LIB_VER)" @ln -fs libjio.so.$(LIB_VER) $O/libjio.so + @ln -fs libjio.so.$(LIB_VER) $O/libjio.so.$(LIB_SO_VER) $O/libjio.a: $O/build-flags $(OBJS) $(N_AR) cr $@ $(OBJS) @@ -137,6 +138,7 @@ doxygen: clean: rm -f $O/libjio.a $O/libjio.so $O/libjio.so.$(LIB_VER) $O/libjio.pc + rm -f $O/libjio.so.$(LIB_SO_VER) rm -f $(OBJS) $O/jiofsck.o $O/jiofsck rm -f $O/*.bb $O/*.bbg $O/*.da $O/*.gcov $O/*.gcno $O/*.gcda $O/gmon.out rm -f $O/build-flags $O/*.o.mak diff --git a/libjio/libjio.3 b/libjio/libjio.3 index 6e946fb..b92634e 100644 --- a/libjio/libjio.3 +++ b/libjio/libjio.3 @@ -1,6 +1,6 @@ .TH libjio 3 "21/Feb/2004" .SH NAME -libjio - A library for Journaled I/O +libjio \- A library for Journaled I/O .SH SYNOPSIS .nf .B #include @@ -17,7 +17,7 @@ libjio - A library for Journaled I/O .BI " off_t " offset ");" .BI "ssize_t jwritev(jfs_t *" fs ", const struct iovec *" vector "," .BI " int " count ");" -.BI "int jtruncate(jfs_t *" fs ", off_t " lenght ");" +.BI "int jtruncate(jfs_t *" fs ", off_t " length ");" .BI "off_t jlseek(jfs_t *" fs ", off_t " offset ", int " whence ");" .BI "int jclose(jfs_t *" fs ");" @@ -44,17 +44,17 @@ libjio - A library for Journaled I/O int invalid; /* invalid files in the journal directory */ int in_progress; /* transactions in progress */ int broken; /* transactions broken */ - int rollbacked; /* transactions that were rollbacked */ + int rollbacked; /* transactions that were rolled back */ ... }; .BR "enum jfsck_return" " {" J_ESUCCESS = 0, /* Success */ - J_ENOENT = -1, /* No such file or directory */ - J_ENOJOURNAL = -2, /* No journal associated with the given file */ - J_ENOMEM = -3, /* Not enough free memory */ - J_ECLEANUP = -4, /* Error cleaning the journal directory */ - J_EIO = -5, /* I/O error */ + J_ENOENT = \-1, /* No such file or directory */ + J_ENOJOURNAL = \-2, /* No journal associated with the given file */ + J_ENOMEM = \-3, /* Not enough free memory */ + J_ECLEANUP = \-4, /* Error cleaning the journal directory */ + J_EIO = \-5, /* I/O error */ }; @@ -216,9 +216,9 @@ commits the given transaction to disk. After it has returned, write operations have been saved to the disk, and read operations have been read from it. The commit operation is atomic with regards to other read or write operations on different processes, as long as they all access it via libjio. It returns the -number 0 on success, -1 if there was an error but atomic warantees were -preserved, or -2 if there was an error and there is a possible break of atomic -warantees (which is an indication of a severe underlying condition). +number 0 on success, \-1 if there was an error but atomic warranties were +preserved, or \-2 if there was an error and there is a possible break of atomic +warranties (which is an indication of a severe underlying condition). .B jtrans_rollback() reverses a transaction that was applied with diff --git a/libjio/libjio.h b/libjio/libjio.h index 446b81a..20ddba6 100644 --- a/libjio/libjio.h +++ b/libjio/libjio.h @@ -384,7 +384,7 @@ ssize_t jwritev(jfs_t *fs, const struct iovec *vector, int count); /** Truncates the file to the given length. Works just like UNIX ftruncate(2). * * @param fs file to truncate - * @param length lenght to truncate to + * @param length length to truncate to * @returns 0 on success, -1 on error * @see ftruncate(2) * @ingroup unix diff --git a/libjio/trans.c b/libjio/trans.c index 1f7b66f..f4c8328 100644 --- a/libjio/trans.c +++ b/libjio/trans.c @@ -546,10 +546,13 @@ ssize_t jtrans_rollback(struct jtrans *ts) rv = jtrans_commit(newts); exit: - /* free the transaction */ + /* Free the transaction, taking care to set buf to NULL first since + * points to the same address as pdata, which would otherwise make + * jtrans_free() attempt to free it twice. We leave the data at + * curop->pdata since it is freed unconditionally, while the action + * on curop->buf depends on the direction of the transaction. */ for (curop = newts->op; curop != NULL; curop = curop->next) { curop->buf = NULL; - curop->pdata = NULL; } jtrans_free(newts); @@ -631,12 +634,12 @@ struct jfs *jopen(const char *name, int flags, int mode, unsigned int jflags) if (!get_jdir(name, jdir)) goto error_exit; - rv = mkdir(jdir, 0750); + mkdir(jdir, 0750); rv = lstat(jdir, &sinfo); if (rv < 0 || !S_ISDIR(sinfo.st_mode)) goto error_exit; - fs->jdir = (char *) malloc(strlen(jdir) + 1); + fs->jdir = malloc(strlen(jdir) + 1); if (fs->jdir == NULL) goto error_exit; strcpy(fs->jdir, jdir); @@ -733,7 +736,7 @@ int jmove_journal(struct jfs *fs, const char *newpath) oldpath = fs->jdir; snprintf(oldjlockfile, PATH_MAX, "%s/lock", fs->jdir); - fs->jdir = (char *) malloc(strlen(newpath) + 1); + fs->jdir = malloc(strlen(newpath) + 1); if (fs->jdir == NULL) return -1; strcpy(fs->jdir, newpath); @@ -800,6 +803,7 @@ int jclose(struct jfs *fs) free(fs->jdir); pthread_mutex_destroy(&(fs->lock)); + pthread_mutex_destroy(&(fs->ltlock)); free(fs); diff --git a/samples/Makefile b/samples/Makefile index fd3d1f4..7248e73 100644 --- a/samples/Makefile +++ b/samples/Makefile @@ -1,5 +1,5 @@ -CFLAGS := -Wall -O3 -D_XOPEN_SOURCE=500 \ +CFLAGS := -Wall -std=c99 -pedantic -O3 -D_XOPEN_SOURCE=500 \ $(shell getconf LFS_CFLAGS 2>/dev/null) LIBS = -ljio diff --git a/samples/jio1.c b/samples/jio1.c index d45901f..1713a46 100644 --- a/samples/jio1.c +++ b/samples/jio1.c @@ -41,6 +41,9 @@ static int classic(void) if (rv != strlen(STR)) perror("write()"); + if (close(fd)) + perror("close()"); + return 0; } diff --git a/samples/jio2.c b/samples/jio2.c index e720630..08521e3 100644 --- a/samples/jio2.c +++ b/samples/jio2.c @@ -43,6 +43,9 @@ static int classic(void) if (rv != strlen(STR)) perror("write()"); + if (close(fd)) + perror("close()"); + return 0; } diff --git a/tests/behaviour/t_normal.py b/tests/behaviour/t_normal.py index a9ba8df..efa70a1 100644 --- a/tests/behaviour/t_normal.py +++ b/tests/behaviour/t_normal.py @@ -370,8 +370,8 @@ def test_n23(): n = f.name c1 = gencontent(1000) - c2 = gencontent(1000) - c3 = gencontent(1000) + c2 = gencontent(2000) + c3 = gencontent(3000) buf1 = bytearray(0 for i in range(30)) buf2 = bytearray(0 for i in range(100)) @@ -379,7 +379,7 @@ def test_n23(): t = jf.new_trans() t.add_w(c1, 0) t.add_r(buf1, 0) - t.add_w(c2, len(c2)) + t.add_w(c2, len(c1)) t.add_r(buf2, len(c1) - len(buf2) / 2) t.add_w(c3, len(c1) + len(c2)) t.commit() @@ -394,3 +394,53 @@ def test_n23(): cleanup(n) +def test_n24(): + "many jtrans_add_w + many jtrans_add_r" + f, jf = bitmp() + n = f.name + + # just randomly chosen numbers + len_c1 = 1293 + len_c2 = 529 + len_c3 = 1621 + block_len = len_c1 + len_c2 + len_c3 + + t = jf.new_trans() + bufs = [] + for i in range(50): + offset = i * block_len + buf1 = bytearray(0 for _ in range(30)) + buf2 = bytearray(0 for _ in range(100)) + bufs.append((buf1, buf2)) + + # We can't use the bytearray directly because add_w requires + # an immutable object, so we convert it to a string + c1 = str(bytearray(i for _ in range(len_c1))) + c2 = str(bytearray(i for _ in range(len_c2))) + c3 = str(bytearray(i for _ in range(len_c3))) + + t.add_w(c1, offset) + t.add_r(buf1, offset) + t.add_w(c2, offset + len(c1)) + t.add_r(buf2, offset + len(c1) - len(buf2) / 2) + t.add_w(c3, offset + len(c1) + len(c2)) + + t.commit() + + cont = content(n) + for i in range(50): + offset = i * block_len + buf1, buf2 = bufs[i] + c1 = str(bytearray(i for _ in range(len_c1))) + c2 = str(bytearray(i for _ in range(len_c2))) + c3 = str(bytearray(i for _ in range(len_c3))) + + assert cont[offset : offset + block_len] == c1 + c2 + c3 + assert buf1 == c1[:len(buf1)] + assert buf2 == c1[-(len(buf2) / 2):] + c2[:len(buf2) / 2] + + del t + del jf + fsck_verify(n) + cleanup(n) + diff --git a/tests/stress/jiostress b/tests/stress/jiostress index 8f0773e..faa5d06 100755 --- a/tests/stress/jiostress +++ b/tests/stress/jiostress @@ -20,11 +20,10 @@ import libjio try: import fiu except ImportError: - print() - print("Error: unable to load fiu module. This test needs libfiu") - print("support. Please install libfiu and recompile libjio with FI=1.") - print() - raise + # It may be mandatory if the appropriate command line options are + # given, but we will check later once we have parsed them + fiu = None + # # Auxiliary stuff @@ -677,9 +676,21 @@ def main(): print("Error: --fi cannot be used with multiple processes") return 1 + if options.use_fi and not fiu: + print("Error: unable to load fiu module, but use_fi was") + print("enabled. Please install libfiu and recompile libjio with FI=1.") + return 1 + if not options.use_internal_locks: options.do_verify = False + + # Open the file with O_EXCL here to make sure we're the ones creating + # it, to avoid stepping over existing files, and also prevent security + # issues with temporary files. We discard the file descriptor, as + # we're not really interested in it here, we just wanted the file. + _ = os.open(fname, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600) + output = OutputHandler(every = 2) if options.use_internal_locks: lockmgr = LockManager() diff --git a/tests/util/README b/tests/util/README new file mode 100644 index 0000000..0e1d61d --- /dev/null +++ b/tests/util/README @@ -0,0 +1,6 @@ +These are useful scripts for running the tests against the built library +(instead of running them against the installed version). + +The most user-friendly script is quick-test-run, which is safe to run after +building the library. + diff --git a/tests/util/quick-test-run b/tests/util/quick-test-run new file mode 100755 index 0000000..87061d5 --- /dev/null +++ b/tests/util/quick-test-run @@ -0,0 +1,72 @@ +#!/usr/bin/env python + +# This is a convenience script for running some of the other tests without +# manual intervention, as a means of a fast and easy correctness test. +# +# If you are making an intrusive or risky change, please take the time to run +# the other tests by hand with more intensive parameters, and check the +# coverage and use the other tools mentioned in the README. + +import sys +import os +import subprocess +import random + + +# Go to our directory, which we will use to find the other tools +os.chdir(os.path.dirname(sys.argv[0])) + +if len(sys.argv) != 2 or sys.argv[1] not in ("normal", "fiu"): + sys.stderr.write("Usage: %s [normal|fiu]" % + os.path.basename(sys.argv[0])) + sys.exit(1) + +def run_behaviour_tests(test_type): + ret = subprocess.call(["./wrap-python", "2", "../behaviour/runtests", + test_type]) + if ret != 0: + sys.exit(ret) + +def run_stress_tests(nops = 0, nprocs = 0, fi = False, fsize = 20): + # Create a temporary path. We can't use os.tempnam() because it emits + # a warning about a security risk, although it is safe for us because + # of how jiostress opens the file. + tmp_path = "%s/libjio-tests-%d-%d" % ( \ + os.environ.get("TMPDIR", "/tmp"), + os.getpid(), + random.randint(0, 1000000000)) + + args = ["./wrap-python", "3", "../stress/jiostress", + tmp_path, str(fsize)] + if nops: + args += ["-n", str(nops)] + if nprocs: + args += ["-p", str(nprocs)] + if fi: + args += ["--fi"] + + ret = subprocess.call(args) + if ret != 0: + sys.exit(ret) + +if sys.argv[1] == "normal": + print "behaviour tests (normal)" + run_behaviour_tests("normal") + print + print "stress tests (normal)" + run_stress_tests(nops = 50, nprocs = 3) +else: + print "behaviour tests (all)" + run_behaviour_tests("all") + print + print "stress tests (normal)" + run_stress_tests(nops = 50, nprocs = 3) + print + print "stress tests (fiu)" + run_stress_tests(nops = 400, fi = True) + +print +print +print "Tests completed successfuly" +print + diff --git a/tests/util/wrap-python b/tests/util/wrap-python new file mode 100755 index 0000000..930c139 --- /dev/null +++ b/tests/util/wrap-python @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +# Python wrapper, which makes it able to import the built (and not the +# installed) version of libjio. +# +# The first parameter must be the python version (2 or 3) + +import sys +import os +import glob + + +if len(sys.argv) < 2 or sys.argv[1] not in ("2", "3"): + sys.stderr.write("Error: the first argument must be the " + + "version (2 or 3)\n") + sys.exit(1) + +py_ver = sys.argv[1] + + +# Find the path where the library was built and add it to the lookup paths, so +# we run against it +lib_bin = os.path.dirname(sys.argv[0]) + "/../../libjio/build/libjio.so" + +if not os.path.exists(lib_bin): + sys.stderr.write("Can't find library (run make)\n") + sys.exit(1) + +lib_path = os.path.dirname(os.path.abspath(lib_bin)) +os.environ["LD_LIBRARY_PATH"] = ":".join([lib_path, \ + os.environ.get("LD_LIBRARY_PATH", "")]) + + +# Find out the corresponding module path for the desired python version. The +# path must be absolute +mod_bins = glob.glob(os.path.dirname(sys.argv[0]) + + "/../../bindings/python/build/lib*-%s.*/libjio.so" \ + % py_ver) +if not mod_bins: + sys.stderr.write(("Can't find python%s bindings, run " + + "make python%s\n") % (py_ver, py_ver)) + sys.exit(1) + +if len(mod_bins) > 1: + sys.stderr.write("Found too many matching python%s bindings" \ + % py_ver) + sys.exit(1) + +mod_path = os.path.dirname(os.path.abspath(mod_bins[0])) +os.environ["PYTHONPATH"] = ":".join([mod_path, + os.environ.get("PYTHONPATH", "")]) + +if py_ver == '2': + py_bin = "python" +else: + py_bin = "python3" + +os.execvp(py_bin, [py_bin] + sys.argv[2:]) +