.gitignore | 5 + INSTALL | 4 - Make.conf | 23 - Makefile | 58 ++- README | 12 +- UPGRADING | 6 + ansi.c | 2 +- bindings/preload/libjio_preload.c | 2 +- bindings/python/libjio.c | 42 +- bindings/python/setup.py | 4 +- check.c | 29 +- checksum.c | 2 +- common.c | 12 +- common.h | 4 +- doc/LICENSE | 234 ++++---- doc/big_transactions | 9 - doc/guide.lyx | 778 -------------------------- doc/guide.rst | 285 ++++++++++ doc/guide.txt | 378 ------------- doc/jiofsck | 12 - doc/libjio.3 | 307 ++++++----- doc/libjio.html | 444 --------------- doc/libjio.lyx | 433 -------------- doc/libjio.rst | 244 ++++++++ doc/libjio.txt | 305 ---------- doc/{layout => source_layout} | 0 doc/threads | 20 - doc/tids | 83 --- doc/tids.rst | 76 +++ jiofsck.c | 4 +- libjio.h | 12 +- libjio.skel.pc | 12 + tests/performance/Makefile | 8 +- tests/performance/{paralell.c => parallel.c} | 8 +- tests/performance/streaming.c | 2 +- trans.c | 23 +- unix.c | 2 +- utils/exporter | 127 ----- utils/release | 47 -- 39 files changed, 1052 insertions(+), 3006 deletions(-) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7857c16 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.o +*.a +*.so +jiofsck +libjio.pc diff --git a/INSTALL b/INSTALL index fc86de0..69b49bc 100644 --- a/INSTALL +++ b/INSTALL @@ -36,7 +36,3 @@ instead. After installing, you're ready to use the library; you can see how by looking at the manpage with "man libjio". - -If you have any question, suggestion or comment, please send it to the mailing -list, at libjio-devel@auriga.wearlab.de. - diff --git a/Make.conf b/Make.conf deleted file mode 100644 index e4a8eee..0000000 --- a/Make.conf +++ /dev/null @@ -1,23 +0,0 @@ - -VERSION="0.23" - -CC = gcc -CFLAGS += -Wall -O3 -fPIC \ - -D_LARGEFILE_SOURCE=1 -D_LARGEFILE64_SOURCE=1 \ - -D_LFS_LARGEFILE=1 -D_LFS64_LARGEFILE=1 \ - -D_FILE_OFFSET_BITS=64 `getconf LFS_CFLAGS 2>/dev/null` \ - -D_XOPEN_SOURCE=500 - -ifdef DEBUG -CFLAGS += -g -pg -fprofile-arcs -ftest-coverage -endif - -ifdef STRICT -CFLAGS += -ansi -pedantic -endif - -# prefix for installing the binaries -PREFIX=/usr/local - - - diff --git a/Makefile b/Makefile index 6b76965..6bad0ab 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,36 @@ -include Make.conf +VERSION="0.24" + +CFLAGS = -std=c99 -pedantic -Wall -O3 + +MANDATORY_CFLAGS := \ + -D_LARGEFILE_SOURCE=1 -D_LARGEFILE64_SOURCE=1 \ + -D_LFS_LARGEFILE=1 -D_LFS64_LARGEFILE=1 \ + -D_FILE_OFFSET_BITS=64 $(shell getconf LFS_CFLAGS 2>/dev/null) \ + -D_XOPEN_SOURCE=500 + +ALL_CFLAGS += $(CFLAGS) $(MANDATORY_CFLAGS) -fPIC + +ifdef DEBUG +ALL_CFLAGS += -g +endif + +ifdef PROFILE +ALL_CFLAGS += -g -pg -fprofile-arcs -ftest-coverage +endif + + +# prefix for installing the binaries +PREFIX=/usr/local + + +ifneq ($(V), 1) + NICE_CC = @echo " CC $@"; $(CC) + NICE_AR = @echo " AR $@"; $(AR) +else + NICE_CC = $(CC) + NICE_AR = $(AR) +endif # objects to build @@ -8,16 +39,23 @@ OBJS = checksum.o common.o trans.o check.o unix.o ansi.o # rules default: all -all: libjio.so libjio.a jiofsck +all: libjio.so libjio.a libjio.pc jiofsck libjio.so: $(OBJS) - $(CC) -shared -fPIC $(OBJS) -o libjio.so + $(NICE_CC) -shared $(ALL_CFLAGS) $(OBJS) -o libjio.so libjio.a: $(OBJS) - $(AR) cr libjio.a $(OBJS) + $(NICE_AR) cr libjio.a $(OBJS) + +libjio.pc: libjio.skel.pc + @echo "generating libjio.pc" + @cat libjio.skel.pc | \ + sed 's@++PREFIX++@$(PREFIX)@g' | \ + sed 's@++CFLAGS++@$(MANDATORY_CFLAGS)@g' \ + > libjio.pc jiofsck: jiofsck.o libjio.a - $(CC) jiofsck.o libjio.a -lpthread -o jiofsck + $(NICE_CC) $(ALL_CFLAGS) jiofsck.o libjio.a -lpthread -o jiofsck install: all install -d $(PREFIX)/lib @@ -25,6 +63,8 @@ install: all install -m 0644 libjio.a $(PREFIX)/lib install -d $(PREFIX)/include install -m 0644 libjio.h $(PREFIX)/include + install -d $(PREFIX)/lib/pkgconfig + install -m 644 libjio.pc $(PREFIX)/lib/pkgconfig install -d $(PREFIX)/bin install -m 0775 jiofsck $(PREFIX)/bin install -d $(PREFIX)/man/man3 @@ -34,7 +74,7 @@ install: all @echo .c.o: - $(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@ + $(NICE_CC) $(ALL_CFLAGS) $(INCLUDES) -c $< -o $@ python: all @@ -46,7 +86,7 @@ python_install: python preload: all install -d bindings/preload/build/ - $(CC) $(INCLUDES) -Wall -O3 -shared -fPIC \ + $(NICE_CC) $(INCLUDES) -Wall -O3 -shared -fPIC \ -D_XOPEN_SOURCE=500 \ -ldl -lpthread -L. -ljio -I. \ bindings/preload/libjio_preload.c \ @@ -58,8 +98,8 @@ preload_install: preload clean: - rm -f $(OBJS) libjio.a libjio.so jiofsck.o jiofsck - rm -f *.bb *.bbg *.da *.gcov gmon.out + rm -f $(OBJS) libjio.a libjio.so libjio.pc jiofsck.o jiofsck + rm -f *.bb *.bbg *.da *.gcov *.gcno *.gcda gmon.out rm -rf bindings/python/build/ rm -rf bindings/preload/build/ diff --git a/README b/README index 4883ef9..1adc539 100644 --- a/README +++ b/README @@ -1,6 +1,6 @@ libjio - A library for Journaled I/O -Alberto Bertogli (albertogli@telpin.com.ar) +Alberto Bertogli (albertito@blitiri.com.ar) ------------------------------------------- As the name says, this is a simple userspace library to do journaled, @@ -19,17 +19,13 @@ everything is isolated. There are more detailed documents: a programming guide, a brief introduction to the design and inner workings, and the manpage; all in the doc/ directory. -The first two, called 'guide' and 'libjio' respectively, are both in txt and -lyx formats, with HTML, Postscript and PDF versions in the website, which are -not included in the package for space reasons. - To see how to install it, please read the INSTALL file. -It is licensed under the Open Software License version 2.0. +It is licensed under the Open Software License version 3.0. -Comments and patches are always welcome; please send them to the mailing list, -libjio-devel@auriga.wearlab.de. +Comments and patches are always welcome; please send them to my email address, +albertito@blitiri.com.ar. Thanks, Alberto diff --git a/UPGRADING b/UPGRADING index 87d6465..72b9c9e 100644 --- a/UPGRADING +++ b/UPGRADING @@ -11,6 +11,12 @@ take much. If it's mandatory, it will be noted. If you want to see what motivated the changes, see the changelog or just ask. +0.22 -> 0.24 +* The return values of jfsck() have changed, so applications using it need to + be recompiled. +* Python bindings' jfsck(), jfsck_cleanup(), jsync(), jwrite() and jpwrite() + now raise an IOError on failures. + 0.21 -> 0.22 * Applications need to be recompiled due to a change in the jfs structure. diff --git a/ansi.c b/ansi.c index 97a9f44..017bd66 100644 --- a/ansi.c +++ b/ansi.c @@ -1,7 +1,7 @@ /* * libjio - A library for Journaled I/O - * Alberto Bertogli (albertogli@telpin.com.ar) + * Alberto Bertogli (albertito@blitiri.com.ar) * * ANSI C API wrappers */ diff --git a/bindings/preload/libjio_preload.c b/bindings/preload/libjio_preload.c index 1924d4f..d71aa36 100644 --- a/bindings/preload/libjio_preload.c +++ b/bindings/preload/libjio_preload.c @@ -1,7 +1,7 @@ /* * libjio C preloader - * Alberto Bertogli (albertogli@telpin.com.ar) + * Alberto Bertogli (albertito@blitiri.com.ar) * * This generates a shared object that, when prelinked, can be used to make an * existing application to use libjio for UNIX I/O. diff --git a/bindings/python/libjio.c b/bindings/python/libjio.c index 51d7745..eb90e5a 100644 --- a/bindings/python/libjio.c +++ b/bindings/python/libjio.c @@ -1,7 +1,7 @@ /* * Python bindings for libjio - * Alberto Bertogli (albertogli@telpin.com.ar) + * Alberto Bertogli (albertito@blitiri.com.ar) * Aug/2004 */ @@ -110,7 +110,7 @@ static PyObject *jf_read(jfileobject *fp, PyObject *args) if (rv < 0) { r = PyErr_SetFromErrno(PyExc_IOError); } else { - r = PyString_FromStringAndSize(buf, rv); + r = PyString_FromStringAndSize((char *) buf, rv); } free(buf); @@ -147,7 +147,7 @@ static PyObject *jf_pread(jfileobject *fp, PyObject *args) if (rv < 0) { r = PyErr_SetFromErrno(PyExc_IOError); } else { - r = PyString_FromStringAndSize(buf, rv); + r = PyString_FromStringAndSize((char *) buf, rv); } free(buf); @@ -166,7 +166,7 @@ static PyObject *jf_write(jfileobject *fp, PyObject *args) { long rv; unsigned char *buf; - long len; + int len; if (!PyArg_ParseTuple(args, "s#:write", &buf, &len)) return NULL; @@ -175,6 +175,9 @@ static PyObject *jf_write(jfileobject *fp, PyObject *args) rv = jwrite(fp->fs, buf, len); Py_END_ALLOW_THREADS + if (rv < 0) + return PyErr_SetFromErrno(PyExc_IOError); + return PyLong_FromLong(rv); } @@ -191,7 +194,7 @@ static PyObject *jf_pwrite(jfileobject *fp, PyObject *args) long rv; unsigned char *buf; long long offset; - long len; + int len; if (!PyArg_ParseTuple(args, "s#L:pwrite", &buf, &len, &offset)) return NULL; @@ -200,6 +203,9 @@ static PyObject *jf_pwrite(jfileobject *fp, PyObject *args) rv = jpwrite(fp->fs, buf, len, offset); Py_END_ALLOW_THREADS + if (rv < 0) + return PyErr_SetFromErrno(PyExc_IOError); + return PyLong_FromLong(rv); } @@ -280,6 +286,9 @@ static PyObject *jf_jsync(jfileobject *fp, PyObject *args) rv = jsync(fp->fs); Py_END_ALLOW_THREADS + if (rv < 0) + return PyErr_SetFromErrno(PyExc_IOError); + return PyLong_FromLong(rv); } @@ -400,7 +409,7 @@ It's a wrapper to jtrans_add().\n"); static PyObject *jt_add(jtransobject *tp, PyObject *args) { long rv; - long len; + int len; long long offset; unsigned char *buf; @@ -545,14 +554,15 @@ PyDoc_STRVAR(jf_jfsck__doc, \n\ Checks the integrity of the file with the given name, using (optionally) jdir\n\ as the journal directory; returns a dictionary with all the different values\n\ -of the check (equivalent to the 'struct jfsck_result'), or None if there was\n\ -nothing to check.\n\ +of the check (equivalent to the 'struct jfsck_result'). If the path is\n\ +incorrect, or there is no journal associated with it, an IOError will be\n\ +raised.\n\ It's a wrapper to jfsck().\n"); static PyObject *jf_jfsck(PyObject *self, PyObject *args) { int rv; - char *name, *jdir; + char *name, *jdir = NULL; struct jfsck_result res; PyObject *dict; @@ -568,9 +578,12 @@ static PyObject *jf_jfsck(PyObject *self, PyObject *args) Py_END_ALLOW_THREADS if (rv == J_ENOMEM) { + Py_XDECREF(dict); return PyErr_NoMemory(); } else if (rv != 0) { - return Py_None; + Py_XDECREF(dict); + PyErr_SetObject(PyExc_IOError, PyInt_FromLong(rv)); + return NULL; } PyDict_SetItemString(dict, "total", PyLong_FromLong(res.total)); @@ -595,7 +608,7 @@ It's a wrapper to jfsck_cleanup().\n"); static PyObject *jf_jfsck_cleanup(PyObject *self, PyObject *args) { long rv; - char *name, *jdir; + char *name, *jdir = NULL; if (!PyArg_ParseTuple(args, "s|s:jfsck_cleanup", &name, &jdir)) return NULL; @@ -604,6 +617,11 @@ static PyObject *jf_jfsck_cleanup(PyObject *self, PyObject *args) rv = jfsck_cleanup(name, jdir); Py_END_ALLOW_THREADS + if (rv != 1) { + PyErr_SetObject(PyExc_IOError, PyInt_FromLong(rv)); + return NULL; + } + return PyInt_FromLong(rv); } @@ -618,7 +636,7 @@ static PyMethodDef libjio_functions[] = { /* module initialization */ PyDoc_STRVAR(libjio__doc, "libjio is a library to do transactional, journaled I/O\n\ -You can find it at http://users.auriga.wearlab.de/~alb/libjio/\n\ +You can find it at http://blitiri.com.ar/p/libjio/\n\ \n\ Use the open() method to create a file object, and then operate on it.\n\ Please read the documentation for more information.\n"); diff --git a/bindings/python/setup.py b/bindings/python/setup.py index c97044d..3332b82 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -9,8 +9,8 @@ setup( name = 'libjio', description = "A library for journaled I/O", author="Alberto Bertogli", - author_email="albertogli@telpin.com.ar", - url="http://users.auriga.wearlab.de/~alb/libjio", + author_email="albertito@blitiri.com.ar", + url="http://blitiri.com.ar/p/libjio", ext_modules = [libjio] ) diff --git a/check.c b/check.c index 2257d78..c430d14 100644 --- a/check.c +++ b/check.c @@ -1,7 +1,7 @@ /* * libjio - A library for Journaled I/O - * Alberto Bertogli (albertogli@telpin.com.ar) + * Alberto Bertogli (albertito@blitiri.com.ar) * * Recovery functions */ @@ -44,7 +44,7 @@ static int fill_trans(unsigned char *map, off_t len, struct jtrans *ts) p += 4; for (i = 0; i < ts->numops; i++) { - if (len < (p - map) + J_DISKOPHEADSIZE) + if (p + J_DISKOPHEADSIZE > map + len) goto error; op = malloc(sizeof(struct joper)); @@ -60,7 +60,7 @@ static int fill_trans(unsigned char *map, off_t len, struct jtrans *ts) op->offset = *( (uint64_t *) p); p += 8; - if (len < (p - map) + op->len) + if (p + op->len > map + len) goto error; op->buf = (void *) p; @@ -73,7 +73,7 @@ static int fill_trans(unsigned char *map, off_t len, struct jtrans *ts) op->prev = NULL; op->next = NULL; } else { - for(tmp = ts->op; tmp->next != NULL; tmp = tmp->next) + for (tmp = ts->op; tmp->next != NULL; tmp = tmp->next) ; tmp->next = op; op->prev = tmp; @@ -147,12 +147,11 @@ int jfsck(const char *name, const char *jdir, struct jfsck_result *res) goto exit; } } else { - fs.jdir = (char *) malloc(strlen(jdir) + 1); + fs.jdir = strdup(jdir); if (fs.jdir == NULL) { ret = J_ENOMEM; goto exit; } - strcpy(fs.jdir, jdir); } rv = lstat(fs.jdir, &sinfo); @@ -227,10 +226,7 @@ int jfsck(const char *name, const char *jdir, struct jfsck_result *res) * really looping in order (recovering transaction in a * different order as they were applied means instant * corruption) */ - if (!get_jtfile(&fs, i, tname)) { - ret = J_ENOMEM; - goto exit; - } + get_jtfile(&fs, i, tname); tfd = open(tname, O_RDWR | O_SYNC, 0600); if (tfd < 0) { res->invalid++; @@ -248,7 +244,7 @@ int jfsck(const char *name, const char *jdir, struct jfsck_result *res) filelen = lseek(tfd, 0, SEEK_END); /* no overflow problems because we know the transaction size * is limited to SSIZE_MAX */ - map = mmap(0, filelen, PROT_READ, MAP_SHARED, tfd, 0); + map = mmap((void *) 0, filelen, PROT_READ, MAP_SHARED, tfd, 0); if (map == MAP_FAILED) { res->broken++; map = NULL; @@ -360,11 +356,16 @@ int jfsck_cleanup(const char *name, const char *jdir) } /* and remove it */ - unlink(tfile); + if (unlink(tfile) != 0) { + closedir(dir); + return 0; + } } - closedir(dir); + if (closedir(dir) != 0) + return 0; - rmdir(path); + if (rmdir(path) != 0) + return 0; return 1; } diff --git a/checksum.c b/checksum.c index 60e9758..92a9227 100644 --- a/checksum.c +++ b/checksum.c @@ -1,7 +1,7 @@ /* * libjio - A library for Journaled I/O - * Alberto Bertogli (albertogli@telpin.com.ar) + * Alberto Bertogli (albertito@blitiri.com.ar) * * Checksum functions * Based on RFC 1071, "Computing the Internet Checksum" diff --git a/common.c b/common.c index 739b5e4..c37f144 100644 --- a/common.c +++ b/common.c @@ -1,7 +1,7 @@ /* * libjio - A library for Journaled I/O - * Alberto Bertogli (albertogli@telpin.com.ar) + * Alberto Bertogli (albertito@blitiri.com.ar) * * Common functions */ @@ -114,8 +114,10 @@ int get_jdir(const char *filename, char *jdir) base = basename(baset); dirt = strdup(filename); - if (dirt == NULL) + if (dirt == NULL) { + free(baset); return 0; + } dir = dirname(dirt); snprintf(jdir, PATH_MAX, "%s/.%s.jio", dir, base); @@ -126,11 +128,11 @@ int get_jdir(const char *filename, char *jdir) return 1; } -/* build the filename of a given transaction */ -int get_jtfile(struct jfs *fs, unsigned int tid, char *jtfile) +/* build the filename of a given transaction; assumes jtfile can hold at least + * PATH_MAX bytes */ +void get_jtfile(struct jfs *fs, unsigned int tid, char *jtfile) { snprintf(jtfile, PATH_MAX, "%s/%u", fs->jdir, tid); - return 1; } diff --git a/common.h b/common.h index b5861f6..b25ec26 100644 --- a/common.h +++ b/common.h @@ -1,7 +1,7 @@ /* * libjio - A library for Journaled I/O - * Alberto Bertogli (albertogli@telpin.com.ar) + * Alberto Bertogli (albertito@blitiri.com.ar) * * Header for internal functions */ @@ -33,7 +33,7 @@ off_t plockf(int fd, int cmd, off_t offset, off_t len); ssize_t spread(int fd, void *buf, size_t count, off_t offset); ssize_t spwrite(int fd, const void *buf, size_t count, off_t offset); int get_jdir(const char *filename, char *jdir); -int get_jtfile(struct jfs *fs, unsigned int tid, char *jtfile); +void get_jtfile(struct jfs *fs, unsigned int tid, char *jtfile); int checksum(int fd, size_t len, uint32_t *csum); uint32_t checksum_map(uint8_t *map, size_t count); diff --git a/doc/LICENSE b/doc/LICENSE index ef6283f..3a91342 100644 --- a/doc/LICENSE +++ b/doc/LICENSE @@ -1,6 +1,6 @@ This project, 'libjio', is copyrighted by Alberto Bertogli and licensed under -the Open Software License version 2.0 as obtained from www.opensource.org (and +the Open Software License version 3.0 as obtained from www.opensource.org (and included here-in for easy reference) (that license itself is copyrighted by Larry Rosen). @@ -10,163 +10,152 @@ program does NOT in itself make that program considered a derivative work of this Original Work. Alberto Bertogli - 21 February 2004 + 15 July 2007 ------------------------------------------------------------------------- - -Open Software License - v. 2.0 - This Open Software License (the "License") applies to any original work of authorship (the "Original Work") whose owner (the "Licensor") has placed the -following notice immediately following the copyright notice for the Original +following licensing notice adjacent to the copyright notice for the Original Work: - Licensed under the Open Software License version 2.0 - + Licensed under the Open Software License version 3.0 -1) Grant of Copyright License. Licensor hereby grants You a world-wide, -royalty-free, non-exclusive, perpetual, sublicenseable license to do the -following: +1. Grant of Copyright License. Licensor grants You a worldwide, royalty-free, +non-exclusive, sublicensable license, for the duration of the copyright, to do +the following: - a) to reproduce the Original Work in copies; + 1. to reproduce the Original Work in copies, either alone or as part of a + collective work; - b) to prepare derivative works ("Derivative Works") based upon the - Original Work; + 2. to translate, adapt, alter, transform, modify, or arrange the Original + Work, thereby creating derivative works ("Derivative Works") based upon + the Original Work; - c) to distribute copies of the Original Work and Derivative Works to - the public, with the proviso that copies of Original Work or - Derivative Works that You distribute shall be licensed under the - Open Software License; + 3. to distribute or communicate copies of the Original Work and Derivative + Works to the public, with the proviso that copies of Original Work or + Derivative Works that You distribute or communicate shall be licensed + under this Open Software License; - d) to perform the Original Work publicly; and + 4. to perform the Original Work publicly; and - e) to display the Original Work publicly. + 5. to display the Original Work publicly. -2) Grant of Patent License. Licensor hereby grants You a world-wide, -royalty-free, non-exclusive, perpetual, sublicenseable license, under patent -claims owned or controlled by the Licensor that are embodied in the Original -Work as furnished by the Licensor, to make, use, sell and offer for sale the -Original Work and Derivative Works. +2. Grant of Patent License. Licensor grants You a worldwide, royalty-free, +non-exclusive, sublicensable license, under patent claims owned or controlled +by the Licensor that are embodied in the Original Work as furnished by the +Licensor, for the duration of the patents, to make, use, sell, offer for sale, +have made, and import the Original Work and Derivative Works. -3) Grant of Source Code License. The term "Source Code" means the preferred +3. Grant of Source Code License. The term "Source Code" means the preferred form of the Original Work for making modifications to it and all available -documentation describing how to modify the Original Work. Licensor hereby -agrees to provide a machine-readable copy of the Source Code of the Original -Work along with each copy of the Original Work that Licensor distributes. -Licensor reserves the right to satisfy this obligation by placing a -machine-readable copy of the Source Code in an information repository -reasonably calculated to permit inexpensive and convenient access by You for -as long as Licensor continues to distribute the Original Work, and by -publishing the address of that information repository in a notice immediately -following the copyright notice that applies to the Original Work. - -4) Exclusions From License Grant. Neither the names of Licensor, nor the names +documentation describing how to modify the Original Work. Licensor agrees to +provide a machine-readable copy of the Source Code of the Original Work along +with each copy of the Original Work that Licensor distributes. Licensor +reserves the right to satisfy this obligation by placing a machine-readable +copy of the Source Code in an information repository reasonably calculated to +permit inexpensive and convenient access by You for as long as Licensor +continues to distribute the Original Work. + +4. Exclusions From License Grant. Neither the names of Licensor, nor the names of any contributors to the Original Work, nor any of their trademarks or service marks, may be used to endorse or promote products derived from this -Original Work without express prior written permission of the Licensor. -Nothing in this License shall be deemed to grant any rights to trademarks, -copyrights, patents, trade secrets or any other intellectual property of -Licensor except as expressly stated herein. No patent license is granted to -make, use, sell or offer to sell embodiments of any patent claims other than -the licensed claims defined in Section 2. No right is granted to the -trademarks of Licensor even if such marks are included in the Original Work. -Nothing in this License shall be interpreted to prohibit Licensor from -licensing under different terms from this License any Original Work that -Licensor otherwise would have a right to license. - -5) External Deployment. The term "External Deployment" means the use or -distribution of the Original Work or Derivative Works in any way such that the -Original Work or Derivative Works may be used by anyone other than You, -whether the Original Work or Derivative Works are distributed to those persons -or made available as an application intended for use over a computer network. -As an express condition for the grants of license hereunder, You agree that -any External Deployment by You of a Derivative Work shall be deemed a -distribution and shall be licensed to all under the terms of this License, as -prescribed in section 1(c) herein. - -6) Attribution Rights. You must retain, in the Source Code of any Derivative -Works that You create, all copyright, patent or trademark notices from the +Original Work without express prior permission of the Licensor. Except as +expressly stated herein, nothing in this License grants any license to +Licensor's trademarks, copyrights, patents, trade secrets or any other +intellectual property. No patent license is granted to make, use, sell, offer +for sale, have made, or import embodiments of any patent claims other than the +licensed claims defined in Section 2. No license is granted to the trademarks +of Licensor even if such marks are included in the Original Work. Nothing in +this License shall be interpreted to prohibit Licensor from licensing under +terms different from this License any Original Work that Licensor otherwise +would have a right to license. + +5. External Deployment. The term "External Deployment" means the use, +distribution, or communication of the Original Work or Derivative Works in any +way such that the Original Work or Derivative Works may be used by anyone +other than You, whether those works are distributed or communicated to those +persons or made available as an application intended for use over a network. +As an express condition for the grants of license hereunder, You must treat +any External Deployment by You of the Original Work or a Derivative Work as a +distribution under section 1(c). + +6. Attribution Rights. You must retain, in the Source Code of any Derivative +Works that You create, all copyright, patent, or trademark notices from the Source Code of the Original Work, as well as any notices of licensing and any descriptive text identified therein as an "Attribution Notice." You must cause the Source Code for any Derivative Works that You create to carry a prominent Attribution Notice reasonably calculated to inform recipients that You have -modified the Original Work. +modified the Original Work. -7) Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that +7. Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that the copyright in and to the Original Work and the patent rights granted herein by Licensor are owned by the Licensor or are sublicensed to You under the terms of this License with the permission of the contributor(s) of those copyrights and patent rights. Except as expressly stated in the immediately -proceeding sentence, the Original Work is provided under this License on an -"AS IS" BASIS and WITHOUT WARRANTY, either express or implied, including, -without limitation, the warranties of NON-INFRINGEMENT, MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY OF THE -ORIGINAL WORK IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an -essential part of this License. No license to Original Work is granted -hereunder except under this disclaimer. - -8) Limitation of Liability. Under no circumstances and under no legal theory, +preceding sentence, the Original Work is provided under this License on an "AS +IS" BASIS and WITHOUT WARRANTY, either express or implied, including, without +limitation, the warranties of non-infringement, merchantability or fitness for +a particular purpose. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK +IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an essential part of this +License. No license to the Original Work is granted by this License except +under this disclaimer. + +8. Limitation of Liability. Under no circumstances and under no legal theory, whether in tort (including negligence), contract, or otherwise, shall the -Licensor be liable to any person for any direct, indirect, special, -incidental, or consequential damages of any character arising as a result of -this License or the use of the Original Work including, without limitation, -damages for loss of goodwill, work stoppage, computer failure or malfunction, -or any and all other commercial damages or losses. This limitation of -liability shall not apply to liability for death or personal injury resulting -from Licensor's negligence to the extent applicable law prohibits such -limitation. Some jurisdictions do not allow the exclusion or limitation of -incidental or consequential damages, so this exclusion and limitation may not -apply to You. - -9) Acceptance and Termination. If You distribute copies of the Original Work -or a Derivative Work, You must make a reasonable effort under the -circumstances to obtain the express assent of recipients to the terms of this -License. Nothing else but this License (or another written agreement between -Licensor and You) grants You permission to create Derivative Works based upon -the Original Work or to exercise any of the rights granted in Section 1 -herein, and any attempt to do so except under the terms of this License (or -another written agreement between Licensor and You) is expressly prohibited by -U.S. copyright law, the equivalent laws of other countries, and by -international treaty. Therefore, by exercising any of the rights granted to -You in Section 1 herein, You indicate Your acceptance of this License and all -of its terms and conditions. This License shall terminate immediately and you -may no longer exercise any of the rights granted to You by this License upon -Your failure to honor the proviso in Section 1(c) herein. - -10) Termination for Patent Action. This License shall terminate automatically +Licensor be liable to anyone for any indirect, special, incidental, or +consequential damages of any character arising as a result of this License or +the use of the Original Work including, without limitation, damages for loss +of goodwill, work stoppage, computer failure or malfunction, or any and all +other commercial damages or losses. This limitation of liability shall not +apply to the extent applicable law prohibits such limitation. + +9. Acceptance and Termination. If, at any time, You expressly assented to this +License, that assent indicates your clear and irrevocable acceptance of this +License and all of its terms and conditions. If You distribute or communicate +copies of the Original Work or a Derivative Work, You must make a reasonable +effort under the circumstances to obtain the express assent of recipients to +the terms of this License. This License conditions your rights to undertake +the activities listed in Section 1, including your right to create Derivative +Works based upon the Original Work, and doing so without honoring these terms +and conditions is prohibited by copyright law and international treaty. +Nothing in this License is intended to affect copyright exceptions and +limitations (including 'fair use' or 'fair dealing'). This License shall +terminate immediately and You may no longer exercise any of the rights granted +to You by this License upon your failure to honor the conditions in Section +1(c). + +10. Termination for Patent Action. This License shall terminate automatically and You may no longer exercise any of the rights granted to You by this License as of the date You commence an action, including a cross-claim or -counterclaim, for patent infringement (i) against Licensor with respect to a -patent applicable to software or (ii) against any entity with respect to a -patent applicable to the Original Work (but excluding combinations of the -Original Work with other software or hardware). +counterclaim, against Licensor or any licensee alleging that the Original Work +infringes a patent. This termination provision shall not apply for an action +alleging patent infringement by combinations of the Original Work with other +software or hardware. -11) Jurisdiction, Venue and Governing Law. Any action or suit relating to this +11. Jurisdiction, Venue and Governing Law. Any action or suit relating to this License may be brought only in the courts of a jurisdiction wherein the Licensor resides or in which Licensor conducts its primary business, and under the laws of that jurisdiction excluding its conflict-of-law provisions. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any use of the Original Work outside the scope of this License or after its termination shall be -subject to the requirements and penalties of the U.S. Copyright Act, 17 U.S.C. -101 et seq., the equivalent laws of other countries, and international treaty. -This section shall survive the termination of this License. +subject to the requirements and penalties of copyright or patent law in the +appropriate jurisdiction. This section shall survive the termination of this +License. -12) Attorneys Fees. In any action to enforce the terms of this License or +12. Attorneys' Fees. In any action to enforce the terms of this License or seeking damages relating thereto, the prevailing party shall be entitled to recover its costs and expenses, including, without limitation, reasonable attorneys' fees and costs incurred in connection with such action, including any appeal of such action. This section shall survive the termination of this License. -13) Miscellaneous. This License represents the complete agreement concerning -the subject matter hereof. If any provision of this License is held to be +13. Miscellaneous. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. -14) Definition of "You" in This License. "You" throughout this License, +14. Definition of "You" in This License. "You" throughout this License, whether in upper or lower case, means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity that controls, is controlled by, @@ -176,11 +165,24 @@ management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. -15) Right to Use. You may use the Original Work in all ways not otherwise +15. Right to Use. You may use the Original Work in all ways not otherwise restricted or conditioned by this License or by law, and Licensor promises not to interfere with or be responsible for such uses by You. -This license is Copyright (C) 2003 Lawrence E. Rosen. All rights reserved. -Permission is hereby granted to copy and distribute this license without -modification. This license may not be modified without the express written -permission of its copyright owner. +16. Modification of This License. This License is Copyright © 2005 Lawrence +Rosen. Permission is granted to copy, distribute, or communicate this License +without modification. Nothing in this License permits You to modify this +License as applied to the Original Work or to Derivative Works. However, You +may modify the text of this License and copy, distribute or communicate your +modified version (the "Modified License") and apply it to other original works +of authorship subject to the following conditions: (i) You may not indicate in +any way that your Modified License is the "Open Software License" or "OSL" and +you may not use those names in the name of your Modified License; (ii) You +must replace the notice specified in the first paragraph above with the notice +"Licensed under " or with a notice of your own +that is not confusingly similar to the notice in this License; and (iii) You +may not claim that your original works are open source software unless your +Modified License has been approved by Open Source Initiative (OSI) and You +comply with its license review and certification process. + + diff --git a/doc/big_transactions b/doc/big_transactions deleted file mode 100644 index 3c5392d..0000000 --- a/doc/big_transactions +++ /dev/null @@ -1,9 +0,0 @@ - -If you have to create big transactions, instead of creating a huge buffer you -can mmap a temporary file and periodically sync it; and when you're done, just -jtrans_commit() the whole thing. - -This would be a quite efficient way, without any performance penalty and a -very simple approach; I originally thought of doing this on the journal, but -it had many drawbacks that made it much expensive, slower and complex. - diff --git a/doc/guide.lyx b/doc/guide.lyx deleted file mode 100644 index 6eaf9fa..0000000 --- a/doc/guide.lyx +++ /dev/null @@ -1,778 +0,0 @@ -#LyX 1.3 created this file. For more info see http://www.lyx.org/ -\lyxformat 221 -\textclass article -\language english -\inputencoding auto -\fontscheme default -\graphics default -\paperfontsize default -\papersize Default -\paperpackage a4 -\use_geometry 0 -\use_amsmath 0 -\use_natbib 0 -\use_numerical_citations 0 -\paperorientation portrait -\secnumdepth 3 -\tocdepth 3 -\paragraph_separation indent -\defskip medskip -\quotes_language english -\quotes_times 2 -\papercolumns 1 -\papersides 1 -\paperpagestyle default - -\layout Title - -libjio Programmer's Guide -\layout Author - -Alberto Bertogli (albertogli@telpin.com.ar) -\layout Standard - - -\begin_inset LatexCommand \tableofcontents{} - -\end_inset - - -\layout Section - -Introduction -\layout Standard - -This small document attempts serve as a guide to the programmer who wants - to make use of the library. - It's not a replacement for the man page or reading the code; but it's a - good starting point for everyone who wants to get involved with it. -\layout Standard - -The library is not complex to use at all, and the interfaces were designed - to be as intuitive as possible, so the text is structured as a guide to - present the reader all the common structures and functions the way they're - normally used. -\layout Section - -Definitions -\layout Standard - -This is a library which provides a journaled transaction-oriented I/O API. - You've probably read this a hundred times already in the documents, and - if you haven't wondered yet what on earth does this mean you should be - reading something else! -\layout Standard - -We say this is a transaction-oriented API because we make transactions the - center of our operations, and journaled because we use a journal (which - takes the form of a directory with files on it) to guarantee coherency - even after a crash at any point. -\layout Standard - -Here we think a transaction as a list of -\emph on -(buffer, length, offset) -\emph default - to be applied to a file. - That triple is called an -\emph on -operation -\emph default -, so we can say that a transaction represent an ordered group of operations - on the same file -\emph on -. -\layout Standard - -The act of -\emph on -committing -\emph default - a transaction means writing all the elements of the list; and -\emph on -rollbacking -\emph default - means to undo a previous commit, and leave the data just as it was before - doing the commit. -\begin_inset Foot -collapsed false - -\layout Standard - -While all this definitions may seem obvious to some people, it requires - special attention because there are a lot of different definitions, and - it's not that common to see -\begin_inset Quotes eld -\end_inset - -transaction -\begin_inset Quotes erd -\end_inset - - applied to file I/O (it's a term used mostly on database stuff), so it's - important to clarify before continuing. -\end_inset - - -\layout Standard - -It's important to note that the library not only provides a convenient and - easy API to perform this kind of operations, but provides a lot of guarantees - while doing this. - The most relevant and useful is that at any point of time, even if we crash - horribly, a transaction will be either fully applied or not applied at - all. - You should not ever see partial transactions or any kind of data corruption. -\layout Standard - -To achieve this, the library uses what is called a -\emph on -journal -\emph default -, a very vague (and fashionable) term we use to describe a set of auxiliary - files that get created to store temporary data at several stages. - The proper definition and how we use them is outside the scope of this - document, and you as a programmer shouldn't need to deal with it. - In case you're curious, it's described in a bit more detail in another - text which talks about how the library works internally. - Now let's get real. -\layout Section - -The data types -\layout Standard - -To understand any library, it's essential to be confident in the knowledge - of their data structures and how they relate each other. - In libjio we have two basic structures which have a very strong relationship, - and represent the essential objects we deal with. - Note that you normally don't manipulate them directly, because they have - their own initializer functions, but they are the building blocks for the - rest of the text, which, once this is understood, is obvious and self-evident. -\layout Standard - -The first structure we face is -\family typewriter -struct\SpecialChar ~ -jfs -\family default -, called the -\emph on -file structure -\emph default -, and it represents an open file, just like a regular file descriptor or - a -\family typewriter -FILE\SpecialChar ~ -* -\family default -. -\layout Standard - -Then you find -\family typewriter -struct\SpecialChar ~ -jtrans -\family default -, called the -\emph on -transaction structure -\emph default -, which represents a single transaction. - You can have as many transactions as you want, and operate on all of them - simultaneously without problems; the library is entirely thread safe so - there's no need to worry about that. -\layout Section - -The basic functions -\layout Standard - -Now that we've described our data types, let's see how we can really operate - with the library. - -\layout Standard - -First of all, as with regular I/O, you need to open your files. - This is done with -\family typewriter -jopen() -\family default -, which looks a lot like -\family typewriter -open() -\family default - but takes a file structure instead of a file descriptor (this will be very - common among all the functions), and adds a new parameter -\emph on -jflags -\emph default - that can be used to modify some subtle library behaviour we'll see later, - and it's normally not used. -\layout Standard - -We have a happy file structure open now, and the next thing to do would - be to create a transaction. - This is what -\family typewriter -jtrans_init() -\family default - is for: it takes a file structure and a transaction structure and initializes - the latter, leaving it ready to use. -\layout Standard - -So we have our transaction, let's add a write operation to it; to do this - we use -\family typewriter -jtrans_add() -\family default -. - We could keep on adding operations to the transaction by keep on calling - -\family typewriter -jtrans_add() -\family default - as many times as we want. -\layout Standard - -Finally, we decide to apply our transaction to the file, that is, write - all the operations we've added. - And this is the easiest part: we call -\family typewriter -jtrans_commit() -\family default -, and that's it! -\layout Standard - -When we're done using the file, we call -\family typewriter -jclose() -\family default -, just like we call -\family typewriter -close() -\family default -. -\layout Standard - -Let's put it all together and code a nice -\begin_inset Quotes eld -\end_inset - -hello world -\begin_inset Quotes erd -\end_inset - - program (return values are ignored for simplicity): -\layout LyX-Code - -char buf[] = "Hello world!"; -\layout LyX-Code - -struct jfs file; -\layout LyX-Code - -struct jtrans trans; -\newline - -\newline - -\layout LyX-Code - -jopen(&file, "filename", O_RDWR | O_CREAT, 0600, 0); -\layout LyX-Code - -jtrans_init(&file, &trans); -\newline - -\newline - -\layout LyX-Code - -jtrans_add(&trans, buf, strlen(buf), 0); -\newline - -\newline - -\layout LyX-Code - -jtrans_commit(&trans); -\newline - -\newline - -\layout LyX-Code - -jclose(&file); -\layout Standard - -As we've seen, we open the file and initialize the structure with -\family typewriter -jopen() -\family default - (with the parameter -\emph on -jflags -\emph default - being the last 0)and -\family typewriter -jtrans_init() -\family default -, then add an operation with -\family typewriter -jtrans_add() -\family default - (the last 0 is the offset, in this case the beginning of the file), commit - the transaction with -\family typewriter -jtrans_commit() -\family default -, and finally close the file with -\family typewriter -jclose() -\family default -. -\layout Section - -Advanced functions -\layout Subsection - -Interaction with reads -\begin_inset LatexCommand \label{sub:Interaction-with-reads} - -\end_inset - - -\layout Standard - -So far we've seen how to use the library to perform writes, but what about - reads? The only and main issue with reads is that, because we provide transacti -on atomicity, a read must never be able to -\begin_inset Quotes eld -\end_inset - -see -\begin_inset Quotes erd -\end_inset - - a transaction partially applied. - This is achieved internally by using fine-grained file locks; but you shouldn't - mind about it if you use the functions the library gives you because they - take care of all the locking. -\layout Standard - -This set of functions are very similar to the UNIX ones ( -\family typewriter -read() -\family default -, -\family typewriter -readv() -\family default -, etc.); and in fact are named after them: they're called -\family typewriter -jread() -\family default -, -\family typewriter -jreadv() -\family default - and -\family typewriter -jpread() -\family default -; and have the same parameters except for the first one, which instead of - a file descriptor is a file structure -\begin_inset Foot -collapsed false - -\layout Standard - -In fact, this set of functions is a part of what is called the -\begin_inset Quotes eld -\end_inset - -UNIX API -\begin_inset Quotes erd -\end_inset - -, which is described below. -\end_inset - -. - Bear in mind that transactions are only visible by reads -\emph on -after -\emph default - you commit them with -\family typewriter -jtrans_commit() -\family default -. -\layout Subsection - -Rollback -\layout Standard - -There is a very nice and important feature in transactions, that allow them - to be -\begin_inset Quotes eld -\end_inset - -undone -\begin_inset Quotes erd -\end_inset - -, which means that you can undo a transaction and leave the file just as - it was the moment before applying it. - The action of undoing it is called to -\emph on -rollback -\emph default -, and the function is called -\family typewriter -jtrans_rollback() -\family default -, which takes the transaction as the only parameter. -\layout Standard - -Be aware that rollbacking a transaction can be dangerous if you're not careful - and cause you a lot of troubles. - For instance, consider you have two transactions (let's call them 1 and - 2, and assume they were applied in that order) that modify the same offset, - and you rollback transaction 1; then 2 would be lost. - It is not an dangerous operation itself, but its use requires care and - thought. -\layout Subsection - -Integrity checking and recovery -\layout Standard - -An essential part of the library is taking care of recovering from crashes - and be able to assure a file is consistent. - When you're working with the file, this is taking care of; but what when - you first open it? To answer that question, the library provides you with - a function named -\family typewriter -jfsck() -\family default -, which checks the integrity of a file and makes sure that everything is - consistent. - It must be called -\begin_inset Quotes eld -\end_inset - -offline -\begin_inset Quotes erd -\end_inset - -, that is when you are not actively committing and rollbacking; it is normally - done before calling -\family typewriter -jopen() -\family default -. - Another good practise is call jfsck_cleanup() after calling jfsck() to - make sure we're starting up with a fresh clean journal. - After both calls, it is safe to assume that the file is and ready to use. -\layout Standard - -You can also do this manually with an utility named -\emph on -jiofsck -\emph default -, which can be used from the shell to perform the checking and cleanup. -\layout Subsection - -Threads and locking -\layout Standard - -The library is completely safe to use in multithreaded applications; however, - there are some very basic and intuitive locking rules you have to bear - in mind. -\layout Standard - -Most is fully threadsafe so you don't need to worry about concurrency; in - fact, a lot of effort has been put in making paralell operation safe and - fast. -\layout Standard - -You need to care only when opening, closing and checking for integrity. - In practise, that means that you shouldn't call -\family typewriter -jopen() -\family default -, -\family typewriter -jclose() -\family default - in paralell with the same jfs structure, or in the middle of an I/O operation, - just like you do when using the normal UNIX calls. - In the case of -\family typewriter -jfsck() -\family default -, you shouldn't invoke it for the same file more than once at the time; - while it will cope with that situation, it's not recommended. -\layout Standard - -All other operations (commiting a transaction, rollbacking it, adding operations -, etc.) and all the wrappers are safe and don't require any special consideration -s. -\layout Subsection - -Lingering transactions -\layout Standard - -If you need to increase performance, you can use lingering transactions. - In this mode, transactions take up more disk space but allows you to do - the synchronous write only once, making commits much faster. - To use them, just add -\family typewriter -J_LINGER -\family default - to the jflags parameter in -\family typewriter -jopen() -\family default -. - It is very wise to call -\family typewriter -jsync() -\family default - frequently to avoid using up too much space. -\layout Section - -Disk layout -\layout Standard - -The library creates a single directory for each file opened, named after - it. - So if we open a file -\begin_inset Quotes eld -\end_inset - -output -\begin_inset Quotes erd -\end_inset - -, a directory named -\begin_inset Quotes eld -\end_inset - -.output.jio -\begin_inset Quotes erd -\end_inset - - will be created. - We call it the -\emph on -journal directory -\emph default -, and it's used internally by the library to save temporary data; you shouldn't - modify any of the files that are inside it, or move it while it's in use. - It doesn't grow much (it only uses space for transactions that are in the - process of committing) and gets automatically cleaned while working with - it so you can (and should) ignore it. - Besides that, the file you work with has no special modification and is - just like any other file, all the internal stuff is kept isolated on the - journal directory. -\layout Section - -Other APIs -\layout Standard - -We're all used to do things our way, and when we learn something new it's - often better if it looks alike what we already know. - With this in mind, the library comes with two sets of APIs that look a - lot like traditional, well known ones. - Bear in mind that they are not as powerful as the transaction API that - is described above, and they can't provide the same functionality in a - lot of cases; however for a lot of common and simple use patterns they - are good enough. -\layout Subsection - -UNIX API -\layout Standard - -There is a set of functions that emulate the UNIX API ( -\family typewriter -read() -\family default -, -\family typewriter -write() -\family default -, and so on) which make each operation a transaction. - This can be useful if you don't need to have the full power of the transactions - but only to provide guarantees between the different functions. - They are a lot like the normal UNIX functions, but instead of getting a - file descriptor as their first parameter, they get a file structure. - You can check out the manual page to see the details, but they work just - like their UNIX version, only that they preserve atomicity and thread-safety - -\emph on -within each call -\emph default -. -\layout Standard - -In particular, the group of functions related to reading (which was described - above in -\begin_inset LatexCommand \ref{sub:Interaction-with-reads} - -\end_inset - -) are extremely useful because they take care of the locking needed for - the library proper behaviour. - You should use them instead of the regular calls. -\layout Standard - -The full function list is available on the man page and I won't reproduce - it here; however the naming is quite simple: just prepend a 'j' to all - the names: -\family typewriter -jread() -\family default -, -\family typewriter -jwrite() -\family default -, etc. -\layout Subsection - -ANSI C API -\layout Standard - -Besides the UNIX API you can find an ANSI C API, which emulates the traditional - -\family typewriter -fread() -\family default -, -\family typewriter -fwrite() -\family default -, etc. - They're still in development and has not been tested carefully, so I won't - spend time documenting them. - Let me know if you need them. -\layout Section - -Compiling and linking -\layout Standard - -When you want to use your library, besides including the -\emph on - -\begin_inset Quotes eld -\end_inset - -libjio.h -\begin_inset Quotes erd -\end_inset - - -\emph default - header, you have to make sure your application uses the Large File Support - ( -\begin_inset Quotes eld -\end_inset - -LFS -\begin_inset Quotes erd -\end_inset - - from now on), to be able to handle large files properly. - This means that you will have to pass some special standard flags to the - compiler, so your C library uses the same data types as the library. - For instance, on 32-bit platforms (like x86), when using LFS, offsets are - usually 64 bits, as opposed to the usual 32. -\layout Standard - -The library is always built with LFS; however, link it against an application - without LFS support could lead to serious problems because this kind of - size differences and ABI compatibility. -\layout Standard - -The Single Unix Specification standard proposes a simple and practical way - to get the flags you need to pass your C compiler to tell you want to compile - your application with LFS: use a program called -\emph on - -\begin_inset Quotes eld -\end_inset - -getconf -\begin_inset Quotes erd -\end_inset - - -\emph default - which should be called like -\emph on - -\begin_inset Quotes eld -\end_inset - -getconf LFS_CFLAGS -\begin_inset Quotes erd -\end_inset - - -\emph default -, and it outputs the appropiate parameters. - Sadly, not all platforms implement it, so it's also wise to pass -\emph on - -\begin_inset Quotes eld -\end_inset - --D_FILE_OFFSET_BITS=64 -\begin_inset Quotes erd -\end_inset - - -\emph default - just in case. -\layout Standard - -In the end, the command line would be something like: -\layout LyX-Code - -gcc `getconf LFS_CFLAGS` -D_FILE_OFFSET_BITS=64 -\backslash - -\layout LyX-Code - - app.c -ljio -lpthread -o app -\layout Standard - -If you want more detailed information or examples, you can check out how - the library and sample applications get built. -\layout Section - -Where to go from here -\layout Standard - -If you're still interested in learning more, you can find some small and - clean samples are in the -\begin_inset Quotes eld -\end_inset - -samples -\begin_inset Quotes erd -\end_inset - - directory (full.c is a simple and complete one), other more advanced examples - can be found in the web page, as well as modifications to well known software - to make use of the library. - For more information about the inner workings of the library, you can read - the -\begin_inset Quotes eld -\end_inset - -libjio -\begin_inset Quotes erd -\end_inset - - document, and the source code. -\the_end diff --git a/doc/guide.rst b/doc/guide.rst new file mode 100644 index 0000000..3f3bdef --- /dev/null +++ b/doc/guide.rst @@ -0,0 +1,285 @@ + +libjio Programmer's Guide +========================= + +Introduction +------------ + +This small document attempts serve as a guide to the programmer who wants to +make use of the library. It's not a replacement for the man page or reading +the code; but it's a good starting point for everyone who wants to get +involved with it. + +The library is not complex to use at all, and the interfaces were designed to +be as intuitive as possible, so the text is structured as a guide to present +the reader all the common structures and functions the way they're normally +used. + +Definitions +----------- + +This is a library which provides a journaled, transaction-oriented I/O API. +You've probably read this a hundred times already in the documents, and if you +haven't wondered yet what on earth does, this mean you should be reading +something else! + +We say this is a transaction-oriented API because we make transactions the +center of our operations, and journaled because we use a journal (which takes +the form of a directory with files on it) to guarantee coherency even after a +crash at any point. + +In this document, we think of a transaction as a list of *(buffer, length, +offset)* to be applied to a file. That triplet is called an *operation*, so we +can say that a transaction represent an ordered group of operations on the +same file. + +The act of *committing* a transaction means writing all the elements of the +list; and rollbacking means to undo a previous commit, and leave the data just +as it was before doing the commit. While all this definitions may seem obvious +to some people, it requires special attention because there are a lot of +different definitions, and it's not that common to see "transaction" applied +to file I/O, because it's a term used mostly on database stuff. + +The library provides several guarantees, the most relevant and useful being +that at any point of time, even if the machine crash horribly, a transaction +will be either fully applied or not applied at all. + +To achieve this, the library uses what is called a journal, a very vague (and +fashionable) term we use to describe a set of auxiliary files that get created +to store temporary data at several stages. The proper definition and how we +use them is outside the scope of this document, and you as a programmer +shouldn't need to deal with it. In case you're curious, it's described in a +bit more detail in another text which talks about how the library works +internally. Now let's get real. + + +The data types +-------------- + +To understand any library, it's essential to be confident in the knowledge of +their data structures and how they relate each other. libjio has two basic +structures which have a very strong relationship, and represent the essential +objects it deals with. Note that you normally don't manipulate them directly, +because they have their own initializer functions, but they are the building +blocks for the rest of the text, which, once this is understood, should be +obvious and self-evident. + +The first structure we face is *struct jfs*, usually called the file +structure, and it represents an open file, just like a regular file descriptor +or a FILE \*. + +Then you find *struct jtrans*, usually called the transaction structure, which +represents a single transaction. + + +Basic operation +--------------- + +Now that we've described our data types, let's see how we can operate with the +library. + +First of all, as with regular I/O, you need to open your files. This is done +with *jopen()*, which looks a lot like *open()* but takes a file structure +instead of a file descriptor (this will be very common among all the +functions), and adds a new parameter *jflags* that can be used to modify some +subtle library behaviour we'll see later, and is normally not used. + +We have a happy file structure open now, and the next thing to do would be to +create a transaction. This is what *jtrans_init()* is for: it takes a file +structure and a transaction structure and initializes the latter, leaving it +ready to use. + +Now that we have our transaction, let's add a write operation to it; to do +this we use *jtrans_add()*. We could keep on adding operations to the +transaction by keep on calling jtrans_add() as many times as we want. +Operations within a transaction may overlap, and will be applied in order. + +Finally, we decide to apply our transaction to the file, that is, write all +the operations we've added. And this is the easiest part: we call +*jtrans_commit()*, and that's it! + +When we're done using the file, we call *jclose()*, just like we would call +*close()*. + +Let's put it all together and code a nice "hello world" +program (return values are ignored for simplicity):: + + char buf[] = "Hello world!"; + struct jfs file; + struct jtrans trans; + + jopen(&file, "filename", O_RDWR | O_CREAT, 0600, 0); + + jtrans_init(&file, &trans); + jtrans_add(&trans, buf, strlen(buf), 0); + jtrans_commit(&trans); + + jclose(&file); + +As we've seen, we open the file and initialize the structure with *jopen()* +(with the parameter *jflags* being the last 0), create a new transaction with +*jtrans_init()*, then add an operation with *jtrans_add()* (the last 0 is the +offset, in this case the beginning of the file), commit the transaction with +*jtrans_commit()*, and finally close the file with *jclose()*. + +Reading is much easier: the library provides three functions, *jread()*, +*jpread()* and *jreadv()*, that behave exactly like *read()*, *pread()* and +*readv()*, except that they play safe with libjio's writing code. You should +use these to read from files you're writing with libjio. + + +Integrity checking and recovery +------------------------------- + +An essential part of the library is taking care of recovering from crashes and +be able to assure a file is consistent. When you're working with the file, +this is taking care of; but what when you first open it? To answer that +question, the library provides you with a function named *jfsck()*, which +checks the integrity of a file and makes sure that everything is consistent. + +It must be called "offline", that is when you are not actively committing and +rollbacking; it is normally done before calling *jopen()* and is **very, very +important**. Another good practise is to call *jfsck_cleanup()* after calling +*jfsck()*, to make sure we're starting up with a fresh clean journal. After +both calls, it is safe to assume that the file is and ready to use. + +You can also do this manually with an utility named jiofsck, which can be used +from the shell to perform the checking and cleanup. + + +Rollback +-------- + +There is a very nice and important feature in transactions, that allow them to +be "undone", which means that you can undo a transaction and leave the file +just as it was the moment before applying it. The action of undoing it is +called to rollback, and the function is called jtrans_rollback(), which takes +the transaction as the only parameter. + +Be aware that rollbacking a transaction can be dangerous if you're not careful +and cause you a lot of troubles. For instance, consider you have two +transactions (let's call them 1 and 2, and assume they were applied in that +order) that modify the same offset, and you rollback transaction 1; then 2 +would be lost. It is not an dangerous operation itself, but its use requires +care and thought. + + +UNIX-alike API +-------------- + +There is a set of functions that emulate the UNIX API (*read()*, *write()*, +and so on) which make each operation a transaction. This can be useful if you +don't need to have the full power of the transactions but only to provide +guarantees between the different functions. They are a lot like the normal +UNIX functions, but instead of getting a file descriptor as their first +parameter, they get a file structure. You can check out the manual page to see +the details, but they work just like their UNIX version, only that they +preserve atomicity and thread-safety within each call. + +In particular, the group of functions related to reading (which was described +above in `Basic operation`_) are extremely useful because they take care of +the locking needed for the library proper behaviour. You should use them +instead of the regular calls. + +The full function list is available on the man page and I won't reproduce it +here; however the naming is quite simple: just prepend a 'j' to all the names: +*jread()*, *jwrite()*, etc. + + +Threads and locking +------------------- + +The library is completely safe to use in multithreaded applications; however, +there are some very basic and intuitive locking rules you have to bear in +mind. + +Most is fully threadsafe so you don't need to worry about concurrency; in +fact, a lot of effort has been put in making parallel operation safe and fast. + +You need to care only when opening, closing and checking for integrity. In +practise, that means that you shouldn't call *jopen()*, *jclose()* in parallel +with the same jfs structure, or in the middle of an I/O operation, just like +you do when using the normal UNIX calls. In the case of *jfsck()*, you +shouldn't invoke it for the same file more than once at the time; while it +will cope with that situation, it's not recommended. + +All other operations (commiting a transaction, rollbacking it, adding +operations, etc.) and all the wrappers are safe and don't require any special +considerations. + + +Lingering transactions +---------------------- + +If you need to increase performance, you can use lingering transactions. In +this mode, transactions take up more disk space but allows you to do the +synchronous write only once, making commits much faster. To use them, just add +*J_LINGER* to the *jflags* parameter in *jopen()*. You should call *jsync()* +frequently to avoid using up too much space. + + +Disk layout +----------- + +The library creates a single directory for each file opened, named after it. +So if we open a file *output*, a directory named *.output.jio* will be +created. We call it the journal directory, and it's used internally by the +library to save temporary data; **you shouldn't modify any of the files that +are inside it, nor move it while it's in use**. + +It doesn't grow much (it only uses space for transactions that are in the +process of committing) and gets automatically cleaned while working with it so +you can (and should) ignore it. Besides that, the file you work with has no +special modification and is just like any other file, all the internal stuff +is kept isolated on the journal directory. + + +ANSI C alike API +---------------- + +Besides the UNIX-alike API you can find an ANSI C alike API, which emulates +the traditional *fread()*, *fwrite()*, etc. It's still in development and has +not been tested carefully, so I won't spend time documenting them. Let me know +if you need them. + + +Compiling and linking +--------------------- + +When you want to use your library, besides including the "libjio.h" header, +you have to make sure your application uses the Large File Support ("LFS" from +now on), to be able to handle large files properly. This means that you will +have to pass some special standard flags to the compiler, so your C library +uses the same data types as the library. For instance, on 32-bit platforms +(like x86), when using LFS, offsets are usually 64 bits, as opposed to the +usual 32. + +The library is always built with LFS; however, link it against an application +without LFS support could lead to serious problems because this kind of size +differences and ABI compatibility. + +The Single Unix Specification standard proposes a simple and practical way to +get the flags you need to pass your C compiler to tell you want to compile +your application with LFS: use a program called "getconf" which should be +called like "getconf LFS_CFLAGS", and it outputs the appropiate parameters. +Sadly, not all platforms implement it, so it's also wise to pass +"-D_FILE_OFFSET_BITS=64" just in case. + +In the end, the command line would be something like:: + + gcc `getconf LFS_CFLAGS` -D_FILE_OFFSET_BITS=64 app.c -ljio -o app + +If you want more detailed information or examples, you can check out how the +library and sample applications get built. + + +Where to go from here +--------------------- + +If you're still interested in learning more, you can find some small and clean +samples are in the "samples" directory (full.c is a simple and complete one), +other more advanced examples can be found in the web page, as well as +modifications to well known software to make use of the library. For more +information about the inner workings of the library, you can read the "libjio" +document, and the source code. + diff --git a/doc/guide.txt b/doc/guide.txt deleted file mode 100644 index a3d9dfd..0000000 --- a/doc/guide.txt +++ /dev/null @@ -1,378 +0,0 @@ -libjio Programmer's Guide - -Alberto Bertogli (albertogli@telpin.com.ar) - -Table of Contents - -1 Introduction -2 Definitions -3 The data types -4 The basic functions -5 Advanced functions - 5.1 Interaction with reads - 5.2 Rollback - 5.3 Integrity checking and recovery - 5.4 Threads and locking - 5.5 Lingering transactions -6 Disk layout -7 Other APIs - 7.1 UNIX API - 7.2 ANSI C API -8 Compiling and linking -9 Where to go from here - - - -1 Introduction - -This small document attempts serve as a guide to the -programmer who wants to make use of the library. It's -not a replacement for the man page or reading the code; -but it's a good starting point for everyone who wants -to get involved with it. - -The library is not complex to use at all, and the -interfaces were designed to be as intuitive as -possible, so the text is structured as a guide to -present the reader all the common structures and -functions the way they're normally used. - -2 Definitions - -This is a library which provides a journaled -transaction-oriented I/O API. You've probably read this -a hundred times already in the documents, and if you -haven't wondered yet what on earth does this mean you -should be reading something else! - -We say this is a transaction-oriented API because we -make transactions the center of our operations, and -journaled because we use a journal (which takes the -form of a directory with files on it) to guarantee -coherency even after a crash at any point. - -Here we think a transaction as a list of (buffer, -length, offset) to be applied to a file. That triple is -called an operation, so we can say that a transaction -represent an ordered group of operations on the same file. - -The act of committing a transaction means writing all -the elements of the list; and rollbacking means to undo -a previous commit, and leave the data just as it was -before doing the commit.While all this definitions may seem obvious to some -people, it requires special attention because there are -a lot of different definitions, and it's not that -common to see "transaction" applied to file I/O (it's a -term used mostly on database stuff), so it's important -to clarify before continuing. - -It's important to note that the library not only -provides a convenient and easy API to perform this kind -of operations, but provides a lot of guarantees while -doing this. The most relevant and useful is that at any -point of time, even if we crash horribly, a transaction -will be either fully applied or not applied at all. You -should not ever see partial transactions or any kind of -data corruption. - -To achieve this, the library uses what is called a -journal, a very vague (and fashionable) term we use to -describe a set of auxiliary files that get created to -store temporary data at several stages. The proper -definition and how we use them is outside the scope of -this document, and you as a programmer shouldn't need -to deal with it. In case you're curious, it's described -in a bit more detail in another text which talks about -how the library works internally. Now let's get real. - -3 The data types - -To understand any library, it's essential to be -confident in the knowledge of their data structures and -how they relate each other. In libjio we have two basic -structures which have a very strong relationship, and -represent the essential objects we deal with. Note that -you normally don't manipulate them directly, because -they have their own initializer functions, but they are -the building blocks for the rest of the text, which, -once this is understood, is obvious and self-evident. - -The first structure we face is struct jfs, called the -file structure, and it represents an open file, just -like a regular file descriptor or a FILE *. - -Then you find struct jtrans, called the transaction -structure, which represents a single transaction. You -can have as many transactions as you want, and operate -on all of them simultaneously without problems; the -library is entirely thread safe so there's no need to -worry about that. - -4 The basic functions - -Now that we've described our data types, let's see how -we can really operate with the library. - -First of all, as with regular I/O, you need to open -your files. This is done with jopen(), which looks a -lot like open() but takes a file structure instead of a -file descriptor (this will be very common among all the -functions), and adds a new parameter jflags that can be -used to modify some subtle library behaviour we'll see -later, and it's normally not used. - -We have a happy file structure open now, and the next -thing to do would be to create a transaction. This is -what jtrans_init() is for: it takes a file structure -and a transaction structure and initializes the latter, -leaving it ready to use. - -So we have our transaction, let's add a write operation -to it; to do this we use jtrans_add(). We could keep on -adding operations to the transaction by keep on calling -jtrans_add() as many times as we want. - -Finally, we decide to apply our transaction to the -file, that is, write all the operations we've added. -And this is the easiest part: we call jtrans_commit(), -and that's it! - -When we're done using the file, we call jclose(), just -like we call close(). - -Let's put it all together and code a nice "hello world" -program (return values are ignored for simplicity): - -char buf[] = "Hello world!"; - -struct jfs file; - -struct jtrans trans; - - - -jopen(&file, "filename", O_RDWR | O_CREAT, 0600, 0); - -jtrans_init(&file, &trans); - - - -jtrans_add(&trans, buf, strlen(buf), 0); - - - -jtrans_commit(&trans); - - - -jclose(&file); - -As we've seen, we open the file and initialize the -structure with jopen() (with the parameter jflags being -the last 0)and jtrans_init(), then add an operation -with jtrans_add() (the last 0 is the offset, in this -case the beginning of the file), commit the transaction -with jtrans_commit(), and finally close the file with jclose(). - -5 Advanced functions - -5.1 Interaction with reads - -So far we've seen how to use the library to perform -writes, but what about reads? The only and main issue -with reads is that, because we provide transaction -atomicity, a read must never be able to "see" a -transaction partially applied. This is achieved -internally by using fine-grained file locks; but you -shouldn't mind about it if you use the functions the -library gives you because they take care of all the locking. - -This set of functions are very similar to the UNIX ones -(read(), readv(), etc.); and in fact are named after -them: they're called jread(), jreadv() and jpread(); -and have the same parameters except for the first one, -which instead of a file descriptor is a file structureIn fact, this set of functions is a part of what is -called the "UNIX API", which is described below. -. Bear in mind that transactions are only visible by -reads after you commit them with jtrans_commit(). - -5.2 Rollback - -There is a very nice and important feature in -transactions, that allow them to be "undone", which means -that you can undo a transaction and leave the file just -as it was the moment before applying it. The action of -undoing it is called to rollback, and the function is -called jtrans_rollback(), which takes the transaction -as the only parameter. - -Be aware that rollbacking a transaction can be -dangerous if you're not careful and cause you a lot of -troubles. For instance, consider you have two -transactions (let's call them 1 and 2, and assume they -were applied in that order) that modify the same -offset, and you rollback transaction 1; then 2 would be -lost. It is not an dangerous operation itself, but its -use requires care and thought. - -5.3 Integrity checking and recovery - -An essential part of the library is taking care of -recovering from crashes and be able to assure a file is -consistent. When you're working with the file, this is -taking care of; but what when you first open it? To -answer that question, the library provides you with a -function named jfsck(), which checks the integrity of a -file and makes sure that everything is consistent. It -must be called "offline", that is when you are not -actively committing and rollbacking; it is normally -done before calling jopen(). Another good practise is -call jfsck_cleanup() after calling jfsck() to make sure -we're starting up with a fresh clean journal. After -both calls, it is safe to assume that the file is and -ready to use. - -You can also do this manually with an utility named -jiofsck, which can be used from the shell to perform -the checking and cleanup. - -5.4 Threads and locking - -The library is completely safe to use in multithreaded -applications; however, there are some very basic and -intuitive locking rules you have to bear in mind. - -Most is fully threadsafe so you don't need to worry -about concurrency; in fact, a lot of effort has been -put in making paralell operation safe and fast. - -You need to care only when opening, closing and -checking for integrity. In practise, that means that -you shouldn't call jopen(), jclose() in paralell with -the same jfs structure, or in the middle of an I/O -operation, just like you do when using the normal UNIX -calls. In the case of jfsck(), you shouldn't invoke it -for the same file more than once at the time; while it -will cope with that situation, it's not recommended. - -All other operations (commiting a transaction, -rollbacking it, adding operations, etc.) and all the -wrappers are safe and don't require any special considerations. - -5.5 Lingering transactions - -If you need to increase performance, you can use -lingering transactions. In this mode, transactions take -up more disk space but allows you to do the synchronous -write only once, making commits much faster. To use -them, just add J_LINGER to the jflags parameter in -jopen(). It is very wise to call jsync() frequently to -avoid using up too much space. - -6 Disk layout - -The library creates a single directory for each file -opened, named after it. So if we open a file "output", a -directory named ".output.jio" will be created. We call it -the journal directory, and it's used internally by the -library to save temporary data; you shouldn't modify -any of the files that are inside it, or move it while -it's in use. It doesn't grow much (it only uses space -for transactions that are in the process of committing) -and gets automatically cleaned while working with it so -you can (and should) ignore it. Besides that, the file -you work with has no special modification and is just -like any other file, all the internal stuff is kept -isolated on the journal directory. - -7 Other APIs - -We're all used to do things our way, and when we learn -something new it's often better if it looks alike what -we already know. With this in mind, the library comes -with two sets of APIs that look a lot like traditional, -well known ones. Bear in mind that they are not as -powerful as the transaction API that is described -above, and they can't provide the same functionality in -a lot of cases; however for a lot of common and simple -use patterns they are good enough. - -7.1 UNIX API - -There is a set of functions that emulate the UNIX API -(read(), write(), and so on) which make each operation -a transaction. This can be useful if you don't need to -have the full power of the transactions but only to -provide guarantees between the different functions. -They are a lot like the normal UNIX functions, but -instead of getting a file descriptor as their first -parameter, they get a file structure. You can check out -the manual page to see the details, but they work just -like their UNIX version, only that they preserve -atomicity and thread-safety within each call. - -In particular, the group of functions related to -reading (which was described above in [sub:Interaction-with-reads]) are extremely -useful because they take care of the locking needed for -the library proper behaviour. You should use them -instead of the regular calls. - -The full function list is available on the man page and -I won't reproduce it here; however the naming is quite -simple: just prepend a 'j' to all the names: jread(), -jwrite(), etc. - -7.2 ANSI C API - -Besides the UNIX API you can find an ANSI C API, which -emulates the traditional fread(), fwrite(), etc. -They're still in development and has not been tested -carefully, so I won't spend time documenting them. Let -me know if you need them. - -8 Compiling and linking - -When you want to use your library, besides including -the "libjio.h" header, you have to make sure your -application uses the Large File Support ("LFS" from now -on), to be able to handle large files properly. This -means that you will have to pass some special standard -flags to the compiler, so your C library uses the same -data types as the library. For instance, on 32-bit -platforms (like x86), when using LFS, offsets are -usually 64 bits, as opposed to the usual 32. - -The library is always built with LFS; however, link it -against an application without LFS support could lead -to serious problems because this kind of size -differences and ABI compatibility. - -The Single Unix Specification standard proposes a -simple and practical way to get the flags you need to -pass your C compiler to tell you want to compile your -application with LFS: use a program called "getconf" -which should be called like "getconf LFS_CFLAGS", and it -outputs the appropiate parameters. Sadly, not all -platforms implement it, so it's also wise to pass " --D_FILE_OFFSET_BITS=64" just in case. - -In the end, the command line would be something like: - -gcc `getconf LFS_CFLAGS` -D_FILE_OFFSET_BITS=64 \ - - app.c -ljio -lpthread -o app - -If you want more detailed information or examples, you -can check out how the library and sample applications -get built. - -9 Where to go from here - -If you're still interested in learning more, you can -find some small and clean samples are in the "samples" -directory (full.c is a simple and complete one), other -more advanced examples can be found in the web page, as -well as modifications to well known software to make -use of the library. For more information about the -inner workings of the library, you can read the "libjio" -document, and the source code. diff --git a/doc/jiofsck b/doc/jiofsck deleted file mode 100644 index b42aeb2..0000000 --- a/doc/jiofsck +++ /dev/null @@ -1,12 +0,0 @@ - -Note that jfsck does not warantee that all the transactions are fully -completed, it can only do so if you run it without any other process accessing -the journal. - -If you want to see this, you can take a look at the struct jfsck_result. It -include a field named in_progress which tell the number of transactions that -were in progress at the moment of checking, and as such weren't checked. - -Be aware that the counter is not atomic, as two checkers can be running at the -same time. - diff --git a/doc/libjio.3 b/doc/libjio.3 index c50c85c..414f47f 100644 --- a/doc/libjio.3 +++ b/doc/libjio.3 @@ -1,78 +1,19 @@ .TH libjio 3 "21/Feb/2004" .SH NAME libjio - A library for Journaled I/O - -.SH FUNCTIONS - +.SH SYNOPSYS +.nf .B #include -.BI "int jopen(struct jfs *" fs ", const char *" name ", int " flags ", int " mode ", int " jflags " ); - -.BI "ssize_t jread(struct jfs *" fs ", void *" buf ", size_t " count " ); - -.BI "ssize_t jpread(struct jfs *" fs ", void *" buf ", size_t " count ", off_t " offset " ); - -.BI "ssize_t jreadv(struct jfs *" fs ", struct iovec *" vector ", int " count " ); - -.BI "ssize_t jwrite(struct jfs *" fs ", const void *" buf ", size_t " count " ); - -.BI "ssize_t jpwrite(struct jfs *" fs ", const void *" buf ", size_t " count ", off_t " offset " ); - -.BI "ssize_t jwritev(struct jfs *" fs ", const struct iovec *" vector ", int " count " ); - -.BI "int jtruncate(struct jfs *" fs ", off_t " lenght " ); +.B struct jfs; -.BI "int jclose(struct jfs *" fs " ); - -.BI "void jtrans_init(struct jfs *" fs " ,struct jtrans *" ts " ); - -.BI "int jtrans_commit(struct jtrans *" ts " ); - -.BI "int jtrans_add(struct jtrans *" ts ", const void * " buf ", size_t " count ", off_t " offset " ); - -.BI "int jtrans_rollback(struct jtrans *" ts " ); - -.BI "void jtrans_free(struct jtrans *" ts " ); - -.BI "int jfsck(const char *" name ", struct jfsck_result *" res " ); - -.BI "int jfsck_cleanup(const char *" name" );" - -.SH STRUCTURES -.PP -.br -.nf -.in +2n -struct jfs { - int fd; /* main file descriptor */ - char *name; /* and its name */ - int jfd; /* journal's lock file descriptor */ - int flags; /* journal mode options used in jopen() */ - pthread_mutex_t lock; /* a soft lock used in some operations */ +.BR "struct jtrans " { ... -}; -.FI -.in -2n - -.PP -.br -.nf -.in +2n -struct jtrans { - struct jfs *fs; /* journal file structure to operate on */ - char *name; /* name of the transaction file */ - int id; /* transaction id */ - int flags; /* misc flags */ + unsigned int flags; ... }; -.FI -.in -2n -.PP -.br -.nf -.in +2n -struct jfsck_result { +.BR "struct jfsck_result" " {" int total; /* total transactions files we looked at */ int invalid; /* invalid files in the journal directory */ int in_progress; /* transactions in progress */ @@ -81,104 +22,175 @@ struct jfsck_result { int rollbacked; /* transactions that were rollbacked */ ... }; -.FI -.in -2n + +.BI "int jopen(struct jfs *" fs ", const char *" name "," +.BI " int " flags ", int " mode ", int " jflags ");" +.BI "ssize_t jread(struct jfs *" fs ", void *" buf ", size_t " count ");" +.BI "ssize_t jpread(struct jfs *" fs ", void *" buf ", size_t " count "," +.BI " off_t " offset ");" +.BI "ssize_t jreadv(struct jfs *" fs ", struct iovec *" vector "," +.BI " int " count ");" +.BI "ssize_t jwrite(struct jfs *" fs ", const void *" buf ", size_t " count ");" +.BI "ssize_t jpwrite(struct jfs *" fs ", const void *" buf ", size_t " count "," +.BI " off_t " offset ");" +.BI "ssize_t jwritev(struct jfs *" fs ", const struct iovec *" vector "," +.BI " int " count ");" +.BI "int jtruncate(struct jfs *" fs ", off_t " lenght ");" + +.BI "int jclose(struct jfs *" fs ");" + +.BI "void jtrans_init(struct jfs *" fs " ,struct jtrans *" ts ");" +.BI "int jtrans_commit(struct jtrans *" ts ");" +.BI "int jtrans_add(struct jtrans *" ts ", const void * " buf "," +.BI " size_t " count ", off_t " offset ");" +.BI "int jtrans_rollback(struct jtrans *" ts ");" +.BI "void jtrans_free(struct jtrans *" ts ");" + +.BI "int jfsck(const char *" name ", struct jfsck_result *" res ");" +.BI "int jfsck_cleanup(const char *" name ");" .SH DESCRIPTION libjio is a library to do transaction-oriented journaled I/O. This manpage -describes it's C API very briefly, further information can be found in the +describes its C API very briefly, further information can be found in the documentation that comes along with the library itself, or on the web at -http://auriga.wearlab.de/~alb/libjio. - -We can group the functions into three groups: the common functions, the basic -functions and the UNIX API. +http://blitiri.com.ar/p/libjio. -The common functions provide functionality common to the other two: jopen to -open files to use them with the library, and jfsck and jfsck_cleanup to -provide integrity checking. +Functions can be grouped in three different groups: the common functions, the +UNIX-alike API, and the basic functions. -The basic functions consists of jtrans_commit, jtrans_add and jtrans_rollback. -They provide a method for manipulating transactions, which are defined in the -jtrans structure (described above). +The common functions provide functionality common to the other two: +.B jopen() +to open files in order to use them with the library, and +.BR "jfsck() " and " jfsck_cleanup()" +to provide integrity checking. The second group mimics somehow the traditional UNIX API by providing similar interfaces to read(), write(), and their friends. -.SH Common functions +The basic functions consists of +.BR "jtrans_commit()" , " jtrans_add() " and " jtrans_rollback()" . +They provide a method for manipulating transactions, which are defined in the +.IR "jtrans structure" " (described above)." + +.SS STRUCTURES + +Both +.IR "struct jfs" " and " "struct jtrans" +are meant to be treated as opaque types, except for the fields documented +above, which you should treat as read-only. + +.B struct jfsck_result +holds the results of a +.B jfsck() +run, see below for details. + +.SS COMMON FUNCTIONS Most functions reference somehow the structures described avobe, specially -struct jfs and struct jtrans. They represent a file to operate on and a single -transaction, respectively. To open a file, you should use the jopen() call, -which is just like the normal open() call but affects a pointer to a struct -jfs. To close a file, use jclose(). They're exactly like the open() and -close() functions but use a struct jfs instead of a file descriptor; take a -look at their manpages if you have any doubts about how to use them. - -There are two functions that differs from the rest, which are jfsck() and -jfsck_cleanup(). - -The first one, jfsck(), is used to perform journal checking and recovery in -case of a crash. It must be performed when nobody else is using the file (like -in the case of a filesystem which can't be mounted), and it returns 0 if -success or -1 in case of a failure. If it succeed, a structure jfsck_result -that summarizes the outcome of the operation. There is also a program named -jiofsck which is just a simple human frontend to this function. - -The second, jfsck_cleanup(), is intended to be used after jfsck() by programs -wanting to remove all the stall transaction files and leave the journal -directory ready to use. After calling jfsck(), the transaction files will no -longer be needed, so by cleaning up the directory you make sure you're -starting over with a clean journal. It returns 0 if there was an error, or 1 -if it succeeded. - -.SH UNIX API - -The UNIX API, as explained before, consists of the functions jread(), -jpread(), jreadv(), jwrite(), jpwrite(), jwritev(), jtruncate(). In most cases -you will only need to use this, because they're simple and familiar. +.IR "struct jfs" " and " "struct jtrans" . +They represent a file to operate on and a single transaction, respectively. To +open a file, you should use the +.B jopen() +call, which is just like the normal +.B open(3) +call but affects a pointer to a +.IR struct jfs . +To close a file, use +.BR jclose() . +They're exactly like the +.BR open(3) " and close() +functions but use a +.I struct jfs +instead of a file descriptor; take a look at their manpages if you have any +doubts about how to use them. + +There are two functions that differ from the rest, which are +.BR jfsck() " and " jfsck_cleanup() . + +The first one, +.BR jfsck() , +is used to perform journal checking and recovery in case of a crash. It must +be performed when nobody else is using the file (like in the case of a +filesystem which can't be mounted), and it returns 0 if success or an error +code != 0 in case of a failure. If it succeeded, it will fill jfsck_result +summarizing the outcome of the operation. The error codes can be either +.I J_ENOENT +(no such file), +.I J_ENOJOURNAL +(no journal associated with that file) or +.I J_ENOMEM +(not enough free memory). There is also a program named +.I jiofsck +which is just a simple human frontend to this function. + +The second, +.BR jfsck_cleanup() , +is intended to be used after +.B jfsck() +by programs wanting to remove all the stall transaction files and leave the +journal directory ready to use. After calling +.BR jfsck() , +the transaction files will no longer be needed, so by cleaning up the +directory you make sure you're starting over with a clean journal. It returns +0 if there was an error, or 1 if it succeeded. The aforementioned +.I jiofsck +can also optionally invoke this function after performing the regular checks. + +.SS UNIX-alike API + +The UNIX-alike API, as explained before, consists of the functions +.BR jread() ", " jpread() ", " jreadv() ", " jwrite() ", " jpwrite() ", " +.BR jwritev() ", " jtruncate() . They are all exactly like the UNIX equivalent (if you still don't get it, take the initial 'j' out), and behave the same way, with the only exception that -instead of a file descriptor you need to pass a pointer to a struct jfs (just -like jopen() and jclose()). Again, I will not duplicate the manpage for all -these functions, just refer to the regular UNIX versions to see how to use -them, they all have the same semantics and behave the same way. +instead of a file descriptor you need to pass a pointer to a +.IR "struct jfs" . +Again, I will not duplicate the manpage for all these functions, just refer to +the regular UNIX versions to see how to use them, they all have the same +semantics and behave the same way. -.SH Basic functions +.SS BASIC FUNCTIONS The basic functions are the ones which manipulate transactions directly; they -are five: jtrans_init(), jtrans_add(), jtrans_commit(), jtrans_rollback() and -jtrans_free(). These are intended to be use in special situations where your -application needs direct control over the transactions. - -jtrans_init() and jtrans_free() just initialize and free a given transaction, -the former should be called prior any use, and the latter when you want to -destroy a transaction. Note that jtrans_free() is not a disk operation, but -only frees the pointers that were previously allocated by the library; all -disk operations are performed by the other two functions. They have no return -value. - -jtrans_add() is used to add operations to a transaction, and it takes the same -parameters as the pwrite() call. It gets a buffer, it's lenght and the offset -where it should be applied, and adds it to the transaction. You can add -multiple operations to a transaction, and they will be applied in order. -Operation within the same transaction must not overlap; if they do, commiting -the transaction will fail. - -jtrans_commit() is in charge of commiting the given transaction, and after its -return the data has been saved to the disk atomically. It returns the number -of bytes written or -1 if there was an error. - -jtrans_rollback() reverses a transaction that was applied with -jtrans_commit(), and leaves the file as it was before applying it. Be very -very careful with this function, it's quite dangerous if you don't know for -sure that you're doing the right thing. It returns as jtrans_commit(). - -.SH BUGS - -None that I'm aware of, but if you find one please let me know at -albertogli@telpin.com.ar. +are five: +.BR jtrans_init() ", " jtrans_add() ", " jtrans_commit() ", " jtrans_rollback() +and +.BR jtrans_free() . +These are intended to be use where your application requires direct control +over the transactions. + +.BR jtrans_init() " and " jtrans_free() +just initialize and free a given transaction; the former should be called +prior any use, and the latter when you want to destroy a transaction. Note +that +.B jtrans_free() +is not a disk operation, but only frees the pointers that were previously +allocated by the library; all disk operations are performed by the other two +functions. + +.B jtrans_add() +is used to add operations to a transaction, and it takes the same parameters +as +.BR pwrite() . +It gets a buffer, its lenght and the offset where it should be applied, and +adds it to the transaction. You can add multiple operations to a transaction, +and they will be applied in order. + +.B jtrans_commit() +commits the given transaction to disk. After it has returned, data has been +saved to the disk. It returns the number of bytes written or -1 if there was +an error. The commit operation is atomic with regards to other read or write +operations on different processes, as long as they all access it via libjio. + +.B jtrans_rollback() +reverses a transaction that was applied with +.BR jtrans_commit() , +and leaves the file as it was before applying it. Be very very careful with +this function, it's quite dangerous if you don't know for sure that you're +doing the right thing. It returns the number of bytes written or -1 if there +was an error. .SH SEE ALSO @@ -191,3 +203,10 @@ albertogli@telpin.com.ar. .BR pwrite (2), .BR ftruncate (2), .BR close (2) + +.SH BUGS + +None that I'm aware of, but if you find one please let me know at +If you want to report bugs, or have any questions or comments, just let me +know at albertito@blitiri.com.ar. + diff --git a/doc/libjio.html b/doc/libjio.html deleted file mode 100644 index c1f99d9..0000000 --- a/doc/libjio.html +++ /dev/null @@ -1,444 +0,0 @@ - - - - - -libjio - A library for journaled I/O - - - - - - - - - - - - - - - -next_inactive -up -previous -
-
-
- - -

- -

- -

-

libjio - A library for journaled I/O

-
- -

Alberto Bertogli (albertogli@telpin.com.ar)

-
-
- -

-Contents -

- - - - - -

- -

-1 Introduction -

- -

-libjio is a library for doing journaled transaction-oriented -I/O, providing atomicity warantees and a simple to use but powerful -API. - -

-This document explains the design of the library, how it works internally -and why it works that way. You should read it even if you don't plan -to do use the library in strange ways, it provides (or at least tries -to =) an insight view on how the library performs its job, which can -be very valuable knowledge when working with it. - -

-To the user, libjio provides two groups of functions, one UNIX-alike -that implements the journaled versions of the classic functions (open(), -read(), write() and friends); and a lower-level one -that center on transactions and allows the user to manipulate them -directly by providing means of commiting and rollbacking. The former, -as expected, are based on the latter and interact safely with them. -Besides, it's designed in a way that allows efficient and safe interaction -with I/O performed from outside the library in case you want to. - -

-The following sections describe different concepts and procedures -that the library bases its work on. It's not intended to be a replace -to reading the source code: please do so if you have any doubts, it's -not big at all (less than 800 lines, including comments) and I hope -it's readable enough. If you think that's not the case, please let -me know and I'll try to give you a hand. - -

- -

-2 General on-disk data organization -

- -

-On the disk, the file you are working on will look exactly as you -expect and hasn't got a single bit different that what you would get -using the regular API. But, besides the working file, you will find -a directory named after it where the journaling information lives. - -

-Inside, there are two kind of files: the lock file and transaction -files. The first one is used as a general lock and holds the next -transaction ID to assign, and there is only one; the second one holds -one transaction, which is composed by a header of fixed size and a -variable-size payload, and can be as many as in-flight transactions. - -

-This impose some restrictions to the kind of operations you can perform -over a file while it's currently being used: you can't move it (because -the journal directory name depends on the filename) and you can't -unlink it (for similar reasons). - -

-This warnings are no different from a normal simultaneous use under -classic UNIX environments, but they are here to remind you that even -tho the library warantees a lot and eases many things from its user -(specially from complex cases, like multiple threads using the file -at the same time), you should still be careful when doing strange -things with files while working on them. - -

- -

-2.1 The transaction file -

- -

-The transaction file is composed of two main parts: the header and -the payload. - -

-The header holds basic information about the transaction itself, including -the ID, some flags, the offset to commit to and the lenght of the -data. The payload holds the data, in three parts: user-defined data, -previous data, and real data. - -

-User-defined data is not used by the library itself, but it's a space -where the user can save private data that can be useful later. Previous -data is saved by the library prior applying the commit, so transactions -can be rollbacked. Real data is just the data to save to the disk, -and it is saved because if a crash occurs when while we are applying -the transaction we can recover gracefuly. - -

- -

-3 The commit procedure -

- -

-We call "commit" to the action of safely -and atomically write some given data to the disk. - -

-The former, safely, means that after a commit has been done -we can assume the data will not get lost and can be retrieved, unless -of course some major event happens (like a hardware failure). For -us, this means that the data was effectively written to the disk and -if a crash occurs after the commit operation has returned, the operation -will be complete and data will be available from the file. - -

-The latter, atomically, warantees that the operation is either -completely done, or not done at all. This is a really common word, -specially if you have worked with multiprocessing, and should be quite -familiar. We implement atomicity by combining fine-grained locks and -journaling, which can assure us both to be able to recover from crashes, -and to have exclusive access to a portion of the file without having -any other transaction overlap it. - -

-Well, so much for talking, now let's get real; libjio applies commits -in a very simple and straightforward way, inside jtrans_commit(): - -

- -

    -
  • Lock the section where the commit takes place -
    • -
  • Open the transaction file -
    • -
  • Write the header -
    • -
  • Write the user data (if any) -
    • -
  • Read the previous data from the file -
    • -
  • Write the previous data in the transaction -
    • -
  • Write the data to the file -
    • -
  • Mark the transaction as commited by setting a flag in the header -
    • -
  • Unlink the transaction file -
    • -
  • Unlock the section where the commit takes place -
    • -
-This may look as a lot of steps, but they're not as much as it looks -like inside the code, and allows a recovery from interruptions in -every step of the way (or even in the middle of a step). - -

- -

-4 The rollback procedure -

- -

-First of all, rollbacking is like ``undo'' a commit: return the -data to the state it had exactly before a given commit was applied. -Due to the way we handle commits, doing this operation becomes quite -simple and straightforward. - -

-In the previous section we said that each transaction held, besides -the data to commit to the disk, the data that was on it before commiting. -That data is saved precisely to be able to rollback. So, to rollback -a transaction all that has to be done is recover that ``previous -data'' from the transaction we want to rollback, and save it to the -disk. In the end, this ends up being a new transaction with the previous -data as the new one, so we do that: create a new transaction structure, -fill in the data from the transaction we want to rollback, and commit -it. All this is performed by jtrans_rollback(). - -

-By doing this we can provide the same warranties a commit has, it's -really fast, eases the recovery, and the code is simple and clean. -What a deal. - -

-But be aware that rollbacking is dangerous. And I really mean it: -you should only do it if you're really sure it's ok. -Consider, for instance, that you commit transaction A, then B, and -then you rollback A. If A and B happen to touch the same portion of -the file, the rollback will, of course, not return the state previous -to B, but previous to A. If it's not done safely, this can lead to -major corruption. Now, if you add to this transactions that extend -the file (and thus rollbacking truncates it back), you not only have -corruption but data loss. So, again, be aware, I can't stress this -enough, rollback only if you really really know what -you are doing. - -

- -

-5 The recovery procedure -

- -

-Recovering from crashes is done by the jfsck() call (or the -program jiofsck which is just a simple invocation to that function), -which opens the file and goes through all transactions in the journal -(remember that transactions are removed from the journal directory -after they're applied), loading and rollbacking them if necessary. -There are several steps where it can fail: there could be no journal, -a given transaction file might be corrupted, incomplete, and so on; -but in the end, there are two cases regarding each transaction: either -it's complete and can be rollbacked, or not. - -

-In the case the transaction is not complete, there is no possibility -that it has been partially applied to the disk, remember that, from -the commit procedure, we only apply the transaction after saving -it in the journal, so there is really nothing left to be done. So -if the transaction is complete, we only need to rollback. - -

-In any case, after making the recovery you can simply remove the journal -entirely and let the library create a new one, and you can be sure -that transaction atomicity was preserved. - -

- -

-6 High-level functions -

- -

-We call high-level functions to the ones provided by the library -that emulate the good old unix file manipulation calls. Most of them -are just wrappers around commits, and implement proper locking when -operating in order to allow simultaneous operations (either across -threads or processes). They are described in detail in the manual -pages, we'll only list them here for completion: - -

- -

    -
  • jopen() -
    • -
  • jread(), jpread(), jreadv() -
    • -
  • jwrite(), jpwrite(), jwritev() -
    • -
  • jtruncate() -
    • -
  • jclose() -
    • -
- -

- -

-7 ACID (or How does libjio fit into theory) -

- -

-I haven't read much theory about this, and the library was implemented -basically by common sense and not theorethical study. - -

-However, I'm aware that database people like ACID (well, that's not -news for anybody ;), which they say mean "Atomicity, Consistency, -Isolation, Durability" (yeah, right!). - -

-So, even libjio is not a purely database thing, it can be used to -achieve those attributes in a simple and efficient way. - -

-Let's take a look one by one: - -

- -

    -
  • Atomicity: In a transaction involving two or more discrete pieces -of information, either all of the pieces are committed or none are. -This has been talked before and we've seen how the library achieves -this point, mostly based on locks and relying on a commit procedure. -
    • -
  • Consistency: A transaction either creates a new and valid state of -data, or, if any failure occurs, returns all data to its state before -the transaction was started. This, like atomicity, has been discussed -before, specially in the recovery section, when we saw how in case -of a crash we end up with a fully applied transaction, or no transaction -applied at all. -
    • -
  • Isolation: A transaction in process and not yet committed must remain -isolated from any other transaction. This comes as a side effect of -doing proper locking on the sections each transaction affect, and -guarantees that there can't be two transactions working on the same -section at the same time. -
    • -
  • Durability: Committed data is saved by the system such that, even -in the event of a failure and system restart, the data is available -in its correct state. For this point we rely on the disk as a method -of permanent storage, and expect that when we do syncronous I/O, data -is safely written and can be recovered after a crash. -
    • -
- -

- -

-8 Working from outside -

- -

-If you want, and are careful enough, you can safely do I/O without -using the library. Here I'll give you some general guidelines that -you need to follow in order to prevent corruption. Of course you can -bend or break them according to your use, this is just a general overview -on how to interact from outside. - -

- -

    -
  • Lock the sections you want to use: the library, as we have already -exposed, relies on fcntl locking; so, if you intend to operate on -parts on the file while using it, you should lock them. -
    • -
  • Don't tuncate, unlink or rename: these operations have serious implications -when they're done while using the library, because the library itself -assumes that names don't change, and files don't dissapear beneath -it. It could potentially lead to corruption, although most of the -time you would just get errors from every call. -
    • -
- -

- -

-About this document ... -

- libjio - A library for journaled I/O

-This document was generated using the -LaTeX2HTML translator Version 2002-2-1 (1.70) -

-Copyright © 1993, 1994, 1995, 1996, -Nikos Drakos, -Computer Based Learning Unit, University of Leeds. -
-Copyright © 1997, 1998, 1999, -Ross Moore, -Mathematics Department, Macquarie University, Sydney. -

-The command line arguments were:
- latex2html -no_subdir -split 0 -show_section_numbers /tmp/lyx_tmpdir2441q5CgGo/lyx_tmpbuf0/libjio.tex -

-The translation was initiated by root on 2004-04-30

- -next_inactive -up -previous -
- -
-root -2004-04-30 -
- - diff --git a/doc/libjio.lyx b/doc/libjio.lyx deleted file mode 100644 index 8c4081e..0000000 --- a/doc/libjio.lyx +++ /dev/null @@ -1,433 +0,0 @@ -#LyX 1.3 created this file. For more info see http://www.lyx.org/ -\lyxformat 221 -\textclass article -\language english -\inputencoding auto -\fontscheme default -\graphics default -\paperfontsize default -\papersize Default -\paperpackage a4 -\use_geometry 0 -\use_amsmath 0 -\use_natbib 0 -\use_numerical_citations 0 -\paperorientation portrait -\secnumdepth 3 -\tocdepth 3 -\paragraph_separation indent -\defskip medskip -\quotes_language english -\quotes_times 2 -\papercolumns 1 -\papersides 1 -\paperpagestyle default - -\layout Title - -libjio - A library for journaled I/O -\layout Author - -Alberto Bertogli (albertogli@telpin.com.ar) -\layout Standard - - -\begin_inset LatexCommand \tableofcontents{} - -\end_inset - - -\layout Section - -Introduction -\layout Standard - - -\emph on -libjio -\emph default - is a library for doing journaled transaction-oriented I/O, providing atomicity - warantees and a simple to use but powerful API. -\layout Standard - -This document explains the design of the library, how it works internally - and why it works that way. - You should read it even if you don't plan to do use the library in strange - ways, it provides (or at least tries to =) an insight view on how the library - performs its job, which can be very valuable knowledge when working with - it. - It assumes that there is some basic knowledge about how the library is - used, which can be found in the manpage or in the programmer's guide. -\layout Standard - -To the user, libjio provides two groups of functions, one UNIX-alike that - implements the journaled versions of the classic functions ( -\emph on -open() -\emph default -, -\emph on -read() -\emph default -, -\emph on -write() -\emph default - and friends); and a lower-level one that center on transactions and allows - the user to manipulate them directly by providing means of commiting and - rollbacking. - The former, as expected, are based on the latter and interact safely with - them. - Besides, it's designed in a way that allows efficient and safe interaction - with I/O performed from outside the library in case you want to. -\layout Standard - -The following sections describe different concepts and procedures that the - library bases its work on. - It's not intended to be a replace to reading the source code: please do - so if you have any doubts, it's not big at all (less than 1500 lines, including - comments) and I hope it's readable enough. - If you think that's not the case, please let me know and I'll try to give - you a hand. -\layout Section - -General on-disk data organization -\layout Standard - -On the disk, the file you are working on will look exactly as you expect - and hasn't got a single bit different that what you would get using the - regular API. - But, besides the working file, you will find a directory named after it - where the journaling information lives. - -\layout Standard - -Inside, there are two kind of files: the lock file and transaction files. - The first one is used as a general lock and holds the next transaction - ID to assign, and there is only one; the second one holds one transaction, - which is composed by a header of fixed size and a variable-size payload, - and can be as many as in-flight transactions. - -\layout Standard - -This impose some restrictions to the kind of operations you can perform - over a file while it's currently being used: you can't move it (because - the journal directory name depends on the filename) and you can't unlink - it (for similar reasons). - -\layout Standard - -This warnings are no different from a normal simultaneous use under classic - UNIX environments, but they are here to remind you that even tho the library - warantees a lot and eases many things from its user (specially from complex - cases, like multiple threads using the file at the same time), you should - still be careful when doing strange things with files while working on - them. - -\layout Subsection - -The transaction file -\layout Standard - -The transaction file is composed of two main parts: the header and the payload. -\layout Standard - -The header holds basic information about the transaction itself, including - the ID, some flags, and the amount of operations it includes. - Then the payload has all the operations one after the other, divided in - two parts: the first one includes static information about the operation - (the lenght of the data, the offset of the file where it should be applied, - etc.) and the data itself, which is saved by the library prior applying - the commit, so transactions can be rollbacked. -\layout Section - -The commit procedure -\layout Standard - -We call "commit" to the action of -\emph on -safely -\emph default - and -\emph on -atomically -\emph default - write some given data to the disk. -\layout Standard - -The former, -\emph on -safely -\emph default -, means that after a commit has been done we can assume the data will not - get lost and can be retrieved, unless of course some major event happens - (like a physical hard disk crash). - For us, this means that the data was effectively written to the disk and - if a crash occurs after the commit operation has returned, the operation - will be complete and data will be available from the file. -\layout Standard - -The latter, -\emph on -atomically -\emph default -, guarantees that the operation is either completely done, or not done at - all. - This is a really common word, specially if you have worked with multiprocessing -, and should be quite familiar. - We implement atomicity by combining fine-grained locks and journaling, - which can assure us both to be able to recover from crashes, and to have - exclusive access to a portion of the file without having any other transaction - overlap it. -\layout Standard - -Well, so much for talking, now let's get real; libjio applies commits in - a very simple and straightforward way, inside -\emph on -jtrans_commit() -\emph default -: -\layout Itemize - -Lock the file offsets where the commit takes place -\layout Itemize - -Open the transaction file -\layout Itemize - -Write the header -\layout Itemize - -Read all the previous data from the file -\layout Itemize - -Write the previous data in the transaction -\layout Itemize - -Write the data to the file -\layout Itemize - -Mark the transaction as commited by setting a flag in the header -\layout Itemize - -Unlink the transaction file -\layout Itemize - -Unlock the offsets where the commit takes place -\layout Standard - -This may look as a lot of steps, but they're not as much as it looks like - inside the code, and allows a recovery from interruptions in every step - of the way (or even in the middle of a step). -\layout Section - -The rollback procedure -\layout Standard - -First of all, rollbacking is like -\begin_inset Quotes eld -\end_inset - -undo -\begin_inset Quotes erd -\end_inset - - a commit: return the data to the state it had exactly before a given commit - was applied. - Due to the way we handle commits, doing this operation becomes quite simple - and straightforward. -\layout Standard - -In the previous section we said that each transaction held the data that - was on it before commiting. - That data is saved precisely to be able to rollback. - So, to rollback a transaction all that has to be done is recover that -\begin_inset Quotes eld -\end_inset - -previous data -\begin_inset Quotes erd -\end_inset - - from the transaction we want to rollback, and save it to the disk. - In the end, this ends up being a new transaction with the previous data - as the new one, so we do that: create a new transaction structure, fill - in the data from the transaction we want to rollback, and commit it. - All this is performed by -\emph on -jtrans_rollback() -\emph default -. -\layout Standard - -By doing this we can provide the same warranties a commit has, it's really - fast, eases the recovery, and the code is simple and clean. - What a deal. -\layout Standard - -But be aware that rollbacking is dangerous. - And I really mean it: you should -\series bold -\emph on -only -\series default -\emph default - do it if you're really sure it's ok. - Consider, for instance, that you commit transaction A, then B, and then - you rollback A. - If A and B happen to touch the same portion of the file, the rollback will, - of course, not return the state previous to B, but previous to A. - If it's not done safely, this can lead to major corruption. - Now, if you add to this transactions that extend the file (and thus rollbacking - truncates it back), you not only have corruption but data loss. - So, again, be aware, I can't stress this enough, -\series bold -\emph on -rollback only if you really really know what you are doing -\series default -\emph default -. -\layout Section - -The recovery procedure -\layout Standard - -Recovering from crashes is done by the -\emph on -jfsck() -\emph default - call (or the program -\emph on -jiofsck -\emph default - which is just a simple invocation to that function), which opens the file - and goes through all transactions in the journal (remember that transactions - are removed from the journal directory after they're applied), loading - and rollbacking them if necessary. - There are several steps where it can fail: there could be no journal, a - given transaction file might be corrupted, incomplete, and so on; but in - the end, there are two cases regarding each transaction: either it's complete - and can be rollbacked, or not. -\layout Standard - -In the case the transaction is not complete, there is no possibility that - it has been partially applied to the disk, remember that, from the commit - procedure, we only apply the transaction -\emph on -after -\emph default - saving it in the journal, so there is really nothing left to be done. - So if the transaction is complete, we only need to rollback. -\layout Standard - -In any case, after making the recovery you can simply remove the journal - entirely and let the library create a new one, and you can be sure that - transaction atomicity was preserved. -\layout Section - -UNIX API -\layout Standard - -We call -\emph on -UNIX API -\emph default - to the functions provided by the library that emulate the good old UNIX - file manipulation calls. - Most of them are just wrappers around commits, and implement proper locking - when operating in order to allow simultaneous operations (either across - threads or processes). - They are described in detail in the manual pages, we'll only list them - here for completion: -\layout Itemize - -jopen() -\layout Itemize - -jread(), jpread(), jreadv() -\layout Itemize - -jwrite(), jpwrite(), jwritev() -\layout Itemize - -jtruncate() -\layout Itemize - -jclose() -\layout Section - -ACID (or How does libjio fit into theory) -\layout Standard - -I haven't read much theory about this, and the library was implemented basically - by common sense and not theorethical study. - -\layout Standard - -However, I'm aware that database people like ACID (well, that's not news - for anybody ;), which they say mean "Atomicity, Consistency, Isolation, - Durability" (yeah, right!). - -\layout Standard - -So, even libjio is not a purely database thing, it can be used to achieve - those attributes in a simple and efficient way. - -\layout Standard - -Let's take a look one by one: -\layout Itemize - -Atomicity: In a transaction involving two or more discrete pieces of information -, either all of the pieces are committed or none are. - This has been talked before and we've seen how the library achieves this - point, mostly based on locks and relying on a commit procedure. -\layout Itemize - -Consistency: A transaction either creates a new and valid state of data, - or, if any failure occurs, returns all data to its state before the transaction - was started. - This, like atomicity, has been discussed before, specially in the recovery - section, when we saw how in case of a crash we end up with a fully applied - transaction, or no transaction applied at all. -\layout Itemize - -Isolation: A transaction in process and not yet committed must remain isolated - from any other transaction. - This comes as a side effect of doing proper locking on the sections each - transaction affect, and guarantees that there can't be two transactions - working on the same section at the same time. -\layout Itemize - -Durability: Committed data is saved by the system such that, even in the - event of a failure and system restart, the data is available in its correct - state. - For this point we rely on the disk as a method of permanent storage, and - expect that when we do syncronous I/O, data is safely written and can be - recovered after a crash. -\layout Section - -Working from outside -\layout Standard - -If you want, and are careful enough, you can safely do I/O without using - the library. - Here I'll give you some general guidelines that you need to follow in order - to prevent corruption. - Of course you can bend or break them according to your use, this is just - a general overview on how to interact from outside. - -\layout Itemize - -Lock the sections you want to use: the library, as we have already exposed, - relies on fcntl locking; so, if you intend to operate on parts on the file - while using it, you should lock them. - -\layout Itemize - -Don't tuncate, unlink or rename: these operations have serious implications - when they're done while using the library, because the library itself assumes - that names don't change, and files don't dissapear beneath it. - It could potentially lead to corruption, although most of the time you - would just get errors from every call. -\the_end diff --git a/doc/libjio.rst b/doc/libjio.rst new file mode 100644 index 0000000..32de123 --- /dev/null +++ b/doc/libjio.rst @@ -0,0 +1,244 @@ + +libjio - A library for journaled I/O +====================================== + +Introduction +------------ + +libjio is a library for doing journaled, transaction-oriented I/O, providing +atomicity warantees and a simple to use but powerful API. + +This document explains the design of the library, how it works internally and +why it works that way. You should read it even if you don't plan to do use the +library in strange ways, it provides (or at least tries to) an insight view on +how the library performs its job, which can be very valuable knowledge when +working with it. It assumes that there is some basic knowledge about how the +library is used, which can be found in the manpage or in the programmer's +guide. + +To the user, libjio provides two groups of functions, one UNIX-alike that +implements the journaled versions of the classic functions (open(), read(), +write() and friends); and a lower-level one that center on transactions and +allows the user to manipulate them directly by providing means of commiting +and rollbacking. The former, as expected, are based on the latter and interact +safely with them. Besides, it's designed in a way that allows efficient and +safe interaction with I/O performed from outside the library in case you want +to. + +The following sections describe different concepts and procedures that the +library bases its work on. It's not intended to be a replace to reading the +source code: please do so if you have any doubts, it's not big at all (less +than 1500 lines, including comments) and I hope it's readable enough. If you +think that's not the case, please let me know and I'll try to give you a hand. + + +General on-disk data organization +--------------------------------- + +On the disk, the file you are working on will look exactly as you expect and +hasn't got a single bit different that what you would get using the regular +UNIX API. But, besides the working file, you will find a directory named after +it where the journaling information lives. + +Inside, there are two kind of files: the lock file and transaction files. The +first one is used as a general lock and holds the next transaction ID to +assign, and there is only one; the second one holds one transaction, which is +composed by a header of fixed size and a variable-size payload, and can be as +many as in-flight transactions. + +This impose some restrictions to the kind of operations you can perform over a +file while it's currently being used: you can't move it (because the journal +directory name depends on the filename) and you can't unlink it (for similar +reasons). + +This warnings are no different from a normal simultaneous use under classic +UNIX environments, but they are here to remind you that even tho the library +warranties a lot and eases many things from its user, you should still be +careful when doing strange things with files while working on them. + +The transaction file +~~~~~~~~~~~~~~~~~~~~ + +The transaction file is composed of two main parts: the header and the +payload. + +The header holds basic information about the transaction itself, including the +ID, some flags, and the amount of operations it includes. Then the payload has +all the operations one after the other, divided in two parts: the first one +includes static information about the operation (the length of the data, the +offset of the file where it should be applied, etc.) and the data itself, +which is saved by the library prior applying the commit, so transactions can +be rollbacked. + +At the end of the transaction file, a checksum is stored, to detect journal +corruption. + + +The commit procedure +-------------------- + +We call *commit* to the action of safely and atomically write some given data +to the disk. + +The former, "safely", means that after a commit has been done we can assume +the data will not get lost and can be retrieved, unless of course some major +event happens (like a physical hard disk crash). For us, this means that the +data was effectively written to the disk and if a crash occurs after the +commit operation has returned, the operation will be complete and data will be +available from the file. + +The latter, "atomically", guarantees that the operation is either completely +done, or not done at all. This is a really common word, specially if you have +worked with multiprocessing, and should be quite familiar. We implement +atomicity by combining fine-grained locks and journaling, which can assure us +both to be able to recover from crashes, and to have exclusive access to a +portion of the file without having any other transaction overlap it. + +Well, so much for talking, now let's get real; libjio applies commits in a +very simple and straightforward way, inside jtrans_commit(): + + - Lock the file offsets where the commit takes place + - Open the transaction file + - Write the header + - Read all the previous data from the file + - Write the previous data in the transaction + - Write the data to the file + - Mark the transaction as committed by setting a flag in the header + - Unlink the transaction file + - Unlock the offsets where the commit takes place + +This may seem like a lot of steps, but they're not as much as it looks like +inside the code, and allows a recovery from interruptions in every step of the +way, and even in the middle of a step. + + +The rollback procedure +---------------------- + +First of all, rollbacking is like "undo" a commit: returns the data to the +state it had exactly before a given commit was applied. Due to the way we +handle commits, doing this operation becomes quite simple and straightforward. + +In the previous section we said that each transaction held the data that was +on it before commiting. That data saved is precisely the one we need to be +able to rollback. + +So, to rollback a transaction all that has to be done is recover the +previous data from the transaction we want to rollback, and save it to the +disk. In the end, this ends up being a new transaction with the previous data +as the new one, and that's how it's done: create a new transaction structure, +fill in the data from the transaction we want to rollback, and commit it. All +this is performed by jtrans_rollback(). + +By doing this we can provide the same warranties a commit has, it's really +fast, eases the recovery, and the code is simple and clean. What a deal. + +But be aware that rollbacking is dangerous. And I really mean it: you should +only do it if you're really sure it's ok. Consider, for instance, that you +commit transaction A, then B, and then you rollback A. If A and B happen to +touch the same portion of the file, the rollback will, of course, not return +the state previous to B, but previous to A. + +If it's not done safely, this can lead to major corruption. Now, if you add to +this transactions that extend the file (and thus rollbacking truncates it +back), it gets even worse. So, again, be aware, I can't stress this enough, +rollback only if you really really know what you are doing. + + +The recovery procedure +---------------------- + +Recovering from crashes is done by the jfsck() call (or the program *jiofsck* +which is just a simple invocation to that function), which opens the file and +goes through all transactions in the journal (remember that transactions are +removed from the journal directory after they're applied), loading and +rollbacking them if necessary. There are several steps where it can fail: +there could be no journal, a given transaction file might be corrupted, +incomplete, and so on; but in the end, there are two cases regarding each +transaction: either it's complete and can be rollbacked, or not. + +In the case the transaction file was not completely written, there is no +possibility that it has been partially applied to the disk: remember that, +from the commit procedure, we only apply the transaction after saving it in +the journal, so there is really nothing left to be done. So if the transaction +is complete, we only need to rollback. + +In any case, after making the recovery you can simply remove the journal +entirely and let the library create a new one, and you can be sure that +transaction atomicity was preserved. You can use jfsck_cleanup() for that +purpose. + + +UNIX-alike API +-------------- + +We call UNIX-alike API to the functions provided by the library that emulate +the good old UNIX file manipulation calls. Most of them are just wrappers +around commits, and implement proper locking when operating in order to allow +simultaneous operations (either across threads or processes). They are +described in detail in the manual pages, we'll only list them here for +completion: + + - jopen() + - jread(), jpread(), jreadv() + - jwrite(), jpwrite(), jwritev() + - jtruncate() + - jclose() + + +ACID warranties +--------------- + +Database people like ACID (well, that's not news for anybody), which they say +mean "Atomicity, Consistency, Isolation, Durability". + +So, even when libjio is not a purely database thing, its transactions provide +those properties. Let's take a look one by one: + +Atomicity + In a transaction involving two or more discrete pieces of information, + either all of the pieces are committed or none are. This has been talked + before and we've seen how the library achieves this point, mostly based on + locks and relying on a commit procedure. + +Consistency + A transaction either creates a new and valid state of data, or, if any + failure occurs, returns all data to its state before the transaction was + started. This, like atomicity, has been discussed before, specially in the + recovery section, when we saw how in case of a crash we end up with a fully + applied transaction, or no transaction applied at all. + +Isolation + A transaction in process and not yet committed must remain isolated from any + other transaction. This comes as a side effect of doing proper locking on + the sections each transaction affect, and guarantees that there can't be two + transactions working on the same section at the same time. + +Durability + Committed data is saved by the system such that, even in the event of a + failure, the data is available in a correct state. To provide this, libjio + relies on the disk as a method of permanent storage, and expects that when + it does syncronous I/O, data is safely written and can be recovered after a + crash. + + +Working from outside +-------------------- + +If you want, and are careful enough, you can safely use the library and still +do I/O using the regular UNIX calls. + +This section provides some general guidelines that you need to follow in order +to prevent corruption. Of course you can bend or break them according to your +use, this is just a general overview on how to interact from outside. + + - Lock the sections you want to use: the library, as we have already exposed, + relies on fcntl() locking; so, if you intend to operate on parts on the + file while using it, you should lock them. + - Don't truncate, unlink or rename: these operations have serious + implications when they're done while using the library, because the library + itself assumes that names don't change, and files don't disappear from + underneath it. It could potentially lead to corruption, although most of + the time you would just get errors from every call. + + diff --git a/doc/libjio.txt b/doc/libjio.txt deleted file mode 100644 index 3e78f13..0000000 --- a/doc/libjio.txt +++ /dev/null @@ -1,305 +0,0 @@ -libjio - A library for journaled I/O - -Alberto Bertogli (albertogli@telpin.com.ar) - -Table of Contents - -1 Introduction -2 General on-disk data organization - 2.1 The transaction file -3 The commit procedure -4 The rollback procedure -5 The recovery procedure -6 UNIX API -7 ACID (or How does libjio fit into theory) -8 Working from outside - - - -1 Introduction - -libjio is a library for doing journaled -transaction-oriented I/O, providing atomicity warantees -and a simple to use but powerful API. - -This document explains the design of the library, how -it works internally and why it works that way. You -should read it even if you don't plan to do use the -library in strange ways, it provides (or at least tries -to =) an insight view on how the library performs its -job, which can be very valuable knowledge when working -with it. It assumes that there is some basic knowledge -about how the library is used, which can be found in -the manpage or in the programmer's guide. - -To the user, libjio provides two groups of functions, -one UNIX-alike that implements the journaled versions -of the classic functions (open(), read(), write() and -friends); and a lower-level one that center on -transactions and allows the user to manipulate them -directly by providing means of commiting and -rollbacking. The former, as expected, are based on the -latter and interact safely with them. Besides, it's -designed in a way that allows efficient and safe -interaction with I/O performed from outside the library -in case you want to. - -The following sections describe different concepts and -procedures that the library bases its work on. It's not -intended to be a replace to reading the source code: -please do so if you have any doubts, it's not big at -all (less than 1500 lines, including comments) and I -hope it's readable enough. If you think that's not the -case, please let me know and I'll try to give you a hand. - -2 General on-disk data organization - -On the disk, the file you are working on will look -exactly as you expect and hasn't got a single bit -different that what you would get using the regular -API. But, besides the working file, you will find a -directory named after it where the journaling -information lives. - -Inside, there are two kind of files: the lock file and -transaction files. The first one is used as a general -lock and holds the next transaction ID to assign, and -there is only one; the second one holds one -transaction, which is composed by a header of fixed -size and a variable-size payload, and can be as many as -in-flight transactions. - -This impose some restrictions to the kind of operations -you can perform over a file while it's currently being -used: you can't move it (because the journal directory -name depends on the filename) and you can't unlink it -(for similar reasons). - -This warnings are no different from a normal -simultaneous use under classic UNIX environments, but -they are here to remind you that even tho the library -warantees a lot and eases many things from its user -(specially from complex cases, like multiple threads -using the file at the same time), you should still be -careful when doing strange things with files while -working on them. - -2.1 The transaction file - -The transaction file is composed of two main parts: the -header and the payload. - -The header holds basic information about the -transaction itself, including the ID, some flags, and -the amount of operations it includes. Then the payload -has all the operations one after the other, divided in -two parts: the first one includes static information -about the operation (the lenght of the data, the offset -of the file where it should be applied, etc.) and the -data itself, which is saved by the library prior -applying the commit, so transactions can be rollbacked. - -3 The commit procedure - -We call "commit" to the action of safely and atomically -write some given data to the disk. - -The former, safely, means that after a commit has been -done we can assume the data will not get lost and can -be retrieved, unless of course some major event happens -(like a physical hard disk crash). For us, this means -that the data was effectively written to the disk and -if a crash occurs after the commit operation has -returned, the operation will be complete and data will -be available from the file. - -The latter, atomically, guarantees that the operation -is either completely done, or not done at all. This is -a really common word, specially if you have worked with -multiprocessing, and should be quite familiar. We -implement atomicity by combining fine-grained locks and -journaling, which can assure us both to be able to -recover from crashes, and to have exclusive access to a -portion of the file without having any other -transaction overlap it. - -Well, so much for talking, now let's get real; libjio -applies commits in a very simple and straightforward -way, inside jtrans_commit(): - -* Lock the file offsets where the commit takes place - -* Open the transaction file - -* Write the header - -* Read all the previous data from the file - -* Write the previous data in the transaction - -* Write the data to the file - -* Mark the transaction as commited by setting a flag in - the header - -* Unlink the transaction file - -* Unlock the offsets where the commit takes place - -This may look as a lot of steps, but they're not as -much as it looks like inside the code, and allows a -recovery from interruptions in every step of the way -(or even in the middle of a step). - -4 The rollback procedure - -First of all, rollbacking is like "undo" a commit: return -the data to the state it had exactly before a given -commit was applied. Due to the way we handle commits, -doing this operation becomes quite simple and straightforward. - -In the previous section we said that each transaction -held the data that was on it before commiting. That -data is saved precisely to be able to rollback. So, to -rollback a transaction all that has to be done is -recover that "previous data" from the transaction we want -to rollback, and save it to the disk. In the end, this -ends up being a new transaction with the previous data -as the new one, so we do that: create a new transaction -structure, fill in the data from the transaction we -want to rollback, and commit it. All this is performed -by jtrans_rollback(). - -By doing this we can provide the same warranties a -commit has, it's really fast, eases the recovery, and -the code is simple and clean. What a deal. - -But be aware that rollbacking is dangerous. And I -really mean it: you should only do it if you're really -sure it's ok. Consider, for instance, that you commit -transaction A, then B, and then you rollback A. If A -and B happen to touch the same portion of the file, the -rollback will, of course, not return the state previous -to B, but previous to A. If it's not done safely, this -can lead to major corruption. Now, if you add to this -transactions that extend the file (and thus rollbacking -truncates it back), you not only have corruption but -data loss. So, again, be aware, I can't stress this -enough, rollback only if you really really know what -you are doing. - -5 The recovery procedure - -Recovering from crashes is done by the jfsck() call (or -the program jiofsck which is just a simple invocation -to that function), which opens the file and goes -through all transactions in the journal (remember that -transactions are removed from the journal directory -after they're applied), loading and rollbacking them if -necessary. There are several steps where it can fail: -there could be no journal, a given transaction file -might be corrupted, incomplete, and so on; but in the -end, there are two cases regarding each transaction: -either it's complete and can be rollbacked, or not. - -In the case the transaction is not complete, there is -no possibility that it has been partially applied to -the disk, remember that, from the commit procedure, we -only apply the transaction after saving it in the -journal, so there is really nothing left to be done. So -if the transaction is complete, we only need to rollback. - -In any case, after making the recovery you can simply -remove the journal entirely and let the library create -a new one, and you can be sure that transaction -atomicity was preserved. - -6 UNIX API - -We call UNIX API to the functions provided by the -library that emulate the good old UNIX file -manipulation calls. Most of them are just wrappers -around commits, and implement proper locking when -operating in order to allow simultaneous operations -(either across threads or processes). They are -described in detail in the manual pages, we'll only -list them here for completion: - -* jopen() - -* jread(), jpread(), jreadv() - -* jwrite(), jpwrite(), jwritev() - -* jtruncate() - -* jclose() - -7 ACID (or How does libjio fit into theory) - -I haven't read much theory about this, and the library -was implemented basically by common sense and not -theorethical study. - -However, I'm aware that database people like ACID -(well, that's not news for anybody ;), which they say -mean "Atomicity, Consistency, Isolation, Durability" -(yeah, right!). - -So, even libjio is not a purely database thing, it can -be used to achieve those attributes in a simple and -efficient way. - -Let's take a look one by one: - -* Atomicity: In a transaction involving two or more - discrete pieces of information, either all of the - pieces are committed or none are. This has been - talked before and we've seen how the library achieves - this point, mostly based on locks and relying on a - commit procedure. - -* Consistency: A transaction either creates a new and - valid state of data, or, if any failure occurs, - returns all data to its state before the transaction - was started. This, like atomicity, has been discussed - before, specially in the recovery section, when we - saw how in case of a crash we end up with a fully - applied transaction, or no transaction applied at all. - -* Isolation: A transaction in process and not yet - committed must remain isolated from any other - transaction. This comes as a side effect of doing - proper locking on the sections each transaction - affect, and guarantees that there can't be two - transactions working on the same section at the same time. - -* Durability: Committed data is saved by the system - such that, even in the event of a failure and system - restart, the data is available in its correct state. - For this point we rely on the disk as a method of - permanent storage, and expect that when we do - syncronous I/O, data is safely written and can be - recovered after a crash. - -8 Working from outside - -If you want, and are careful enough, you can safely do -I/O without using the library. Here I'll give you some -general guidelines that you need to follow in order to -prevent corruption. Of course you can bend or break -them according to your use, this is just a general -overview on how to interact from outside. - -* Lock the sections you want to use: the library, as we - have already exposed, relies on fcntl locking; so, if - you intend to operate on parts on the file while - using it, you should lock them. - -* Don't tuncate, unlink or rename: these operations - have serious implications when they're done while - using the library, because the library itself assumes - that names don't change, and files don't dissapear - beneath it. It could potentially lead to corruption, - although most of the time you would just get errors - from every call. diff --git a/doc/layout b/doc/source_layout similarity index 100% rename from doc/layout rename to doc/source_layout diff --git a/doc/threads b/doc/threads deleted file mode 100644 index bf4b6fb..0000000 --- a/doc/threads +++ /dev/null @@ -1,20 +0,0 @@ - -The library is entirely threadsafe. - -This will make some people who worked with threads a bit concerned, because -everybody knows that if a file descriptor is shared among threads, and two -threads decide to read/write/perform any op that moves the file pointer, a -mess is waiting to happen. And almost operations do touch the file pointer. - -But don't worry, the library is _truly_ threadsafe: it uses pread/pwrite, -which do not touch the file pointer, and allows working on the same file -simultaneously without concerns. Besides, it slightly improves performance by -having less locking, less system calls, lower overhead and less calculation to -perform the operation. - - -Still, bear in mind that if you decide to work on the file outside libjio you -need to lockf() the sections you're going to work on, because libjio relies on -lockf() locking to warantee atomicity. - - diff --git a/doc/tids b/doc/tids deleted file mode 100644 index bf65f23..0000000 --- a/doc/tids +++ /dev/null @@ -1,83 +0,0 @@ - -Transaction ID assignment procedure -Alberto Bertogli (albertogli@telpin.com.ar) -4/October/2004 ---------------------------------------------- - -This brief document describes how libjio assigns an unique number to each -transaction that identifies it univocally during its lifetime. - -It is a very delicate issue, because the rest of the library depends on the -uniqueness of the ID; it has to be coherent across threads and procesess; and -it can't take long: it serializes transaction creation (and it's the only -contention point for independent non-overlapping transactions). - - -Description ------------ - -We have two functions: get_tid() and free_tid(), which return a new -transaction ID, and mark a given transaction ID as no longer in use, -respectively. - -The main piece of the mechanism is the lockfile: a file named "lock" which -holds the maximum transaction ID in use. This file gets opened and mmap()'ed -for faster use inside jopen(). That way, we can treat it directly as an -integer holding the max tid. - -To avoid paralell modifications, we will always lock the file with fcntl() -before accessing it. - -Let's begin by describing how get_tid() works, because it's quite simple: it -locks the lockfile, gets the max tid, adds 1 to it, unlock the file and return -that value. That way, the new tid is always the new max, and with the locking -we can be sure it's impossible to assign the same tid to two different -transactions. - -After a tid has been assigned, the commit process will create a file named -after it inside the journal directory. Then, it will operate on that file all -it wants, and when the moment comes, the transaction is no longer needed and -has to be freed. - -The first thing we do is to unlink that transaction file. And then, we call -free_tid(), which will update the lockfile to represent the new max tid, in -case it has changed. - -free_tid() begins by checking that if the transaction we're freeing is the -greatest, and if not, just returns. - -But if it is, we need to find out the new max tid. We do it by "walking" the -journal directory looking for the file with the greatest number, and that's -our new max tid. If there are no files, we use 0. - - -Things to notice ----------------- - -The following is a list of small things to notice about the mechanism. They're -useful because races tend to be subtle, and I _will_ forget about them. The -descriptions are not really detailed, just enough to give a general idea. - - -* It is possible that we get in free_tid() and the transaction we want to free -is greater than the max tid. In that case, we do nothing: it's a valid -situation. How to get there: two threads about to free two tids. The first one -calls unlink() and just after its return (before it gets a chance to call -free_tid()), another thread, the holder of the current max, steps in and -performs both the unlink() and free_tid(), which would force a lookup to find -a new tid, and as in the first thread we have removed the file, the max tid -could be lower (in particular, it could be 0). This is why we only test for -equalty. - -* Unlink after free_tid() is not desirable: in that case, it'd be normal for -the tid to increment even if we have only one thread writing. It overflows -quite easily. - -* The fact that new tids are always bigger than the current max is not only -because the code is cleaner and faster: that way when recovering we know the -order to apply transactions. A nice catch: this doesn't matter if we're -working with non-overlapping transactions, but if they overlap, we know that -it's impossible that transaction A and B (B gets committed after A) get -applied in the wrong order, because B will only begin to commit _after_ A has -been worked on. - diff --git a/doc/tids.rst b/doc/tids.rst new file mode 100644 index 0000000..4b21416 --- /dev/null +++ b/doc/tids.rst @@ -0,0 +1,76 @@ +Transaction ID assignment procedure +=================================== + +This brief document describes how libjio assigns an unique number to each +transaction that identifies it univocally during its lifetime. + +It is a very delicate issue, because the rest of the library depends on the +uniqueness of the ID. An ID has to be coherent across threads and procesess, +and choosing one it can't take long: it serializes transaction creation (and +it's the only contention point for independent non-overlapping transactions). + + +Description +----------- + +We have two functions: *get_tid()* and *free_tid()*, which respectively return +a new transaction ID, and mark a given transaction ID as no longer in use. + +The main piece of the mechanism is the lockfile: a file named *lock* which +holds the maximum transaction ID in use. This file gets opened and mmap()'ed +for faster use inside *jopen()*. That way, we can treat it directly as an +integer holding the max tid. + +To avoid parallel modifications, we will always lock the file with *fcntl()* +before accessing it. + +Let's begin by describing how *get_tid()* works, because it's quite simple: it +locks the lockfile, gets the max tid, adds 1 to it, unlock the file and return +that value. That way, the new tid is always the new max, and with the locking +we can be sure it's impossible to assign the same tid to two different +transactions. + +After a tid has been assigned, the commit process will create a file named +after it inside the journal directory. Then, it will operate on that file all +it wants, and when the moment comes, the transaction is no longer needed and +has to be freed. + +The first thing we do is to unlink that transaction file. And then, we call +*free_tid()*, which will update the lockfile to represent the new max tid, in +case it has changed. + +*free_tid()* begins by checking that if the transaction we're freeing is the +greatest, and if not, just returns. + +But if it is, we need to find out the new max tid. We do it by "walking" the +journal directory looking for the file with the greatest number, and that's +our new max tid. If there are no files, we use 0. + + +Things to notice +---------------- + +The following is a list of small things to notice about the mechanism. They're +useful because races tend to be subtle, and I *will* forget about them. The +descriptions are not really detailed, just enough to give a general idea. + + - It is possible that we get in *free_tid()* and the transaction we want to + free is greater than the max tid. In that case, we do nothing: it's a valid + situation. How to get there: two threads about to free two tids. The first + one calls *unlink()* and just after its return (before it gets a chance to + call *free_tid()*), another thread, the holder of the current max, steps in + and performs both the *unlink()* and *free_tid()*, which would force a + lookup to find a new tid, and as in the first thread we have removed the + file, the max tid could be lower (in particular, it could be 0). This is + why we only test for equalty. + - Unlink after *free_tid()* is not desirable: in that case, it'd be normal + for the tid to increment even if we have only one thread writing. It + overflows quite easily. + - The fact that new tids are always bigger than the current max is not only + because the code is cleaner and faster: that way when recovering we know + the order to apply transactions. A nice catch: this doesn't matter if we're + working with non-overlapping transactions, but if they overlap, we know + that it's impossible that transaction A and B (B gets committed after A) + get applied in the wrong order, because B will only begin to commit *after* + A has been worked on. + diff --git a/jiofsck.c b/jiofsck.c index 46331fa..a22dcd7 100644 --- a/jiofsck.c +++ b/jiofsck.c @@ -1,7 +1,7 @@ /* * jiofsck - A journal checker and recovery tool for libjio - * Alberto Bertogli (albertogli@telpin.com.ar) + * Alberto Bertogli (albertito@blitiri.com.ar) */ #include @@ -9,7 +9,7 @@ #include "libjio.h" -static void usage() +static void usage(void) { printf("\ Use: jiofsck [clean=1] [dir=DIR] FILE\n\ diff --git a/libjio.h b/libjio.h index ce7651e..39be877 100644 --- a/libjio.h +++ b/libjio.h @@ -1,7 +1,7 @@ /* * libjio - A library for Journaled I/O - * Alberto Bertogli (albertogli@telpin.com.ar) + * Alberto Bertogli (albertito@blitiri.com.ar) */ #ifndef _LIBJIO_H @@ -19,7 +19,7 @@ * library (which uses LFS) and that's just begging for problems. There should * be a portable way for the C library to do some of this for us, but until I * find one, this is the best we can do */ -#if (!defined _FILE_OFFSET_BITS) && (_FILE_OFFSET_BITS != 64) +#if (!defined _FILE_OFFSET_BITS) || (_FILE_OFFSET_BITS != 64) #error "You must compile your application with Large File Support" #endif @@ -161,13 +161,13 @@ FILE *jfsopen(struct jfs *stream, const char *mode); /* jfsck constants (return values) */ #define J_ESUCCESS 0 /* success - shouldn't be used */ -#define J_ENOENT 1 /* no such file */ -#define J_ENOJOURNAL 2 /* no journal associated */ -#define J_ENOMEM 3 /* no enough free memory */ +#define J_ENOENT -1 /* no such file */ +#define J_ENOJOURNAL -2 /* no journal associated */ +#define J_ENOMEM -3 /* no enough free memory */ #ifdef __cplusplus -} /* from extern "C" avobe */ +} /* from extern "C" above */ #endif #endif diff --git a/libjio.skel.pc b/libjio.skel.pc new file mode 100644 index 0000000..3a7b0f1 --- /dev/null +++ b/libjio.skel.pc @@ -0,0 +1,12 @@ + +prefix=++PREFIX++ +libdir=${prefix}/lib +includedir=${prefix}/include + +Name: libjio +Description: A library for Journaled I/O +URL: http://blitiri.com.ar/p/libjio/ +Version: 0.23 +Libs: -L${libdir} -ljio +Cflags: -I${includedir} ++CFLAGS++ + diff --git a/tests/performance/Makefile b/tests/performance/Makefile index c545dc3..36a953f 100644 --- a/tests/performance/Makefile +++ b/tests/performance/Makefile @@ -4,10 +4,10 @@ CFLAGS = -Wall -O6 default: all -all: paralell streaming +all: parallel streaming -paralell: paralell.o - $(CC) $(LIBS) paralell.o -o paralell +parallell: parallel.o + $(CC) $(LIBS) parallel.o -o parallel streaming: streaming.o $(CC) $(LIBS) streaming.o -o streaming @@ -16,7 +16,7 @@ streaming: streaming.o $(CC) $(CFLAGS) -c $< -o $@ clean: - rm -f streaming.o streaming paralell.o paralell + rm -f streaming.o streaming parallel.o parallel rm -f *.bb *.bbg *.da *.gcov gmon.out diff --git a/tests/performance/paralell.c b/tests/performance/parallel.c similarity index 90% rename from tests/performance/paralell.c rename to tests/performance/parallel.c index 5bc086d..205bb90 100644 --- a/tests/performance/paralell.c +++ b/tests/performance/parallel.c @@ -1,7 +1,7 @@ /* - * streaming.c - A program to test speed of paralell writes using libjio. - * Alberto Bertogli (albertogli@telpin.com.ar) + * streaming.c - A program to test speed of parallel writes using libjio. + * Alberto Bertogli (albertito@blitiri.com.ar) */ /* @@ -19,7 +19,7 @@ #include #include -#define FILENAME "test_file-paralell" +#define FILENAME "test_file-parallel" /* Declare here what's shared among threads * It's not the cleanest design ever, but let's face it, it's a simple @@ -30,7 +30,7 @@ int blocksize, towrite, mb; void help(void) { - printf("Use: paralell MBs_to_write_per_thread blocksize nthreads\n"); + printf("Use: parallel MBs_to_write_per_thread blocksize nthreads\n"); exit(1); } diff --git a/tests/performance/streaming.c b/tests/performance/streaming.c index 44b0892..211b388 100644 --- a/tests/performance/streaming.c +++ b/tests/performance/streaming.c @@ -1,7 +1,7 @@ /* * streaming.c - A program to test speed of a streaming write using libjio. - * Alberto Bertogli (albertogli@telpin.com.ar) + * Alberto Bertogli (albertito@blitiri.com.ar) */ diff --git a/trans.c b/trans.c index e1f33d0..ba30674 100644 --- a/trans.c +++ b/trans.c @@ -1,7 +1,7 @@ /* * libjio - A library for Journaled I/O - * Alberto Bertogli (albertogli@telpin.com.ar) + * Alberto Bertogli (albertito@blitiri.com.ar) * * Core transaction API and recovery functions */ @@ -68,12 +68,13 @@ static void free_tid(struct jfs *fs, unsigned int tid) if (tid == curid) { /* look up the new max. */ for (i = curid - 1; i > 0; i--) { - /* this can fail if we're low on mem, but we don't - * care checking here because the problem will come - * out later and we can fail more properly */ get_jtfile(fs, i, name); if (access(name, R_OK | W_OK) == 0) { - curid = i; + break; + } else if (errno != EACCES) { + /* Real error, stop looking for a new max. It + * doesn't hurt us because it's ok if the max + * is higher than it could be */ break; } } @@ -235,8 +236,7 @@ ssize_t jtrans_commit(struct jtrans *ts) goto exit; /* open the transaction file */ - if (!get_jtfile(ts->fs, id, name)) - goto exit; + get_jtfile(ts->fs, id, name); fd = open(name, O_RDWR | O_CREAT | O_TRUNC, 0600); if (fd < 0) goto exit; @@ -490,8 +490,11 @@ ssize_t jtrans_rollback(struct jtrans *ts) * if for some reason, after the previous transacton, we have * extended the file further, this will cut it back to what it * was; read the docs for more detail */ - if (op->plen < op->len) - ftruncate(ts->fs->fd, op->offset + op->plen); + if (op->plen < op->len) { + rv = ftruncate(ts->fs->fd, op->offset + op->plen); + if (rv != 0) + goto exit; + } /* manually add the operation to the new transaction */ curop = malloc(sizeof(struct joper)); @@ -574,7 +577,7 @@ int jopen(struct jfs *fs, const char *name, int flags, int mode, int jflags) * pointer. This means that it must only be held while performing * operations that depend or alter the file pointer (jread, jreadv, * jwrite, jwritev), but the others (jpread, jpwrite) are left - * unprotected because they can be performed in paralell as long as + * unprotected because they can be performed in parallel as long as * they don't affect the same portion of the file (this is protected * by lockf). The lock doesn't slow things down tho: any threaded app * MUST implement this kind of locking anyways if it wants to prevent diff --git a/unix.c b/unix.c index 7e235b1..a2d8e9e 100644 --- a/unix.c +++ b/unix.c @@ -1,7 +1,7 @@ /* * libjio - A library for Journaled I/O - * Alberto Bertogli (albertogli@telpin.com.ar) + * Alberto Bertogli (albertito@blitiri.com.ar) * * UNIX API wrappers */ diff --git a/utils/exporter b/utils/exporter deleted file mode 100644 index 5e57713..0000000 --- a/utils/exporter +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/python - -import sys -import os -from xml.sax import saxutils -from xml.sax import make_parser -from xml.sax.handler import feature_namespaces - - -class Patch: - "Represents a single patch/record" - def __init__(self): - self.hash = '' - self.author = '' - self.date = '' - self.local_date = '' - self.name = '' - self.comment = '' - - def tostr(self): - s = "%s\n\tAuthor: %s\n\tDate: %s\n\tHash: %s\n" % \ - (self.name, self.author, self.date, self.hash) - return s - - def export(self, order, path): - # '/'s are not allowed in filenames - name = self.name.replace('/', '-') - - # avoid 'name..patch' - if name[-1] == '.': - name = name[:-1] - - file = "%s/%.2d - %s.patch" % (path, order, name) - cmd = 'darcs diff -u --match "hash %s" > "%s"' % \ - (self.hash, file) - if os.system(cmd): - print "Command failed: '%s'" % cmd - - -class BuildPatchList(saxutils.DefaultHandler): - def __init__(self): - self.db = {} - self.list = [] - self.cur_hash = '' - self.cur_elem = None - self.cur_val = '' - - def startElement(self, name, attrs): - if name == 'patch': - p = Patch() - p.author = attrs.get('author', None) - p.date = attrs.get('date', None) - p.local_date = attrs.get('local_date', None) - p.hash = attrs.get('hash') - self.db[p.hash] = p - self.current = p.hash - self.list.append(p.hash) - elif name == 'name': - self.db[self.current].name = '' - self.cur_elem = 'name' - elif name == 'comment': - self.db[self.current].comment = '' - self.cur_elem = 'name' - else: - self.cur_elem = None - - def characters(self, s): - if not self.cur_elem: - return - self.cur_val += s - - def endElement(self, name): - if name == 'name': - self.db[self.current].name = self.cur_val - elif name == 'comment': - self.db[self.current].current = self.cur_val - - self.cur_elem = None - self.cur_val = '' - - -# main - -if len(sys.argv) < 3: - print "Use: exporter [xmlfile|-] [list|export destdir]" - print - print "Examples:" - print " # darcs changes --xml-output | exporter - export /tmp" - print " # darcs changes --xml-output | exporter - list" - sys.exit(1) - -if sys.argv[1] == '-': - file = sys.stdin -else: - file = sys.argv[1] - -parser = make_parser() -parser.setFeature(feature_namespaces, 0) - -handler = BuildPatchList() -parser.setContentHandler(handler) -parser.parse(file) - -# reverse the list so the oldest is the first, and the newest is the last -handler.list.reverse() - -# we now have two main structures: handler.db is the hash table of Patches, -# indexed by their hash, and handler.list is the ordered list of hashes. - -if sys.argv[2] == 'list': - c = 1 - for h in handler.list: - print "%.2d:" % c, handler.db[h].tostr() - c += 1 -elif sys.argv[2] == 'export': - if len(sys.argv) < 4: - print "Destination directory missing" - sys.exit(1) - c = 1 - for h in handler.list: - p = handler.db[h] - print "%.2d: %s" % (c, p.name) - p.export(c, sys.argv[3]) - c += 1 -else: - print "Unknown parameter" - diff --git a/utils/release b/utils/release deleted file mode 100644 index 72000dd..0000000 --- a/utils/release +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -PKG=libjio - -if [ "$1" == "" -o "$2" == "" ]; then - echo "Use: release OLDREL NEWREL" - echo "Run from the repo root" - exit -fi - -OLDREL=$1 -NEWREL=$2 - -TARGZBALL="$PKG-$NEWREL.tar.gz" -TARBZBALL="$PKG-$NEWREL.tar.bz2" -RELDIR="../$NEWREL" - -if [ -d $RELDIR ]; then - echo "$RELDIR already exists!" - exit -fi - -echo "* making $RELDIR" -mkdir $RELDIR > /dev/null 2> /dev/null - -echo "* darcs dist" -darcs dist -d $PKG-$NEWREL -mv $TARGZBALL $RELDIR - -echo "* darcs changes" -darcs changes --from-tag $OLDREL > $RELDIR/Changelog-$NEWREL - -echo "* darcs diff" -darcs diff -u --from-tag $OLDREL > $RELDIR/$PKG-$NEWREL.patch - -echo "* export patches" -mkdir $RELDIR/$PKG-$NEWREL-broken-out -darcs changes --xml-output --from-tag $OLDREL | \ - ./utils/exporter - export $RELDIR/$PKG-$NEWREL-broken-out/ - -echo "* unpack" -cd $RELDIR -tar -zxf $TARGZBALL - -echo "* tar.bz2" -tar -cjf $TARBZBALL $PKG-$NEWREL -