jread_advance.patch Properly advance file pointers in jread, jwrite and jwritev. use_longs.patch Make use of longer data types in python bindings. file_offset.patch Add some CFLAGS to Make.conf for more standards compatibility. jlseek_return_offset.patch Make jlseek properly return the offset. always_O_LARGEFILE.patch Don't use O_LARGEFILE, it's not standard and it shouldn't be needed. python_fileno.patch Implement fileno() in python bindings. install_permissions.patch Fix install permissions. samples_string.patch Fix the string length in samples/full.c. init_mutexes.patch Initialize mutexes' attributes. linger_deadlock.patch Fix an ugly deadlock when using lingering transactions. upgrading_doc.patch Update UPDATING document. check_lfs.patch Include a check in the header to force the use of Large File Support. linking_doc.patch Add a section on compiling and linking in the programming guide. gmake_on_bsds.patch Tell BSD people to use "gmake" instead of "make". version-0.20.patch Version 0.20 INSTALL | 2 + Make.conf | 3 + Makefile | 18 ++++----- UPGRADING | 4 ++ bindings/python/libjio.c | 44 ++++++++++++++-------- check.c | 4 +- doc/guide.lyx | 93 +++++++++++++++++++++++++++++++++++++++++++++++ doc/guide.txt | 41 +++++++++++++++++++- libjio.h | 11 +++++ samples/full.c | 6 +-- trans.c | 64 ++++++++++++++++++-------------- unix.c | 14 +++---- 12 files changed, 238 insertions(+), 66 deletions(-) unchanged: --- cur-root/unix.c 2004-09-05 23:38:44.351479984 -0300 +++ cur-root/unix.c 2004-09-08 13:29:08.018813672 -0300 @@ -31,9 +31,9 @@ rv = spread(fs->fd, buf, count, pos); plockf(fs->fd, F_UNLOCK, pos, count); - if (rv >= 0) { + if (rv > 0) { /* if success, advance the file pointer */ - lseek(fs->fd, count, SEEK_CUR); + lseek(fs->fd, rv, SEEK_CUR); } pthread_mutex_unlock(&(fs->lock)); @@ -92,9 +92,9 @@ rv = jtrans_commit(&ts); - if (rv >= 0) { + if (rv > 0) { /* if success, advance the file pointer */ - lseek(fs->fd, count, SEEK_CUR); + lseek(fs->fd, rv, SEEK_CUR); } pthread_mutex_unlock(&(fs->lock)); @@ -143,9 +143,9 @@ rv = jtrans_commit(&ts); - if (rv >= 0) { + if (rv > 0) { /* if success, advance the file pointer */ - lseek(fs->fd, sum, SEEK_CUR); + lseek(fs->fd, rv, SEEK_CUR); } pthread_mutex_unlock(&(fs->lock)); @@ -172,7 +172,7 @@ /* lseek wrapper */ off_t jlseek(struct jfs *fs, off_t offset, int whence) { - int rv; + off_t rv; pthread_mutex_lock(&(fs->lock)); rv = lseek(fs->fd, offset, whence); unchanged: --- cur-root/bindings/python/libjio.c 2004-09-14 02:24:49.517300904 -0300 +++ cur-root/bindings/python/libjio.c 2004-09-14 02:24:56.129295728 -0300 @@ -67,6 +67,20 @@ PyObject_Del(fp); } +/* fileno */ +PyDoc_STRVAR(jf_fileno__doc, +"fileno()\n\ +\n\ +Return the file descriptor number for the file.\n"); + +static PyObject *jf_fileno(jfileobject *fp, PyObject *args) +{ + if (!PyArg_ParseTuple(args, ":fileno")) + return NULL; + + return PyInt_FromLong(fp->fs->fd); +} + /* read */ PyDoc_STRVAR(jf_read__doc, "read(size)\n\ @@ -115,11 +129,11 @@ { long rv; long len; - long offset; + long long offset; unsigned char *buf; PyObject *r; - if (!PyArg_ParseTuple(args, "il:pread", &len, &offset)) + if (!PyArg_ParseTuple(args, "iL:pread", &len, &offset)) return NULL; buf = malloc(len); @@ -176,10 +190,10 @@ { long rv; unsigned char *buf; - long offset; + long long offset; long len; - if (!PyArg_ParseTuple(args, "s#l:pwrite", &buf, &len, &offset)) + if (!PyArg_ParseTuple(args, "s#L:pwrite", &buf, &len, &offset)) return NULL; Py_BEGIN_ALLOW_THREADS @@ -199,9 +213,9 @@ static PyObject *jf_truncate(jfileobject *fp, PyObject *args) { int rv; - long lenght; + long long lenght; - if (!PyArg_ParseTuple(args, "l:truncate", &lenght)) + if (!PyArg_ParseTuple(args, "L:truncate", &lenght)) return NULL; Py_BEGIN_ALLOW_THREADS @@ -211,7 +225,7 @@ if (rv != 0) return PyErr_SetFromErrno(PyExc_IOError); - return PyInt_FromLong(rv); + return PyLong_FromLongLong(rv); } /* lseek */ @@ -230,11 +244,11 @@ static PyObject *jf_lseek(jfileobject *fp, PyObject *args) { - long rv; + long long rv; int whence; - long offset; + long long offset; - if (!PyArg_ParseTuple(args, "li:lseek", &offset, &whence)) + if (!PyArg_ParseTuple(args, "Li:lseek", &offset, &whence)) return NULL; Py_BEGIN_ALLOW_THREADS @@ -244,7 +258,7 @@ if (rv == -1) return PyErr_SetFromErrno(PyExc_IOError); - return PyLong_FromLong(rv); + return PyLong_FromLongLong(rv); } /* jsync */ @@ -304,6 +318,7 @@ /* method table */ static PyMethodDef jfile_methods[] = { + { "fileno", (PyCFunction)jf_fileno, METH_VARARGS, jf_fileno__doc }, { "read", (PyCFunction)jf_read, METH_VARARGS, jf_read__doc }, { "pread", (PyCFunction)jf_pread, METH_VARARGS, jf_pread__doc }, { "write", (PyCFunction)jf_write, METH_VARARGS, jf_write__doc }, @@ -359,10 +374,10 @@ { long rv; long len; - long offset; + long long offset; unsigned char *buf; - if (!PyArg_ParseTuple(args, "s#l:add", &buf, &len, &offset)) + if (!PyArg_ParseTuple(args, "s#L:add", &buf, &len, &offset)) return NULL; rv = jtrans_add(tp->ts, buf, len, offset); @@ -465,7 +480,7 @@ jfileobject *fp; flags = O_RDWR; - mode = 0666; + mode = 0600; jflags = 0; if (!PyArg_ParseTuple(args, "s|iii:open", &file, &flags, &mode, @@ -616,7 +631,6 @@ PyModule_AddIntConstant(m, "O_NDELAY", O_NDELAY); PyModule_AddIntConstant(m, "O_SYNC", O_SYNC); PyModule_AddIntConstant(m, "O_ASYNC", O_ASYNC); - PyModule_AddIntConstant(m, "O_LARGEFILE", O_LARGEFILE); /* lseek constants */ PyModule_AddIntConstant(m, "SEEK_SET", SEEK_SET); diff -u cur-root/Make.conf cur-root/Make.conf --- cur-root/Make.conf 2004-09-08 13:29:06.144098672 -0300 +++ cur-root/Make.conf 2004-09-19 01:32:51.364065064 -0300 @@ -1,10 +1,11 @@ -VERSION="0.19" +VERSION="0.20" CC = gcc CFLAGS += -Wall -O6 \ -D_LARGEFILE_SOURCE=1 -D_LARGEFILE64_SOURCE=1 \ -D_LFS_LARGEFILE=1 -D_LFS64_LARGEFILE=1 \ + -D_FILE_OFFSET_BITS=64 `getconf LFS_CFLAGS 2>/dev/null` \ -D_XOPEN_SOURCE=500 ifdef DEBUG unchanged: --- cur-root/trans.c 2004-09-19 01:30:06.120185952 -0300 +++ cur-root/trans.c 2004-09-19 01:32:25.085060080 -0300 @@ -96,13 +96,18 @@ /* initialize a transaction structure */ void jtrans_init(struct jfs *fs, struct jtrans *ts) { + pthread_mutexattr_t attr; + ts->fs = fs; ts->name = NULL; ts->id = 0; ts->flags = fs->flags; ts->op = NULL; ts->numops = 0; - pthread_mutex_init( &(ts->lock), NULL); + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); + pthread_mutex_init( &(ts->lock), &attr); + pthread_mutexattr_destroy(&attr); } @@ -209,7 +214,7 @@ /* open the transaction file */ if (!get_jtfile(ts->fs->name, id, name)) goto exit; - fd = open(name, O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, 0600); + fd = open(name, O_RDWR | O_CREAT | O_TRUNC, 0600); if (fd < 0) goto exit; @@ -368,10 +373,10 @@ linger->id = id; linger->name = strdup(name); - pthread_mutex_lock(&(ts->fs->lock)); + pthread_mutex_lock(&(ts->fs->ltlock)); linger->next = ts->fs->ltrans; ts->fs->ltrans = linger; - pthread_mutex_unlock(&(ts->fs->lock)); + pthread_mutex_unlock(&(ts->fs->ltlock)); } else { /* the transaction has been applied, so we cleanup and remove * it from the disk */ @@ -511,6 +516,7 @@ unsigned int t; char jdir[PATH_MAX], jlockfile[PATH_MAX]; struct stat sinfo; + pthread_mutexattr_t attr; fs->fd = -1; fs->jfd = -1; @@ -529,18 +535,23 @@ fs->ltrans = NULL; /* Note on fs->lock usage: this lock is used only to protect the file - * pointer and the linger list. This means that it must only be held - * while performing operations that depend or alter the file pointer - * or the linger list (jread, jreadv, jwrite, jwritev), but the others - * (jpread, jpwrite) are left unprotected because they can be - * performed in paralell as long as they don't affect the same portion - * of the file (this is protected by lockf). The lock doesn't slow - * things down tho: any threaded app MUST implement this kind of - * locking anyways if it wants to prevent data corruption, we only - * make it easier for them by taking care of it here. If performance - * is essential, the jpread/jpwrite functions should be used, just as - * real life. */ - pthread_mutex_init( &(fs->lock), NULL); + * pointer. This means that it must only be held while performing + * operations that depend or alter the file pointer (jread, jreadv, + * jwrite, jwritev), but the others (jpread, jpwrite) are left + * unprotected because they can be performed in paralell as long as + * they don't affect the same portion of the file (this is protected + * by lockf). The lock doesn't slow things down tho: any threaded app + * MUST implement this kind of locking anyways if it wants to prevent + * data corruption, we only make it easier for them by taking care of + * it here. If performance is essential, the jpread/jpwrite functions + * should be used, just as real life. + * About fs->ltlock, it's used to protect the lingering transactions + * list, fs->ltrans. */ + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); + pthread_mutex_init( &(fs->lock), &attr); + pthread_mutex_init( &(fs->ltlock), &attr); + pthread_mutexattr_destroy(&attr); fs->fd = open(name, flags, mode); if (fs->fd < 0) @@ -600,7 +611,7 @@ int jsync(struct jfs *fs) { int rv; - struct jlinger *linger, *ltmp; + struct jlinger *ltmp; if (fs->fd < 0) return -1; @@ -609,20 +620,19 @@ if (rv != 0) return rv; - pthread_mutex_lock(&(fs->lock)); - linger = fs->ltrans; - while (linger != NULL) { - free_tid(fs, linger->id); - unlink(linger->name); - free(linger->name); + pthread_mutex_lock(&(fs->ltlock)); + while (fs->ltrans != NULL) { + free_tid(fs, fs->ltrans->id); + unlink(fs->ltrans->name); + free(fs->ltrans->name); - ltmp = linger->next; - free(linger); + ltmp = fs->ltrans->next; + free(fs->ltrans); - linger = ltmp; + fs->ltrans = ltmp; } - pthread_mutex_unlock(&(fs->lock)); + pthread_mutex_unlock(&(fs->ltlock)); return 0; } unchanged: --- cur/check.c~always_O_LARGEFILE 2004-09-14 02:05:25.407272528 -0300 +++ cur-root/check.c 2004-09-14 02:05:38.805235728 -0300 @@ -126,7 +126,7 @@ int jfsck(const char *name, struct jfsck res->apply_error = 0; res->reapplied = 0; - fs.fd = open(name, O_RDWR | O_SYNC | O_LARGEFILE); + fs.fd = open(name, O_RDWR | O_SYNC); if (fs.fd < 0) { ret = J_ENOENT; goto exit; @@ -214,7 +214,7 @@ int jfsck(const char *name, struct jfsck ret = J_ENOMEM; goto exit; } - tfd = open(tname, O_RDWR | O_SYNC | O_LARGEFILE, 0600); + tfd = open(tname, O_RDWR | O_SYNC, 0600); if (tfd < 0) { res->invalid++; goto loop; unchanged: --- cur/Makefile~install_permissions 2004-09-14 02:25:01.422491040 -0300 +++ cur-root/Makefile 2004-09-14 02:25:01.425490584 -0300 @@ -20,15 +20,15 @@ jiofsck: jiofsck.o static $(CC) jiofsck.o libjio.a -lpthread -o jiofsck install: all - install -g root -o root -d $(PREFIX)/lib - install -g root -o root -m 0755 libjio.so $(PREFIX)/lib - install -g root -o root -m 0644 libjio.a $(PREFIX)/lib - install -g root -o root -d $(PREFIX)/include - install -g root -o root -m 0644 libjio.h $(PREFIX)/include - install -g root -o root -d $(PREFIX)/bin - install -g root -o root -m 0775 jiofsck $(PREFIX)/bin - install -g root -o root -d $(PREFIX)/man/man3 - install -g root -o root -m 0644 doc/libjio.3 $(PREFIX)/man/man3/ + install -d $(PREFIX)/lib + install -m 0755 libjio.so $(PREFIX)/lib + install -m 0644 libjio.a $(PREFIX)/lib + install -d $(PREFIX)/include + install -m 0644 libjio.h $(PREFIX)/include + install -d $(PREFIX)/bin + install -m 0775 jiofsck $(PREFIX)/bin + install -d $(PREFIX)/man/man3 + install -m 0644 doc/libjio.3 $(PREFIX)/man/man3/ @echo @echo "Please run ldconfig to update your library cache" @echo unchanged: --- cur/samples/full.c~samples_string 2004-09-17 10:22:40.906958816 -0300 +++ cur-root/samples/full.c 2004-09-17 10:27:18.066824112 -0300 @@ -7,7 +7,7 @@ #include #define FILENAME "test1" -#define TEXT "Hello world!" +#define TEXT "Hello world!\n" int main(void) { @@ -30,14 +30,14 @@ int main(void) /* write two "Hello world"s next to each other */ jtrans_init(&file, &trans); jtrans_add(&trans, TEXT, strlen(TEXT), 0); - jtrans_add(&trans, TEXT, strlen(TEXT), strlen(TEXT) - 1); + jtrans_add(&trans, TEXT, strlen(TEXT), strlen(TEXT)); r = jtrans_commit(&trans); if (r < 0) { perror("jtrans_commit"); return 1; } - /* at this point the file has "Hello world!Hello world!" */ + /* at this point the file has "Hello world!\nHello world!\n" */ /* now we rollback */ r = jtrans_rollback(&trans); unchanged: --- cur-root/libjio.h 2004-09-19 01:32:08.438590728 -0300 +++ cur-root/libjio.h 2004-09-19 01:32:35.668451160 -0300 @@ -12,6 +12,16 @@ #include #include #include +#include + +/* Check if we're using Large File Support - otherwise refuse to build. + * Otherwise, we would allow applications not using LFS to link with the + * library (which uses LFS) and that's just begging for problems. There should + * be a portable way for the C library to do some of this for us, but until I + * find one, this is the best we can do */ +#if (!defined _FILE_OFFSET_BITS) && (_FILE_OFFSET_BITS != 64) +#error "You must compile your application with Large File Support" +#endif #ifdef __cplusplus extern "C" { @@ -29,6 +39,7 @@ unsigned int *jmap; /* journal's lock file mmap area */ uint32_t flags; /* journal flags */ struct jlinger *ltrans; /* lingered transactions */ + pthread_mutex_t ltlock; /* lingered transactions' lock */ pthread_mutex_t lock; /* a soft lock used in some operations */ }; unchanged: --- cur/UPGRADING~upgrading_doc 2004-09-19 01:32:31.230125888 -0300 +++ cur-root/UPGRADING 2004-09-19 01:32:31.233125432 -0300 @@ -11,6 +11,10 @@ take much. If it's mandatory, it will be If you want to see what motivated the changes, see the changelog or just ask. +0.19 -> 0.20 +* Applications need to be recompiled due to a change in the jfs structure. +* When you link your applications with libjio, you need to make sure you + compile using large file support. 0.18 -> 0.19 * Applications need to be recompiled due to a change in the flag numbering. unchanged: --- cur/doc/guide.lyx~linking_doc 2004-09-19 01:32:39.735832824 -0300 +++ cur-root/doc/guide.lyx 2004-09-19 01:32:39.739832216 -0300 @@ -657,6 +657,99 @@ fwrite() Let me know if you need them. \layout Section +Compiling and linking +\layout Standard + +When you want to use your library, besides including the +\emph on + +\begin_inset Quotes eld +\end_inset + +libjio.h +\begin_inset Quotes erd +\end_inset + + +\emph default + header, you have to make sure your application uses the Large File Support + ( +\begin_inset Quotes eld +\end_inset + +LFS +\begin_inset Quotes erd +\end_inset + + from now on), to be able to handle large files properly. + This means that you will have to pass some special standard flags to the + compiler, so your C library uses the same data types as the library. + For instance, on 32-bit platforms (like x86), when using LFS, offsets are + usually 64 bits, as opposed to the usual 32. +\layout Standard + +The library is always built with LFS; however, link it against an application + without LFS support could lead to serious problems because this kind of + size differences and ABI compatibility. +\layout Standard + +The Single Unix Specification standard proposes a simple and practical way + to get the flags you need to pass your C compiler to tell you want to compile + your application with LFS: use a program called +\emph on + +\begin_inset Quotes eld +\end_inset + +getconf +\begin_inset Quotes erd +\end_inset + + +\emph default + which should be called like +\emph on + +\begin_inset Quotes eld +\end_inset + +getconf LFS_CFLAGS +\begin_inset Quotes erd +\end_inset + + +\emph default +, and it outputs the appropiate parameters. + Sadly, not all platforms implement it, so it's also wise to pass +\emph on + +\begin_inset Quotes eld +\end_inset + +-D_FILE_OFFSET_BITS=64 +\begin_inset Quotes erd +\end_inset + + +\emph default + just in case. +\layout Standard + +In the end, the command line would be something like: +\layout LyX-Code + +gcc `getconf LFS_CFLAGS` -D_FILE_OFFSET_BITS=64 +\backslash + +\layout LyX-Code + + app.c -ljio -lpthread -o app +\layout Standard + +If you want more detailed information or examples, you can check out how + the library and sample applications get built. +\layout Section + Where to go from here \layout Standard unchanged: --- cur/doc/guide.txt~linking_doc 2004-09-19 01:32:39.736832672 -0300 +++ cur-root/doc/guide.txt 2004-09-19 01:32:39.740832064 -0300 @@ -18,7 +18,8 @@ Table of Contents 7 Other APIs 7.1 UNIX API 7.2 ANSI C API -8 Where to go from here +8 Compiling and linking +9 Where to go from here @@ -329,7 +330,43 @@ They're still in development and has not carefully, so I won't spend time documenting them. Let me know if you need them. -8 Where to go from here +8 Compiling and linking + +When you want to use your library, besides including +the "libjio.h" header, you have to make sure your +application uses the Large File Support ("LFS" from now +on), to be able to handle large files properly. This +means that you will have to pass some special standard +flags to the compiler, so your C library uses the same +data types as the library. For instance, on 32-bit +platforms (like x86), when using LFS, offsets are +usually 64 bits, as opposed to the usual 32. + +The library is always built with LFS; however, link it +against an application without LFS support could lead +to serious problems because this kind of size +differences and ABI compatibility. + +The Single Unix Specification standard proposes a +simple and practical way to get the flags you need to +pass your C compiler to tell you want to compile your +application with LFS: use a program called "getconf" +which should be called like "getconf LFS_CFLAGS", and it +outputs the appropiate parameters. Sadly, not all +platforms implement it, so it's also wise to pass " +-D_FILE_OFFSET_BITS=64" just in case. + +In the end, the command line would be something like: + +gcc `getconf LFS_CFLAGS` -D_FILE_OFFSET_BITS=64 \ + + app.c -ljio -lpthread -o app + +If you want more detailed information or examples, you +can check out how the library and sample applications +get built. + +9 Where to go from here If you're still interested in learning more, you can find some small and clean samples are in the "samples" unchanged: --- cur/INSTALL~gmake_on_bsds 2004-09-19 01:32:45.142010960 -0300 +++ cur-root/INSTALL 2004-09-19 01:32:45.145010504 -0300 @@ -30,6 +30,8 @@ ldconfig There are other small options that might be useful (like compiling with debugging information), you can check the Make.conf file for more information. +If you use any BSD, or if the default "make" is not GNU make, use "gmake" +instead. After installing, you're ready to use the library; you can see how by looking at the manpage with "man libjio".