manpage-typos.patch Fix some manpage typos. diffstat_in_prerelease.patch Make the prerelease script add diffstat output in the release's patch. always_open_read.patch Always open files with reading capabilities. split_wrappers.patch Move wrappers to an independant file and split the code more logically. ansi_wrappers.patch Implement wrappers for most common ANSI C I/O functions (fread, fwrite, etc.). remove_html.patch Remove doc/libjio.html benchmarks_readme.patch Add a comment about the benchmark patches on the testing README. version-0.15.patch Version 0.15 cur-root/Make.conf | 2 cur-root/Makefile | 12 cur-root/README | 8 cur-root/ansi.c | 206 ++++++++++ cur-root/common.c | 92 ++++ cur-root/common.h | 17 cur-root/doc/layout | 21 + cur-root/doc/libjio.3 | 4 cur-root/libjio.c | 7 cur-root/libjio.h | 38 + cur-root/tests/README | 3 cur-root/trans.c | 680 +++++++++++++++++++++++++++++++++ cur-root/unix.c | 191 +++++++++ cur-root/utils/prerelease | 3 cur/doc/libjio.html | 444 --------------------- cur/libjio.c | 934 ---------------------------------------------- 16 files changed, 1259 insertions(+), 1403 deletions(-) unchanged: --- cur/doc/libjio.3~manpage-typos 2004-05-06 21:49:26.442867264 -0300 +++ cur-root/doc/libjio.3 2004-05-06 21:51:21.163427088 -0300 @@ -100,8 +100,8 @@ http://auriga.wearlab.de/~alb/libjio. We can group the functions into three groups: a common one, with functions common to the other two; low-level one, which consists of jtrans_commit and -jtrans_receive. They provide a method for manipulating transactions, which are -defined in a structure named struct jtrans (described above). +jtrans_rollback. They provide a method for manipulating transactions, which +are defined in the jtrans structure (described above). The second group mimics somehow the traditional UNIX API by providing similar interfaces to read(), write(), and their friends. unchanged: --- cur/utils/prerelease~diffstat_in_prerelease 2004-05-06 21:55:44.892334200 -0300 +++ cur-root/utils/prerelease 2004-05-06 21:56:17.230418064 -0300 @@ -11,6 +11,9 @@ patchdesc $(cat ps/series) > ../Changelo echo >> ../libjio-$1.patch echo >> ../libjio-$1.patch combine-series ../libjio-$1-t1.patch +diffstat ../libjio-$1-t1.patch >> ../libjio-$1.patch +echo >> ../libjio-$1.patch +echo >> ../libjio-$1.patch cat ../libjio-$1-t1.patch >> ../libjio-$1.patch rm ../libjio-$1-t1.patch unchanged: --- cur/libjio.c~always_open_read 2004-05-20 11:26:21.236390968 -0300 +++ cur-root/libjio.c 2004-05-20 11:42:56.569077400 -0300 @@ -441,6 +441,13 @@ int jopen(struct jfs *fs, const char *na unsigned int t; char jdir[PATH_MAX], jlockfile[PATH_MAX]; struct stat sinfo; + + /* we always need read access, because when we commit a transaction we + * read the current contents before applying */ + if (flags & O_WRONLY) { + flags = flags & ~O_WRONLY; + flags = flags | O_RDWR; + } fd = open(name, flags, mode); if (fd < 0) unchanged: --- cur/libjio.c +++ /dev/null 2004-04-13 23:59:22.000000000 -0300 @@ -1,934 +0,0 @@ - -/* - * libjio - A library for Journaled I/O - * Alberto Bertogli (albertogli@telpin.com.ar) - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "libjio.h" - - -/* - * small util functions - */ - -/* like lockf, but lock always from the beginning of the file */ -static off_t plockf(int fd, int cmd, off_t offset, off_t len) -{ - struct flock fl; - int op; - - if (cmd == F_LOCK) { - fl.l_type = F_WRLCK; - op = F_SETLKW; - } else if (cmd == F_ULOCK) { - fl.l_type = F_UNLCK; - op = F_SETLKW; - } else if (cmd == F_TLOCK) { - fl.l_type = F_WRLCK; - op = F_SETLK; - } else - return 0; - - fl.l_whence = SEEK_SET; - fl.l_start = offset; - fl.l_len = len; - - return fcntl(fd, op, &fl); -} - -/* like pread but either fails, or return a complete read; if we return less - * than count is because EOF was reached */ -static ssize_t spread(int fd, void *buf, size_t count, off_t offset) -{ - int rv, c; - - c = 0; - - while (c < count) { - rv = pread(fd, (char *) buf + c, count - c, offset + c); - - if (rv == count) - /* we're done */ - return count; - else if (rv < 0) - /* error */ - return rv; - else if (rv == 0) - /* got EOF */ - return c; - - /* incomplete read, keep on reading */ - c += rv; - } - - return count; -} - -/* like spread() but for pwrite() */ -static ssize_t spwrite(int fd, const void *buf, size_t count, off_t offset) -{ - int rv, c; - - c = 0; - - while (c < count) { - rv = pwrite(fd, (char *) buf + c, count - c, offset + c); - - if (rv == count) - /* we're done */ - return count; - else if (rv <= 0) - /* error/nothing was written */ - return rv; - - /* incomplete write, keep on writing */ - c += rv; - } - - return count; -} - -/* build the journal directory name out of the filename */ -static int get_jdir(const char *filename, char *jdir) -{ - char *base, *baset; - char *dir, *dirt; - - baset = strdup(filename); - if (baset == NULL) - return 0; - base = basename(baset); - - dirt = strdup(filename); - if (dirt == NULL) - return 0; - dir = dirname(dirt); - - snprintf(jdir, PATH_MAX, "%s/.%s.jio", dir, base); - - free(baset); - free(dirt); - - return 1; -} - -/* build the filename of a given transaction */ -static int get_jtfile(const char *filename, int tid, char *jtfile) -{ - char *base, *baset; - char *dir, *dirt; - - baset = strdup(filename); - if (baset == NULL) - return 0; - base = basename(baset); - - dirt = strdup(filename); - if (dirt == NULL) - return 0; - dir = dirname(dirt); - - snprintf(jtfile, PATH_MAX, "%s/.%s.jio/%d", dir, base, tid); - - free(baset); - free(dirt); - - return 1; -} - -/* gets a new transaction id */ -static unsigned int get_tid(struct jfs *fs) -{ - unsigned int curid; - int r, rv; - - /* lock the whole file */ - plockf(fs->jfd, F_LOCK, 0, 0); - - /* read the current max. curid */ - r = spread(fs->jfd, &curid, sizeof(curid), 0); - if (r != sizeof(curid)) { - rv = 0; - goto exit; - } - - /* increment it and handle overflows */ - rv = curid + 1; - if (rv == 0) - rv = 1; - - /* write to the file descriptor */ - r = spwrite(fs->jfd, &rv, sizeof(rv), 0); - if (r != sizeof(curid)) { - rv = 0; - goto exit; - } - -exit: - plockf(fs->jfd, F_ULOCK, 0, 0); - return rv; -} - -/* frees a transaction id */ -static void free_tid(struct jfs *fs, unsigned int tid) -{ - unsigned int curid, i; - int r; - char name[PATH_MAX]; - - /* lock the whole file */ - plockf(fs->jfd, F_LOCK, 0, 0); - - /* read the current max. curid */ - r = spread(fs->jfd, &curid, sizeof(curid), 0); - if (r != sizeof(curid)) { - goto exit; - } - - if (tid < curid) { - /* we're not freeing the max. curid, so we just return */ - goto exit; - } else { - /* look up the new max. */ - for (i = curid - 1; i > 0; i--) { - /* this can fail if we're low on mem, but we don't - * care checking here because the problem will come - * out later and we can fail more properly */ - get_jtfile(fs->name, i, name); - if (access(name, R_OK | W_OK) == 0) { - curid = i; - break; - } - } - - /* and save it */ - r = spwrite(fs->jfd, &i, sizeof(i), 0); - if (r != sizeof(curid)) { - goto exit; - } - } - -exit: - plockf(fs->jfd, F_ULOCK, 0, 0); - return; -} - - -/* - * transaction functions - */ - -/* initialize a transaction structure */ -void jtrans_init(struct jfs *fs, struct jtrans *ts) -{ - ts->fs = fs; - ts->name = NULL; - ts->id = 0; - ts->flags = 0; - ts->buf = NULL; - ts->len = 0; - ts->offset = 0; - ts->udata = NULL; - ts->ulen = 0; - ts->pdata = NULL; - ts->plen = 0; -} - -/* free the contents of a transaction structure */ -void jtrans_free(struct jtrans *ts) -{ - /* NOTE: we only really free the name and previous data, which are the - * things _we_ allocate; the user data is caller stuff */ - ts->fs = NULL; - if (ts->name) - free(ts->name); - if (ts->pdata) - free(ts->pdata); - - /* don't free ts itself, it's very common to allocate it in the stack, - * so let the caller take care of it; and, after all, he was the one - * doing the alloc in the first place */ -} - -/* commit a transaction */ -int jtrans_commit(struct jtrans *ts) -{ - int id, fd, rv, t; - char *name; - unsigned char *buf_init, *bufp; - - name = (char *) malloc(PATH_MAX); - if (name == NULL) - return -1; - - id = get_tid(ts->fs); - if (id == 0) - return -1; - - /* open the transaction file */ - if (!get_jtfile(ts->fs->name, id, name)) - return -1; - fd = open(name, O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, 0600); - if (fd < 0) - return -1; - - /* and lock it */ - plockf(fd, F_LOCK, 0, 0); - - ts->id = id; - ts->name = name; - - /* lock the file region to work on */ - if (!(ts->fs->flags & J_NOLOCK)) - plockf(ts->fs->fd, F_LOCK, ts->offset, ts->len); - - /* read the current content and fill in the transaction structure */ - ts->pdata = malloc(ts->len); - if (ts->pdata == NULL) - goto exit; - - ts->plen = ts->len; - - rv = spread(ts->fs->fd, ts->pdata, ts->len, ts->offset); - if (rv < 0) - goto exit; - if (rv < ts->len) { - /* we are extending the file! use ftruncate() to do it */ - ftruncate(ts->fs->fd, ts->offset + ts->len); - ts->plen = rv; - } - - /* now save the transaction to the file, static data first */ - - buf_init = malloc(J_DISKTFIXSIZE); - if (buf_init == NULL) - return -1; - - bufp = buf_init; - - memcpy(bufp, (void *) &(ts->id), sizeof(ts->id)); - bufp += 4; - - memcpy(bufp, (void *) &(ts->flags), sizeof(ts->flags)); - bufp += 4; - - memcpy(bufp, (void *) &(ts->len), sizeof(ts->len)); - bufp += 4; - - memcpy(bufp, (void *) &(ts->plen), sizeof(ts->plen)); - bufp += 4; - - memcpy(bufp, (void *) &(ts->ulen), sizeof(ts->ulen)); - bufp += 4; - - memcpy(bufp, (void *) &(ts->offset), sizeof(ts->offset)); - bufp += 8; - - rv = spwrite(fd, buf_init, J_DISKTFIXSIZE, 0); - if (rv != J_DISKTFIXSIZE) - goto exit; - - free(buf_init); - - - /* and now the variable data */ - - if (ts->udata) { - rv = spwrite(fd, ts->udata, ts->ulen, J_DISKTFIXSIZE); - if (rv != ts->ulen) - goto exit; - } - - t = J_DISKTFIXSIZE + ts->ulen; - rv = spwrite(fd, ts->pdata, ts->plen, t); - if (rv != ts->plen) - goto exit; - - /* this is a simple but efficient optimization: instead of doing - * everything O_SYNC, we sync at this point only, this way we avoid - * doing a lot of very small writes; in case of a crash the - * transaction file is only useful if it's complete (ie. after this - * point) so we only flush here */ - fsync(fd); - - /* now that we have a safe transaction file, let's apply it */ - rv = spwrite(ts->fs->fd, ts->buf, ts->len, ts->offset); - if (rv != ts->len) - goto exit; - - /* the transaction has been applied, so we cleanup and remove it from - * the disk */ - free_tid(ts->fs, ts->id); - unlink(name); - - /* mark the transaction as commited, _after_ it was removed */ - ts->flags = ts->flags | J_COMMITED; - - -exit: - close(fd); - - if (!(ts->fs->flags & J_NOLOCK)) - plockf(ts->fs->fd, F_ULOCK, ts->offset, ts->len); - - /* return the lenght only if it was properly commited */ - if (ts->flags & J_COMMITED) - return ts->len; - else - return -1; - -} - -/* rollback a transaction */ -int jtrans_rollback(struct jtrans *ts) -{ - int rv; - struct jtrans newts; - - /* copy the old transaction to the new one */ - jtrans_init(ts->fs, &newts); - - newts.flags = ts->flags; - newts.offset = ts->offset; - - newts.buf = ts->pdata; - newts.len = ts->plen; - - if (ts->plen < ts->len) { - /* we extended the data in the previous transaction, so we - * should truncate it back */ - /* DANGEROUS: this is one of the main reasons why rollbacking - * is dangerous and should only be done with extreme caution: - * if for some reason, after the previous transacton, we have - * extended the file further, this will cut it back to what it - * was; read the docs for more detail */ - ftruncate(ts->fs->fd, ts->offset + ts->plen); - - } - - newts.pdata = ts->pdata; - newts.plen = ts->plen; - - newts.udata = ts->udata; - newts.ulen = ts->ulen; - - rv = jtrans_commit(&newts); - return rv; -} - - -/* - * basic operations - */ - -/* open a file */ -int jopen(struct jfs *fs, const char *name, int flags, int mode, int jflags) -{ - int fd, jfd, rv; - unsigned int t; - char jdir[PATH_MAX], jlockfile[PATH_MAX]; - struct stat sinfo; - - /* we always need read access, because when we commit a transaction we - * read the current contents before applying */ - if (flags & O_WRONLY) { - flags = flags & ~O_WRONLY; - flags = flags | O_RDWR; - } - - fd = open(name, flags, mode); - if (fd < 0) - return -1; - - fs->fd = fd; - fs->name = strdup(name); - fs->flags = jflags; - - /* Note on fs->lock usage: this lock is used only inside the wrappers, - * and exclusively to protect the file pointer. This means that it - * must only be held while performing operations that depend or alter - * the file pointer (jread, jreadv, jwrite, jwritev), but the others - * (jpread, jpwrite) are left unprotected because they can be - * performed in paralell as long as they don't affect the same portion - * of the file (this is protected by lockf). The lock doesn't slow - * things down tho: any threaded app MUST implement this kind of - * locking anyways if it wants to prevent data corruption, we only - * make it easier for them by taking care of it here. If performance - * is essential, the jpread/jpwrite functions should be used, just as - * real life. */ - - pthread_mutex_init( &(fs->lock), NULL); - - if (!get_jdir(name, jdir)) - return -1; - rv = mkdir(jdir, 0750); - rv = lstat(jdir, &sinfo); - if (rv < 0 || !S_ISDIR(sinfo.st_mode)) - return -1; - - snprintf(jlockfile, PATH_MAX, "%s/%s", jdir, "lock"); - jfd = open(jlockfile, O_RDWR | O_CREAT, 0600); - if (jfd < 0) - return -1; - - /* initialize the lock file by writing the first tid to it, but only - * if its empty, otherwise there is a race if two processes call - * jopen() simultaneously and both initialize the file */ - plockf(jfd, F_LOCK, 0, 0); - lstat(jlockfile, &sinfo); - if (sinfo.st_size == 0) { - t = 1; - rv = write(jfd, &t, sizeof(t)); - if (rv != sizeof(t)) { - plockf(jfd, F_ULOCK, 0, 0); - return -1; - } - } - plockf(jfd, F_ULOCK, 0, 0); - - fs->jfd = jfd; - - return fd; -} - - -/* read() family wrappers */ - -/* read wrapper */ -ssize_t jread(struct jfs *fs, void *buf, size_t count) -{ - int rv; - off_t pos; - - pthread_mutex_lock(&(fs->lock)); - - pos = lseek(fs->fd, 0, SEEK_CUR); - - plockf(fs->fd, F_LOCK, pos, count); - rv = spread(fs->fd, buf, count, pos); - plockf(fs->fd, F_ULOCK, pos, count); - - if (rv == count) { - /* if success, advance the file pointer */ - lseek(fs->fd, count, SEEK_CUR); - } - - pthread_mutex_unlock(&(fs->lock)); - - return rv; -} - -/* pread wrapper */ -ssize_t jpread(struct jfs *fs, void *buf, size_t count, off_t offset) -{ - int rv; - - plockf(fs->fd, F_LOCK, offset, count); - rv = spread(fs->fd, buf, count, offset); - plockf(fs->fd, F_ULOCK, offset, count); - - return rv; -} - -/* readv wrapper */ -ssize_t jreadv(struct jfs *fs, struct iovec *vector, int count) -{ - int rv, i; - size_t sum; - off_t pos; - - sum = 0; - for (i = 0; i < count; i++) - sum += vector[i].iov_len; - - pthread_mutex_lock(&(fs->lock)); - pos = lseek(fs->fd, 0, SEEK_CUR); - plockf(fs->fd, F_LOCK, pos, count); - rv = readv(fs->fd, vector, count); - plockf(fs->fd, F_ULOCK, pos, count); - pthread_mutex_unlock(&(fs->lock)); - - return rv; -} - -/* write wrapper */ -ssize_t jwrite(struct jfs *fs, const void *buf, size_t count) -{ - int rv; - off_t pos; - struct jtrans ts; - - pthread_mutex_lock(&(fs->lock)); - - jtrans_init(fs, &ts); - pos = lseek(fs->fd, 0, SEEK_CUR); - ts.offset = pos; - - ts.buf = buf; - ts.len = count; - - rv = jtrans_commit(&ts); - - if (rv >= 0) { - /* if success, advance the file pointer */ - lseek(fs->fd, count, SEEK_CUR); - } - - pthread_mutex_unlock(&(fs->lock)); - - jtrans_free(&ts); - - return rv; -} - -/* write family wrappers */ - -/* pwrite wrapper */ -ssize_t jpwrite(struct jfs *fs, const void *buf, size_t count, off_t offset) -{ - int rv; - struct jtrans ts; - - jtrans_init(fs, &ts); - ts.offset = offset; - - ts.buf = buf; - ts.len = count; - - rv = jtrans_commit(&ts); - - jtrans_free(&ts); - - return rv; -} - -/* writev wrapper */ -ssize_t jwritev(struct jfs *fs, const struct iovec *vector, int count) -{ - int rv, i, bufp; - ssize_t sum; - char *buf; - off_t pos; - struct jtrans ts; - - sum = 0; - for (i = 0; i < count; i++) - sum += vector[i].iov_len; - - /* unify the buffers into one big chunk to commit */ - /* FIXME: can't we do this more efficient? It ruins the whole purpose - * of using writev() :\ - * maybe we should do one transaction per vector */ - buf = malloc(sum); - if (buf == NULL) - return -1; - bufp = 0; - - for (i = 0; i < count; i++) { - memcpy(buf + bufp, vector[i].iov_base, vector[i].iov_len); - bufp += vector[i].iov_len; - } - - pthread_mutex_lock(&(fs->lock)); - - jtrans_init(fs, &ts); - pos = lseek(fs->fd, 0, SEEK_CUR); - ts.offset = pos; - - ts.buf = buf; - ts.len = sum; - - rv = jtrans_commit(&ts); - - if (rv >= 0) { - /* if success, advance the file pointer */ - lseek(fs->fd, count, SEEK_CUR); - } - - pthread_mutex_unlock(&(fs->lock)); - - jtrans_free(&ts); - - return rv; - -} - -/* truncate a file - be careful with this */ -int jtruncate(struct jfs *fs, off_t lenght) -{ - int rv; - - /* lock from lenght to the end of file */ - plockf(fs->fd, F_LOCK, lenght, 0); - rv = ftruncate(fs->fd, lenght); - plockf(fs->fd, F_ULOCK, lenght, 0); - - return rv; -} - -/* close a file */ -int jclose(struct jfs *fs) -{ - if (close(fs->fd)) - return -1; - if (close(fs->jfd)) - return -1; - if (fs->name) - /* allocated by strdup() in jopen() */ - free(fs->name); - return 0; -} - - -/* - * journal recovery - */ - -/* check the journal and replay the incomplete transactions */ -int jfsck(const char *name, struct jfsck_result *res) -{ - int fd, tfd, rv, i, maxtid; - char jdir[PATH_MAX], jlockfile[PATH_MAX], tname[PATH_MAX]; - unsigned char *buf = NULL; - struct stat sinfo; - struct jfs fs; - struct jtrans *curts; - DIR *dir; - struct dirent *dent; - off_t offset; - - fd = open(name, O_RDWR | O_SYNC | O_LARGEFILE); - if (fd < 0) - return J_ENOENT; - - fs.fd = fd; - fs.name = (char *) name; - - if (!get_jdir(name, jdir)) - return J_ENOMEM; - rv = lstat(jdir, &sinfo); - if (rv < 0 || !S_ISDIR(sinfo.st_mode)) - return J_ENOJOURNAL; - - /* open the lock file, which is only used to complete the jfs - * structure */ - snprintf(jlockfile, PATH_MAX, "%s/%s", jdir, "lock"); - rv = open(jlockfile, O_RDWR | O_CREAT, 0600); - if (rv < 0) - return J_ENOJOURNAL; - fs.jfd = rv; - - dir = opendir(jdir); - if (dir == NULL) - return J_ENOJOURNAL; - - /* loop for each file in the journal directory to find out the greater - * transaction number */ - maxtid = 0; - for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) { - /* see if the file is named like a transaction, ignore - * otherwise; as transactions are named as numbers > 0, a - * simple atoi() is enough testing */ - rv = atoi(dent->d_name); - if (rv <= 0) - continue; - if (rv > maxtid) - maxtid = rv; - } - closedir(dir); - - /* rewrite the lockfile, writing the new maxtid on it, so that when we - * rollback a transaction it doesn't step over existing ones */ - rv = spwrite(fs.jfd, &maxtid, sizeof(maxtid), 0); - if (rv != sizeof(maxtid)) { - return J_ENOMEM; - } - - /* we loop all the way up to the max transaction id */ - for (i = 1; i <= maxtid; i++) { - curts = malloc(sizeof(struct jtrans)); - if (curts == NULL) - return J_ENOMEM; - - jtrans_init(&fs, curts); - curts->id = i; - - /* open the transaction file, using i as its name, so we are - * really looping in order (recovering transaction in a - * different order as they were applied means instant - * corruption) */ - if (!get_jtfile(name, i, tname)) - return J_ENOMEM; - tfd = open(tname, O_RDWR | O_SYNC | O_LARGEFILE, 0600); - if (tfd < 0) { - res->invalid++; - goto loop; - } - - /* try to lock the transaction file, if it's locked then it is - * currently being used so we skip it */ - rv = plockf(tfd, F_TLOCK, 0, 0); - if (rv == -1) { - res->in_progress++; - goto loop; - } - - /* load from disk, header first */ - buf = (unsigned char *) malloc(J_DISKTFIXSIZE); - if (buf == NULL) { - res->load_error++; - goto loop; - } - - rv = read(tfd, buf, J_DISKTFIXSIZE); - if (rv != J_DISKTFIXSIZE) { - res->broken_head++; - free(buf); - goto loop; - } - - curts->flags = *( (uint32_t *) (buf + 4)); - curts->len = *( (uint32_t *) (buf + 8)); - curts->plen = *( (uint32_t *) (buf + 12)); - curts->ulen = *( (uint32_t *) (buf + 16)); - curts->offset = *( (uint64_t *) (buf + 20)); - - free(buf); - - /* if we got here, the transaction was not applied, so we - * check if the transaction file is complete (we only need to - * rollback it) or not (so we can't do anything but ignore it) - */ - - lstat(tname, &sinfo); - rv = J_DISKTFIXSIZE + curts->ulen + curts->plen; - if (sinfo.st_size != rv) { - /* the transaction file is incomplete, some of the - * body is missing */ - res->broken_body++; - goto loop; - } - - /* we have a complete transaction file which commit was not - * successful, so we read it to complete the transaction - * structure and rollback it */ - curts->pdata = malloc(curts->plen); - if (curts->pdata == NULL) { - res->load_error++; - goto loop; - } - - curts->udata = malloc(curts->ulen); - if (curts->udata == NULL) { - res->load_error++; - goto loop; - } - - /* user data */ - offset = J_DISKTFIXSIZE; - rv = spread(tfd, curts->udata, curts->ulen, offset); - if (rv != curts->ulen) { - res->load_error++; - goto loop; - } - - /* previous data */ - offset = J_DISKTFIXSIZE + curts->ulen; - rv = spread(tfd, curts->pdata, curts->plen, offset); - if (rv != curts->plen) { - res->load_error++; - goto loop; - } - - /* rollback */ - rv = jtrans_rollback(curts); - if (rv < 0) { - res->apply_error++; - goto loop; - } - res->rollbacked++; - - /* free the data we just allocated */ - if (curts->plen) { - free(curts->pdata); - curts->pdata = NULL; - } - if (curts->ulen) { - free(curts->udata); - curts->udata = NULL; - } - if (curts->name) { - free(curts->name); - curts->name = NULL; - } - -loop: - if (tfd > 0) - close(tfd); - - free(curts); - - res->total++; - } - - close(fs.fd); - close(fs.jfd); - - return 0; - -} - -/* remove all the files in the journal directory (if any) */ -int jfsck_cleanup(const char *name) -{ - char jdir[PATH_MAX], tfile[PATH_MAX*3]; - DIR *dir; - struct dirent *dent; - - if (!get_jdir(name, jdir)) - return 0; - - dir = opendir(jdir); - if (dir == NULL && errno == ENOENT) - /* it doesn't exist, so it's clean */ - return 1; - else if (dir == NULL) - return 0; - - for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) { - /* we only care about transactions (named as numbers > 0) and - * the lockfile (named "lock"); ignore everything else */ - if (strcmp(dent->d_name, "lock") && atoi(dent->d_name) <= 0) - continue; - - /* build the full path to the transaction file */ - memset(tfile, 0, PATH_MAX * 3); - strcat(tfile, jdir); - strcat(tfile, "/"); - strcat(tfile, dent->d_name); - - /* the full filename is too large */ - if (strlen(tfile) > PATH_MAX) - return 0; - - /* and remove it */ - unlink(tfile); - } - closedir(dir); - - return 1; -} - unchanged: --- /dev/null 2004-04-13 23:59:22.000000000 -0300 +++ cur-root/unix.c 2004-05-29 22:17:51.351558248 -0300 @@ -0,0 +1,191 @@ + +/* + * libjio - A library for Journaled I/O + * Alberto Bertogli (albertogli@telpin.com.ar) + * + * UNIX API wrappers + */ + +#include +#include +#include +#include + +#include "libjio.h" +#include "common.h" + + +/* read() family wrappers */ + +/* read wrapper */ +ssize_t jread(struct jfs *fs, void *buf, size_t count) +{ + int rv; + off_t pos; + + pthread_mutex_lock(&(fs->lock)); + + pos = lseek(fs->fd, 0, SEEK_CUR); + + plockf(fs->fd, F_LOCK, pos, count); + rv = spread(fs->fd, buf, count, pos); + plockf(fs->fd, F_ULOCK, pos, count); + + if (rv == count) { + /* if success, advance the file pointer */ + lseek(fs->fd, count, SEEK_CUR); + } + + pthread_mutex_unlock(&(fs->lock)); + + return rv; +} + +/* pread wrapper */ +ssize_t jpread(struct jfs *fs, void *buf, size_t count, off_t offset) +{ + int rv; + + plockf(fs->fd, F_LOCK, offset, count); + rv = spread(fs->fd, buf, count, offset); + plockf(fs->fd, F_ULOCK, offset, count); + + return rv; +} + +/* readv wrapper */ +ssize_t jreadv(struct jfs *fs, struct iovec *vector, int count) +{ + int rv, i; + size_t sum; + off_t pos; + + sum = 0; + for (i = 0; i < count; i++) + sum += vector[i].iov_len; + + pthread_mutex_lock(&(fs->lock)); + pos = lseek(fs->fd, 0, SEEK_CUR); + plockf(fs->fd, F_LOCK, pos, count); + rv = readv(fs->fd, vector, count); + plockf(fs->fd, F_ULOCK, pos, count); + pthread_mutex_unlock(&(fs->lock)); + + return rv; +} + + +/* write family wrappers */ + +/* write wrapper */ +ssize_t jwrite(struct jfs *fs, const void *buf, size_t count) +{ + int rv; + off_t pos; + struct jtrans ts; + + pthread_mutex_lock(&(fs->lock)); + + jtrans_init(fs, &ts); + pos = lseek(fs->fd, 0, SEEK_CUR); + ts.offset = pos; + + ts.buf = buf; + ts.len = count; + + rv = jtrans_commit(&ts); + + if (rv >= 0) { + /* if success, advance the file pointer */ + lseek(fs->fd, count, SEEK_CUR); + } + + pthread_mutex_unlock(&(fs->lock)); + + jtrans_free(&ts); + + return rv; +} + +/* pwrite wrapper */ +ssize_t jpwrite(struct jfs *fs, const void *buf, size_t count, off_t offset) +{ + int rv; + struct jtrans ts; + + jtrans_init(fs, &ts); + ts.offset = offset; + + ts.buf = buf; + ts.len = count; + + rv = jtrans_commit(&ts); + + jtrans_free(&ts); + + return rv; +} + +/* writev wrapper */ +ssize_t jwritev(struct jfs *fs, const struct iovec *vector, int count) +{ + int rv, i, bufp; + ssize_t sum; + char *buf; + off_t pos; + struct jtrans ts; + + sum = 0; + for (i = 0; i < count; i++) + sum += vector[i].iov_len; + + /* unify the buffers into one big chunk to commit */ + /* FIXME: can't we do this more efficient? It ruins the whole purpose + * of using writev()! maybe we should do one transaction per vector */ + buf = malloc(sum); + if (buf == NULL) + return -1; + bufp = 0; + + for (i = 0; i < count; i++) { + memcpy(buf + bufp, vector[i].iov_base, vector[i].iov_len); + bufp += vector[i].iov_len; + } + + pthread_mutex_lock(&(fs->lock)); + + jtrans_init(fs, &ts); + pos = lseek(fs->fd, 0, SEEK_CUR); + ts.offset = pos; + + ts.buf = buf; + ts.len = sum; + + rv = jtrans_commit(&ts); + + if (rv >= 0) { + /* if success, advance the file pointer */ + lseek(fs->fd, count, SEEK_CUR); + } + + pthread_mutex_unlock(&(fs->lock)); + + jtrans_free(&ts); + + return rv; + +} + +/* truncate a file - be careful with this */ +int jtruncate(struct jfs *fs, off_t lenght) +{ + int rv; + + /* lock from lenght to the end of file */ + plockf(fs->fd, F_LOCK, lenght, 0); + rv = ftruncate(fs->fd, lenght); + plockf(fs->fd, F_ULOCK, lenght, 0); + + return rv; +} + unchanged: --- /dev/null 2004-04-13 23:59:22.000000000 -0300 +++ cur-root/doc/layout 2004-05-29 22:28:41.598705656 -0300 @@ -0,0 +1,21 @@ + +Source layout +------------- + +The source is structured so code can be read and reviewed in an independant +way, following the way the code really works. + +The main file is called "trans.c" which contains the transaction API; all the +other wrappers depend on it, and they're the core of the library. + +There is also a "common.c" file with some common functions. + +And finally, "unix.c" which implement the wrappers for the UNIX API (read(), +write() and their family), and "ansi.c" where ANSI wrappers live (fopen(), +fread(), etc.). + +If you're trying to read the code for the first time, you can start from +either from the more simple unix.c which relies on trans.c; or from trans.c +which is what everything else relies on, and then the wrappers will be +obvious. + unchanged: --- /dev/null 2004-04-13 23:59:22.000000000 -0300 +++ cur-root/common.c 2004-05-29 22:08:37.645734328 -0300 @@ -0,0 +1,92 @@ + +/* + * libjio - A library for Journaled I/O + * Alberto Bertogli (albertogli@telpin.com.ar) + * + * Common functions + */ + +#include +#include +#include + +#include "common.h" + + +/* like lockf, but lock always from the beginning of the file */ +off_t plockf(int fd, int cmd, off_t offset, off_t len) +{ + struct flock fl; + int op; + + if (cmd == F_LOCK) { + fl.l_type = F_WRLCK; + op = F_SETLKW; + } else if (cmd == F_ULOCK) { + fl.l_type = F_UNLCK; + op = F_SETLKW; + } else if (cmd == F_TLOCK) { + fl.l_type = F_WRLCK; + op = F_SETLK; + } else + return 0; + + fl.l_whence = SEEK_SET; + fl.l_start = offset; + fl.l_len = len; + + return fcntl(fd, op, &fl); +} + +/* like pread but either fails, or return a complete read; if we return less + * than count is because EOF was reached */ +ssize_t spread(int fd, void *buf, size_t count, off_t offset) +{ + int rv, c; + + c = 0; + + while (c < count) { + rv = pread(fd, (char *) buf + c, count - c, offset + c); + + if (rv == count) + /* we're done */ + return count; + else if (rv < 0) + /* error */ + return rv; + else if (rv == 0) + /* got EOF */ + return c; + + /* incomplete read, keep on reading */ + c += rv; + } + + return count; +} + +/* like spread() but for pwrite() */ +ssize_t spwrite(int fd, const void *buf, size_t count, off_t offset) +{ + int rv, c; + + c = 0; + + while (c < count) { + rv = pwrite(fd, (char *) buf + c, count - c, offset + c); + + if (rv == count) + /* we're done */ + return count; + else if (rv <= 0) + /* error/nothing was written */ + return rv; + + /* incomplete write, keep on writing */ + c += rv; + } + + return count; +} + unchanged: --- /dev/null 2004-04-13 23:59:22.000000000 -0300 +++ cur-root/common.h 2004-05-29 22:29:21.226681288 -0300 @@ -0,0 +1,17 @@ + +/* + * libjio - A library for Journaled I/O + * Alberto Bertogli (albertogli@telpin.com.ar) + * + * Header for internal functions + */ + +#ifndef _COMMON_H +#define _COMMON_H + +off_t plockf(int fd, int cmd, off_t offset, off_t len); +ssize_t spread(int fd, void *buf, size_t count, off_t offset); +ssize_t spwrite(int fd, const void *buf, size_t count, off_t offset); + +#endif + unchanged: --- cur-root/Makefile 2004-05-29 22:17:16.405870800 -0300 +++ cur-root/Makefile 2004-05-29 22:57:42.000000000 -0300 @@ -3,18 +3,18 @@ # objects to build -OBJS = libjio.o +OBJS = common.o trans.o unix.o ansi.o # rules default: all all: shared static jiofsck -shared: libjio.o - $(CC) -shared libjio.o -o libjio.so +shared: $(OBJS) + $(CC) -shared $(OBJS) -o libjio.so -static: libjio.o - $(AR) cr libjio.a libjio.o +static: $(OBJS) + $(AR) cr libjio.a $(OBJS) jiofsck: jiofsck.o static $(CC) jiofsck.o libjio.a -lpthread -o jiofsck @@ -38,7 +38,7 @@ clean: - rm -f libjio.o libjio.a libjio.so jiofsck.o jiofsck + rm -f $(OBJS) libjio.a libjio.so jiofsck.o jiofsck rm -f *.bb *.bbg *.da *.gcov gmon.out unchanged: --- /dev/null 2004-04-13 23:59:22.000000000 -0300 +++ cur-root/trans.c 2004-05-29 22:15:41.087361416 -0300 @@ -0,0 +1,680 @@ + +/* + * libjio - A library for Journaled I/O + * Alberto Bertogli (albertogli@telpin.com.ar) + * + * Core transaction API and recovery functions + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libjio.h" +#include "common.h" + + +/* build the journal directory name out of the filename */ +static int get_jdir(const char *filename, char *jdir) +{ + char *base, *baset; + char *dir, *dirt; + + baset = strdup(filename); + if (baset == NULL) + return 0; + base = basename(baset); + + dirt = strdup(filename); + if (dirt == NULL) + return 0; + dir = dirname(dirt); + + snprintf(jdir, PATH_MAX, "%s/.%s.jio", dir, base); + + free(baset); + free(dirt); + + return 1; +} + +/* build the filename of a given transaction */ +static int get_jtfile(const char *filename, int tid, char *jtfile) +{ + char *base, *baset; + char *dir, *dirt; + + baset = strdup(filename); + if (baset == NULL) + return 0; + base = basename(baset); + + dirt = strdup(filename); + if (dirt == NULL) + return 0; + dir = dirname(dirt); + + snprintf(jtfile, PATH_MAX, "%s/.%s.jio/%d", dir, base, tid); + + free(baset); + free(dirt); + + return 1; +} + +/* gets a new transaction id */ +static unsigned int get_tid(struct jfs *fs) +{ + unsigned int curid; + int r, rv; + + /* lock the whole file */ + plockf(fs->jfd, F_LOCK, 0, 0); + + /* read the current max. curid */ + r = spread(fs->jfd, &curid, sizeof(curid), 0); + if (r != sizeof(curid)) { + rv = 0; + goto exit; + } + + /* increment it and handle overflows */ + rv = curid + 1; + if (rv == 0) + rv = 1; + + /* write to the file descriptor */ + r = spwrite(fs->jfd, &rv, sizeof(rv), 0); + if (r != sizeof(curid)) { + rv = 0; + goto exit; + } + +exit: + plockf(fs->jfd, F_ULOCK, 0, 0); + return rv; +} + +/* frees a transaction id */ +static void free_tid(struct jfs *fs, unsigned int tid) +{ + unsigned int curid, i; + int r; + char name[PATH_MAX]; + + /* lock the whole file */ + plockf(fs->jfd, F_LOCK, 0, 0); + + /* read the current max. curid */ + r = spread(fs->jfd, &curid, sizeof(curid), 0); + if (r != sizeof(curid)) { + goto exit; + } + + if (tid < curid) { + /* we're not freeing the max. curid, so we just return */ + goto exit; + } else { + /* look up the new max. */ + for (i = curid - 1; i > 0; i--) { + /* this can fail if we're low on mem, but we don't + * care checking here because the problem will come + * out later and we can fail more properly */ + get_jtfile(fs->name, i, name); + if (access(name, R_OK | W_OK) == 0) { + curid = i; + break; + } + } + + /* and save it */ + r = spwrite(fs->jfd, &i, sizeof(i), 0); + if (r != sizeof(curid)) { + goto exit; + } + } + +exit: + plockf(fs->jfd, F_ULOCK, 0, 0); + return; +} + + +/* + * transaction functions + */ + +/* initialize a transaction structure */ +void jtrans_init(struct jfs *fs, struct jtrans *ts) +{ + ts->fs = fs; + ts->name = NULL; + ts->id = 0; + ts->flags = 0; + ts->buf = NULL; + ts->len = 0; + ts->offset = 0; + ts->udata = NULL; + ts->ulen = 0; + ts->pdata = NULL; + ts->plen = 0; +} + +/* free the contents of a transaction structure */ +void jtrans_free(struct jtrans *ts) +{ + /* NOTE: we only really free the name and previous data, which are the + * things _we_ allocate; the user data is caller stuff */ + ts->fs = NULL; + if (ts->name) + free(ts->name); + if (ts->pdata) + free(ts->pdata); + + /* don't free ts itself, it's very common to allocate it in the stack, + * so let the caller take care of it; and, after all, he was the one + * doing the alloc in the first place */ +} + +/* commit a transaction */ +int jtrans_commit(struct jtrans *ts) +{ + int id, fd, rv, t; + char *name; + unsigned char *buf_init, *bufp; + + name = (char *) malloc(PATH_MAX); + if (name == NULL) + return -1; + + id = get_tid(ts->fs); + if (id == 0) + return -1; + + /* open the transaction file */ + if (!get_jtfile(ts->fs->name, id, name)) + return -1; + fd = open(name, O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, 0600); + if (fd < 0) + return -1; + + /* and lock it */ + plockf(fd, F_LOCK, 0, 0); + + ts->id = id; + ts->name = name; + + /* lock the file region to work on */ + if (!(ts->fs->flags & J_NOLOCK)) + plockf(ts->fs->fd, F_LOCK, ts->offset, ts->len); + + /* read the current content and fill in the transaction structure */ + ts->pdata = malloc(ts->len); + if (ts->pdata == NULL) + goto exit; + + ts->plen = ts->len; + + rv = spread(ts->fs->fd, ts->pdata, ts->len, ts->offset); + if (rv < 0) + goto exit; + if (rv < ts->len) { + /* we are extending the file! use ftruncate() to do it */ + ftruncate(ts->fs->fd, ts->offset + ts->len); + ts->plen = rv; + } + + /* now save the transaction to the file, static data first */ + + buf_init = malloc(J_DISKTFIXSIZE); + if (buf_init == NULL) + return -1; + + bufp = buf_init; + + memcpy(bufp, (void *) &(ts->id), sizeof(ts->id)); + bufp += 4; + + memcpy(bufp, (void *) &(ts->flags), sizeof(ts->flags)); + bufp += 4; + + memcpy(bufp, (void *) &(ts->len), sizeof(ts->len)); + bufp += 4; + + memcpy(bufp, (void *) &(ts->plen), sizeof(ts->plen)); + bufp += 4; + + memcpy(bufp, (void *) &(ts->ulen), sizeof(ts->ulen)); + bufp += 4; + + memcpy(bufp, (void *) &(ts->offset), sizeof(ts->offset)); + bufp += 8; + + rv = spwrite(fd, buf_init, J_DISKTFIXSIZE, 0); + if (rv != J_DISKTFIXSIZE) + goto exit; + + free(buf_init); + + + /* and now the variable data */ + + if (ts->udata) { + rv = spwrite(fd, ts->udata, ts->ulen, J_DISKTFIXSIZE); + if (rv != ts->ulen) + goto exit; + } + + t = J_DISKTFIXSIZE + ts->ulen; + rv = spwrite(fd, ts->pdata, ts->plen, t); + if (rv != ts->plen) + goto exit; + + /* this is a simple but efficient optimization: instead of doing + * everything O_SYNC, we sync at this point only, this way we avoid + * doing a lot of very small writes; in case of a crash the + * transaction file is only useful if it's complete (ie. after this + * point) so we only flush here */ + fsync(fd); + + /* now that we have a safe transaction file, let's apply it */ + rv = spwrite(ts->fs->fd, ts->buf, ts->len, ts->offset); + if (rv != ts->len) + goto exit; + + /* the transaction has been applied, so we cleanup and remove it from + * the disk */ + free_tid(ts->fs, ts->id); + unlink(name); + + /* mark the transaction as commited, _after_ it was removed */ + ts->flags = ts->flags | J_COMMITED; + + +exit: + close(fd); + + if (!(ts->fs->flags & J_NOLOCK)) + plockf(ts->fs->fd, F_ULOCK, ts->offset, ts->len); + + /* return the lenght only if it was properly commited */ + if (ts->flags & J_COMMITED) + return ts->len; + else + return -1; + +} + +/* rollback a transaction */ +int jtrans_rollback(struct jtrans *ts) +{ + int rv; + struct jtrans newts; + + /* copy the old transaction to the new one */ + jtrans_init(ts->fs, &newts); + + newts.flags = ts->flags; + newts.offset = ts->offset; + + newts.buf = ts->pdata; + newts.len = ts->plen; + + if (ts->plen < ts->len) { + /* we extended the data in the previous transaction, so we + * should truncate it back */ + /* DANGEROUS: this is one of the main reasons why rollbacking + * is dangerous and should only be done with extreme caution: + * if for some reason, after the previous transacton, we have + * extended the file further, this will cut it back to what it + * was; read the docs for more detail */ + ftruncate(ts->fs->fd, ts->offset + ts->plen); + + } + + newts.pdata = ts->pdata; + newts.plen = ts->plen; + + newts.udata = ts->udata; + newts.ulen = ts->ulen; + + rv = jtrans_commit(&newts); + return rv; +} + + +/* + * basic operations + */ + +/* open a file */ +int jopen(struct jfs *fs, const char *name, int flags, int mode, int jflags) +{ + int fd, jfd, rv; + unsigned int t; + char jdir[PATH_MAX], jlockfile[PATH_MAX]; + struct stat sinfo; + + /* we always need read access, because when we commit a transaction we + * read the current contents before applying */ + if (flags & O_WRONLY) { + flags = flags & ~O_WRONLY; + flags = flags | O_RDWR; + } + + fd = open(name, flags, mode); + if (fd < 0) + return -1; + + fs->fd = fd; + fs->name = strdup(name); + fs->flags = jflags; + + /* Note on fs->lock usage: this lock is used only inside the wrappers, + * and exclusively to protect the file pointer. This means that it + * must only be held while performing operations that depend or alter + * the file pointer (jread, jreadv, jwrite, jwritev), but the others + * (jpread, jpwrite) are left unprotected because they can be + * performed in paralell as long as they don't affect the same portion + * of the file (this is protected by lockf). The lock doesn't slow + * things down tho: any threaded app MUST implement this kind of + * locking anyways if it wants to prevent data corruption, we only + * make it easier for them by taking care of it here. If performance + * is essential, the jpread/jpwrite functions should be used, just as + * real life. */ + + pthread_mutex_init( &(fs->lock), NULL); + + if (!get_jdir(name, jdir)) + return -1; + rv = mkdir(jdir, 0750); + rv = lstat(jdir, &sinfo); + if (rv < 0 || !S_ISDIR(sinfo.st_mode)) + return -1; + + snprintf(jlockfile, PATH_MAX, "%s/%s", jdir, "lock"); + jfd = open(jlockfile, O_RDWR | O_CREAT, 0600); + if (jfd < 0) + return -1; + + /* initialize the lock file by writing the first tid to it, but only + * if its empty, otherwise there is a race if two processes call + * jopen() simultaneously and both initialize the file */ + plockf(jfd, F_LOCK, 0, 0); + lstat(jlockfile, &sinfo); + if (sinfo.st_size == 0) { + t = 1; + rv = write(jfd, &t, sizeof(t)); + if (rv != sizeof(t)) { + plockf(jfd, F_ULOCK, 0, 0); + return -1; + } + } + plockf(jfd, F_ULOCK, 0, 0); + + fs->jfd = jfd; + + return fd; +} + +/* close a file */ +int jclose(struct jfs *fs) +{ + if (close(fs->fd)) + return -1; + if (close(fs->jfd)) + return -1; + if (fs->name) + /* allocated by strdup() in jopen() */ + free(fs->name); + return 0; +} + + +/* + * journal recovery + */ + +/* check the journal and replay the incomplete transactions */ +int jfsck(const char *name, struct jfsck_result *res) +{ + int fd, tfd, rv, i, maxtid; + char jdir[PATH_MAX], jlockfile[PATH_MAX], tname[PATH_MAX]; + unsigned char *buf = NULL; + struct stat sinfo; + struct jfs fs; + struct jtrans *curts; + DIR *dir; + struct dirent *dent; + off_t offset; + + fd = open(name, O_RDWR | O_SYNC | O_LARGEFILE); + if (fd < 0) + return J_ENOENT; + + fs.fd = fd; + fs.name = (char *) name; + + if (!get_jdir(name, jdir)) + return J_ENOMEM; + rv = lstat(jdir, &sinfo); + if (rv < 0 || !S_ISDIR(sinfo.st_mode)) + return J_ENOJOURNAL; + + /* open the lock file, which is only used to complete the jfs + * structure */ + snprintf(jlockfile, PATH_MAX, "%s/%s", jdir, "lock"); + rv = open(jlockfile, O_RDWR | O_CREAT, 0600); + if (rv < 0) + return J_ENOJOURNAL; + fs.jfd = rv; + + dir = opendir(jdir); + if (dir == NULL) + return J_ENOJOURNAL; + + /* loop for each file in the journal directory to find out the greater + * transaction number */ + maxtid = 0; + for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) { + /* see if the file is named like a transaction, ignore + * otherwise; as transactions are named as numbers > 0, a + * simple atoi() is enough testing */ + rv = atoi(dent->d_name); + if (rv <= 0) + continue; + if (rv > maxtid) + maxtid = rv; + } + closedir(dir); + + /* rewrite the lockfile, writing the new maxtid on it, so that when we + * rollback a transaction it doesn't step over existing ones */ + rv = spwrite(fs.jfd, &maxtid, sizeof(maxtid), 0); + if (rv != sizeof(maxtid)) { + return J_ENOMEM; + } + + /* we loop all the way up to the max transaction id */ + for (i = 1; i <= maxtid; i++) { + curts = malloc(sizeof(struct jtrans)); + if (curts == NULL) + return J_ENOMEM; + + jtrans_init(&fs, curts); + curts->id = i; + + /* open the transaction file, using i as its name, so we are + * really looping in order (recovering transaction in a + * different order as they were applied means instant + * corruption) */ + if (!get_jtfile(name, i, tname)) + return J_ENOMEM; + tfd = open(tname, O_RDWR | O_SYNC | O_LARGEFILE, 0600); + if (tfd < 0) { + res->invalid++; + goto loop; + } + + /* try to lock the transaction file, if it's locked then it is + * currently being used so we skip it */ + rv = plockf(tfd, F_TLOCK, 0, 0); + if (rv == -1) { + res->in_progress++; + goto loop; + } + + /* load from disk, header first */ + buf = (unsigned char *) malloc(J_DISKTFIXSIZE); + if (buf == NULL) { + res->load_error++; + goto loop; + } + + rv = read(tfd, buf, J_DISKTFIXSIZE); + if (rv != J_DISKTFIXSIZE) { + res->broken_head++; + free(buf); + goto loop; + } + + curts->flags = *( (uint32_t *) (buf + 4)); + curts->len = *( (uint32_t *) (buf + 8)); + curts->plen = *( (uint32_t *) (buf + 12)); + curts->ulen = *( (uint32_t *) (buf + 16)); + curts->offset = *( (uint64_t *) (buf + 20)); + + free(buf); + + /* if we got here, the transaction was not applied, so we + * check if the transaction file is complete (we only need to + * rollback it) or not (so we can't do anything but ignore it) + */ + + lstat(tname, &sinfo); + rv = J_DISKTFIXSIZE + curts->ulen + curts->plen; + if (sinfo.st_size != rv) { + /* the transaction file is incomplete, some of the + * body is missing */ + res->broken_body++; + goto loop; + } + + /* we have a complete transaction file which commit was not + * successful, so we read it to complete the transaction + * structure and rollback it */ + curts->pdata = malloc(curts->plen); + if (curts->pdata == NULL) { + res->load_error++; + goto loop; + } + + curts->udata = malloc(curts->ulen); + if (curts->udata == NULL) { + res->load_error++; + goto loop; + } + + /* user data */ + offset = J_DISKTFIXSIZE; + rv = spread(tfd, curts->udata, curts->ulen, offset); + if (rv != curts->ulen) { + res->load_error++; + goto loop; + } + + /* previous data */ + offset = J_DISKTFIXSIZE + curts->ulen; + rv = spread(tfd, curts->pdata, curts->plen, offset); + if (rv != curts->plen) { + res->load_error++; + goto loop; + } + + /* rollback */ + rv = jtrans_rollback(curts); + if (rv < 0) { + res->apply_error++; + goto loop; + } + res->rollbacked++; + + /* free the data we just allocated */ + if (curts->plen) { + free(curts->pdata); + curts->pdata = NULL; + } + if (curts->ulen) { + free(curts->udata); + curts->udata = NULL; + } + if (curts->name) { + free(curts->name); + curts->name = NULL; + } + +loop: + if (tfd > 0) + close(tfd); + + free(curts); + + res->total++; + } + + close(fs.fd); + close(fs.jfd); + + return 0; + +} + +/* remove all the files in the journal directory (if any) */ +int jfsck_cleanup(const char *name) +{ + char jdir[PATH_MAX], tfile[PATH_MAX*3]; + DIR *dir; + struct dirent *dent; + + if (!get_jdir(name, jdir)) + return 0; + + dir = opendir(jdir); + if (dir == NULL && errno == ENOENT) + /* it doesn't exist, so it's clean */ + return 1; + else if (dir == NULL) + return 0; + + for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) { + /* we only care about transactions (named as numbers > 0) and + * the lockfile (named "lock"); ignore everything else */ + if (strcmp(dent->d_name, "lock") && atoi(dent->d_name) <= 0) + continue; + + /* build the full path to the transaction file */ + memset(tfile, 0, PATH_MAX * 3); + strcat(tfile, jdir); + strcat(tfile, "/"); + strcat(tfile, dent->d_name); + + /* the full filename is too large */ + if (strlen(tfile) > PATH_MAX) + return 0; + + /* and remove it */ + unlink(tfile); + } + closedir(dir); + + return 1; +} + unchanged: --- /dev/null 2004-04-13 23:59:22.000000000 -0300 +++ cur-root/ansi.c 2004-05-30 12:44:45.795160896 -0300 @@ -0,0 +1,206 @@ + +/* + * libjio - A library for Journaled I/O + * Alberto Bertogli (albertogli@telpin.com.ar) + * + * ANSI C API wrappers + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libjio.h" + + +/* + * To avoid completely useless code duplication, this functions rely on the + * UNIX wrappers from unix.c. + * + * The API is not nice, and you I wouldn't recommend it for any serious I/O; + * this wrappers are done more as code samples than anything else. + * + * TODO: it should be possible to implement something like this at some + * LD_PRELOAD level without too much harm for most apps. + * TODO: this is still experimental, it hasn't received too much testing + * (that's why it's not even documented), so use it at your own risk. + */ + + +/* fopen wrapper */ +struct jfs *jfopen(const char *path, const char *mode) +{ + int fd; + int flags; + int pos_at_the_beginning; + struct jfs *fs; + + if (strlen(mode) < 1) + return NULL; + + if (mode[0] == 'r') { + pos_at_the_beginning = 1; + if (strlen(mode) > 1 && strchr(mode, '+')) + flags = O_RDWR; + else + flags = O_RDONLY; + } else if (mode[0] == 'a') { + /* in this case, make no distinction between "a" and "a+" + * because the file is _always_ open for reading anyways */ + pos_at_the_beginning = 0; + flags = O_RDWR | O_CREAT; + } else if (mode[0] == 'w') { + /* the same as before */ + pos_at_the_beginning = 1; + flags = O_RDWR | O_CREAT | O_TRUNC; + } else { + return NULL; + } + + fs = malloc(sizeof(struct jfs)); + + fd = jopen(fs, path, flags, 0666, 0); + if (fd < 0) { + free(fs); + return NULL; + } + + if (pos_at_the_beginning) + lseek(fd, 0, SEEK_SET); + else + lseek(fd, 0, SEEK_END); + + return fs; +} + +/* fclose wrapper */ +int jfclose(struct jfs *stream) +{ + int rv; + rv = jclose(stream); + free(stream); + + if (rv == 0) + return 0; + else + return EOF; +} + +/* freopen wrapper */ +struct jfs *jfreopen(const char *path, const char *mode, struct jfs *stream) +{ + if (stream) + jfclose(stream); + + stream = jfopen(path, mode); + return stream; +} + +/* fread wrapper */ +size_t jfread(void *ptr, size_t size, size_t nmemb, struct jfs *stream) +{ + int rv; + rv = jread(stream, ptr, size * nmemb); + + if (rv <= 0) + return 0; + + return rv / size; +} + +/* fwrite wrapper */ +size_t jfwrite(const void *ptr, size_t size, size_t nmemb, struct jfs *stream) +{ + int rv; + rv = jwrite(stream, ptr, size * nmemb); + + if (rv <= 0) + return 0; + + return rv / size; +} + +/* fileno wrapper */ +int jfileno(struct jfs *stream) +{ + return stream->fd; +} + +/* feof wrapper */ +int jfeof(struct jfs *stream) +{ + /* ANSI expects that when an EOF is reached in any operation (like + * fread() or fwrite()) some internal flag is set, and this function + * can be used to check if it is set or unset. + * As we don't do that (it's pointless for this kind of I/O), this + * just checks if the file pointer is at the end of the file */ + + off_t curpos, endpos; + + pthread_mutex_lock(&(stream->lock)); + + curpos = lseek(jfileno(stream), 0, SEEK_CUR); + endpos = lseek(jfileno(stream), 0, SEEK_END); + + lseek(jfileno(stream), curpos, SEEK_SET); + + pthread_mutex_unlock(&(stream->lock)); + + if (curpos >= endpos) + return 1; + else + return 0; +} + +/* clearerr wrapper */ +void jclearerr(struct jfs *stream) +{ + /* As we do not carry any kind of error state (like explained in + * jfeof()), this function has no effect. */ +} + +/* ferror wrapper */ +int jferror(struct jfs *stream) +{ + /* The same as the above; however not returning this might have some + * side effects on very subtle programs relying on this behaviour */ + return 0; +} + +/* fseek wrapper */ +int jfseek(struct jfs *stream, long offset, int whence) +{ + long pos; + + pthread_mutex_lock(&(stream->lock)); + pos = lseek(stream->fd, offset, whence); + pthread_mutex_unlock(&(stream->lock)); + + return pos; +} + +/* ftell wrapper */ +int jftell(struct jfs *stream) +{ + return lseek(stream->fd, 0, SEEK_CUR); +} + +/* rewind wrapper */ +void frewind(struct jfs *stream) +{ + lseek(stream->fd, 0, SEEK_SET); +} + +/* convert a struct jfs to a FILE so you can use it with other functions that + * require a FILE pointer; be aware that you're bypassing the journaling layer + * and it can cause severe corruption if you're not extremely careful */ +FILE *jfsopen(struct jfs *stream, const char *mode) +{ + return fdopen(stream->fd, mode); +} + unchanged: --- cur/libjio.h~ansi_wrappers 2004-05-29 23:57:44.000000000 -0300 +++ cur-root/libjio.h 2004-05-30 00:00:27.000000000 -0300 @@ -8,6 +8,7 @@ #define _LIBJIO_H #include +#include #include #include #include @@ -68,27 +69,42 @@ struct disk_trans { }; -/* basic operations */ +/* core operations */ int jopen(struct jfs *fs, const char *name, int flags, int mode, int jflags); -ssize_t jread(struct jfs *fs, void *buf, size_t count); -ssize_t jpread(struct jfs *fs, void *buf, size_t count, off_t offset); -ssize_t jreadv(struct jfs *fs, struct iovec *vector, int count); -ssize_t jwrite(struct jfs *fs, const void *buf, size_t count); -ssize_t jpwrite(struct jfs *fs, const void *buf, size_t count, off_t offset); -ssize_t jwritev(struct jfs *fs, const struct iovec *vector, int count); -int jtruncate(struct jfs *fs, off_t lenght); -int jclose(struct jfs *fs); - -/* transaction operations */ void jtrans_init(struct jfs *fs, struct jtrans *ts); int jtrans_commit(struct jtrans *ts); int jtrans_rollback(struct jtrans *ts); void jtrans_free(struct jtrans *ts); +int jclose(struct jfs *fs); /* journal checker */ int jfsck(const char *name, struct jfsck_result *res); int jfsck_cleanup(const char *name); +/* UNIX API wrappers */ +ssize_t jread(struct jfs *fs, void *buf, size_t count); +ssize_t jpread(struct jfs *fs, void *buf, size_t count, off_t offset); +ssize_t jreadv(struct jfs *fs, struct iovec *vector, int count); +ssize_t jwrite(struct jfs *fs, const void *buf, size_t count); +ssize_t jpwrite(struct jfs *fs, const void *buf, size_t count, off_t offset); +ssize_t jwritev(struct jfs *fs, const struct iovec *vector, int count); +int jtruncate(struct jfs *fs, off_t lenght); + +/* ANSI C stdio wrappers */ +struct jfs *jfopen(const char *path, const char *mode); +int jfclose(struct jfs *stream); +struct jfs *jfreopen(const char *path, const char *mode, struct jfs *stream); +size_t jfread(void *ptr, size_t size, size_t nmemb, struct jfs *stream); +size_t jfwrite(const void *ptr, size_t size, size_t nmemb, struct jfs *stream); +int jfileno(struct jfs *stream); +int jfeof(struct jfs *stream); +void jclearerr(struct jfs *stream); +int jferror(struct jfs *stream); +int jfseek(struct jfs *stream, long offset, int whence); +int jftell(struct jfs *stream); +void frewind(struct jfs *stream); +FILE *jfsopen(struct jfs *stream, const char *mode); + /* jfs constants */ #define J_NOLOCK 1 /* don't lock the file before operating on it */ unchanged: --- cur/doc/libjio.html +++ /dev/null 2004-04-13 23:59:22.000000000 -0300 @@ -1,444 +0,0 @@ - - - - - -libjio - A library for journaled I/O - - - - - - - - - - - - - - - -next_inactive -up -previous -
-
-
- - -

- -

- -

-

libjio - A library for journaled I/O

-
- -

Alberto Bertogli (albertogli@telpin.com.ar)

-
-
- -

-Contents -

- - - - - -

- -

-1 Introduction -

- -

-libjio is a library for doing journaled transaction-oriented -I/O, providing atomicity warantees and a simple to use but powerful -API. - -

-This document explains the design of the library, how it works internally -and why it works that way. You should read it even if you don't plan -to do use the library in strange ways, it provides (or at least tries -to =) an insight view on how the library performs its job, which can -be very valuable knowledge when working with it. - -

-To the user, libjio provides two groups of functions, one UNIX-alike -that implements the journaled versions of the classic functions (open(), -read(), write() and friends); and a lower-level one -that center on transactions and allows the user to manipulate them -directly by providing means of commiting and rollbacking. The former, -as expected, are based on the latter and interact safely with them. -Besides, it's designed in a way that allows efficient and safe interaction -with I/O performed from outside the library in case you want to. - -

-The following sections describe different concepts and procedures -that the library bases its work on. It's not intended to be a replace -to reading the source code: please do so if you have any doubts, it's -not big at all (less than 800 lines, including comments) and I hope -it's readable enough. If you think that's not the case, please let -me know and I'll try to give you a hand. - -

- -

-2 General on-disk data organization -

- -

-On the disk, the file you are working on will look exactly as you -expect and hasn't got a single bit different that what you would get -using the regular API. But, besides the working file, you will find -a directory named after it where the journaling information lives. - -

-Inside, there are two kind of files: the lock file and transaction -files. The first one is used as a general lock and holds the next -transaction ID to assign, and there is only one; the second one holds -one transaction, which is composed by a header of fixed size and a -variable-size payload, and can be as many as in-flight transactions. - -

-This impose some restrictions to the kind of operations you can perform -over a file while it's currently being used: you can't move it (because -the journal directory name depends on the filename) and you can't -unlink it (for similar reasons). - -

-This warnings are no different from a normal simultaneous use under -classic UNIX environments, but they are here to remind you that even -tho the library warantees a lot and eases many things from its user -(specially from complex cases, like multiple threads using the file -at the same time), you should still be careful when doing strange -things with files while working on them. - -

- -

-2.1 The transaction file -

- -

-The transaction file is composed of two main parts: the header and -the payload. - -

-The header holds basic information about the transaction itself, including -the ID, some flags, the offset to commit to and the lenght of the -data. The payload holds the data, in three parts: user-defined data, -previous data, and real data. - -

-User-defined data is not used by the library itself, but it's a space -where the user can save private data that can be useful later. Previous -data is saved by the library prior applying the commit, so transactions -can be rollbacked. Real data is just the data to save to the disk, -and it is saved because if a crash occurs when while we are applying -the transaction we can recover gracefuly. - -

- -

-3 The commit procedure -

- -

-We call "commit" to the action of safely -and atomically write some given data to the disk. - -

-The former, safely, means that after a commit has been done -we can assume the data will not get lost and can be retrieved, unless -of course some major event happens (like a hardware failure). For -us, this means that the data was effectively written to the disk and -if a crash occurs after the commit operation has returned, the operation -will be complete and data will be available from the file. - -

-The latter, atomically, warantees that the operation is either -completely done, or not done at all. This is a really common word, -specially if you have worked with multiprocessing, and should be quite -familiar. We implement atomicity by combining fine-grained locks and -journaling, which can assure us both to be able to recover from crashes, -and to have exclusive access to a portion of the file without having -any other transaction overlap it. - -

-Well, so much for talking, now let's get real; libjio applies commits -in a very simple and straightforward way, inside jtrans_commit(): - -

- -

    -
  • Lock the section where the commit takes place -
    • -
  • Open the transaction file -
    • -
  • Write the header -
    • -
  • Write the user data (if any) -
    • -
  • Read the previous data from the file -
    • -
  • Write the previous data in the transaction -
    • -
  • Write the data to the file -
    • -
  • Mark the transaction as commited by setting a flag in the header -
    • -
  • Unlink the transaction file -
    • -
  • Unlock the section where the commit takes place -
    • -
-This may look as a lot of steps, but they're not as much as it looks -like inside the code, and allows a recovery from interruptions in -every step of the way (or even in the middle of a step). - -

- -

-4 The rollback procedure -

- -

-First of all, rollbacking is like ``undo'' a commit: return the -data to the state it had exactly before a given commit was applied. -Due to the way we handle commits, doing this operation becomes quite -simple and straightforward. - -

-In the previous section we said that each transaction held, besides -the data to commit to the disk, the data that was on it before commiting. -That data is saved precisely to be able to rollback. So, to rollback -a transaction all that has to be done is recover that ``previous -data'' from the transaction we want to rollback, and save it to the -disk. In the end, this ends up being a new transaction with the previous -data as the new one, so we do that: create a new transaction structure, -fill in the data from the transaction we want to rollback, and commit -it. All this is performed by jtrans_rollback(). - -

-By doing this we can provide the same warranties a commit has, it's -really fast, eases the recovery, and the code is simple and clean. -What a deal. - -

-But be aware that rollbacking is dangerous. And I really mean it: -you should only do it if you're really sure it's ok. -Consider, for instance, that you commit transaction A, then B, and -then you rollback A. If A and B happen to touch the same portion of -the file, the rollback will, of course, not return the state previous -to B, but previous to A. If it's not done safely, this can lead to -major corruption. Now, if you add to this transactions that extend -the file (and thus rollbacking truncates it back), you not only have -corruption but data loss. So, again, be aware, I can't stress this -enough, rollback only if you really really know what -you are doing. - -

- -

-5 The recovery procedure -

- -

-Recovering from crashes is done by the jfsck() call (or the -program jiofsck which is just a simple invocation to that function), -which opens the file and goes through all transactions in the journal -(remember that transactions are removed from the journal directory -after they're applied), loading and rollbacking them if necessary. -There are several steps where it can fail: there could be no journal, -a given transaction file might be corrupted, incomplete, and so on; -but in the end, there are two cases regarding each transaction: either -it's complete and can be rollbacked, or not. - -

-In the case the transaction is not complete, there is no possibility -that it has been partially applied to the disk, remember that, from -the commit procedure, we only apply the transaction after saving -it in the journal, so there is really nothing left to be done. So -if the transaction is complete, we only need to rollback. - -

-In any case, after making the recovery you can simply remove the journal -entirely and let the library create a new one, and you can be sure -that transaction atomicity was preserved. - -

- -

-6 High-level functions -

- -

-We call high-level functions to the ones provided by the library -that emulate the good old unix file manipulation calls. Most of them -are just wrappers around commits, and implement proper locking when -operating in order to allow simultaneous operations (either across -threads or processes). They are described in detail in the manual -pages, we'll only list them here for completion: - -

- -

    -
  • jopen() -
    • -
  • jread(), jpread(), jreadv() -
    • -
  • jwrite(), jpwrite(), jwritev() -
    • -
  • jtruncate() -
    • -
  • jclose() -
    • -
- -

- -

-7 ACID (or How does libjio fit into theory) -

- -

-I haven't read much theory about this, and the library was implemented -basically by common sense and not theorethical study. - -

-However, I'm aware that database people like ACID (well, that's not -news for anybody ;), which they say mean "Atomicity, Consistency, -Isolation, Durability" (yeah, right!). - -

-So, even libjio is not a purely database thing, it can be used to -achieve those attributes in a simple and efficient way. - -

-Let's take a look one by one: - -

- -

    -
  • Atomicity: In a transaction involving two or more discrete pieces -of information, either all of the pieces are committed or none are. -This has been talked before and we've seen how the library achieves -this point, mostly based on locks and relying on a commit procedure. -
    • -
  • Consistency: A transaction either creates a new and valid state of -data, or, if any failure occurs, returns all data to its state before -the transaction was started. This, like atomicity, has been discussed -before, specially in the recovery section, when we saw how in case -of a crash we end up with a fully applied transaction, or no transaction -applied at all. -
    • -
  • Isolation: A transaction in process and not yet committed must remain -isolated from any other transaction. This comes as a side effect of -doing proper locking on the sections each transaction affect, and -guarantees that there can't be two transactions working on the same -section at the same time. -
    • -
  • Durability: Committed data is saved by the system such that, even -in the event of a failure and system restart, the data is available -in its correct state. For this point we rely on the disk as a method -of permanent storage, and expect that when we do syncronous I/O, data -is safely written and can be recovered after a crash. -
    • -
- -

- -

-8 Working from outside -

- -

-If you want, and are careful enough, you can safely do I/O without -using the library. Here I'll give you some general guidelines that -you need to follow in order to prevent corruption. Of course you can -bend or break them according to your use, this is just a general overview -on how to interact from outside. - -

- -

    -
  • Lock the sections you want to use: the library, as we have already -exposed, relies on fcntl locking; so, if you intend to operate on -parts on the file while using it, you should lock them. -
    • -
  • Don't tuncate, unlink or rename: these operations have serious implications -when they're done while using the library, because the library itself -assumes that names don't change, and files don't dissapear beneath -it. It could potentially lead to corruption, although most of the -time you would just get errors from every call. -
    • -
- -

- -

-About this document ... -

- libjio - A library for journaled I/O

-This document was generated using the -LaTeX2HTML translator Version 2002-2-1 (1.70) -

-Copyright © 1993, 1994, 1995, 1996, -Nikos Drakos, -Computer Based Learning Unit, University of Leeds. -
-Copyright © 1997, 1998, 1999, -Ross Moore, -Mathematics Department, Macquarie University, Sydney. -

-The command line arguments were:
- latex2html -no_subdir -split 0 -show_section_numbers /tmp/lyx_tmpdir2441q5CgGo/lyx_tmpbuf0/libjio.tex -

-The translation was initiated by root on 2004-04-30

- -next_inactive -up -previous -
- -
-root -2004-04-30 -
- - unchanged: --- cur/README~remove_html 2004-05-30 12:47:52.162828704 -0300 +++ cur-root/README 2004-05-30 12:47:52.165828248 -0300 @@ -16,10 +16,10 @@ special directory is created to store in This allows both simple file manipulation, recovery and debugging because everything is isolated. -There's a more detailed document about the library itself in doc/libjio.lyx -(to the 'LyX' editor; there are HTML, Postscript, PDF and TXT versions in the -website, which are not included here for space reasons), and a manpage where -you will find the programming reference. +There's a more detailed document about the library itself in doc/libjio.txt +which is based on doc/libjio.lyx (there are HTML, Postscript, PDF and TXT +versions in the website, which are not included here for space reasons), and a +manpage where you will find the API reference. To see how to install it, please read the INSTALL file. unchanged: --- cur/tests/README~benchmarks_readme 2004-05-30 12:47:59.312741752 -0300 +++ cur-root/tests/README 2004-05-30 12:47:59.314741448 -0300 @@ -5,5 +5,6 @@ not much documentation besides the code. Another really useful way of doing testing is using some of the well known filesystem benchmarking applications and modify them to use libjio. I regulary -use Bonnie and dbench, so if you want the patches please let me know. +use dbench, fsx, tiobench and the tdb test suite, the patches are available on +the website. only in patch2: unchanged: --- cur/Make.conf~version-0.15 2004-05-30 12:48:05.029872616 -0300 +++ cur-root/Make.conf 2004-05-30 12:48:05.031872312 -0300 @@ -1,5 +1,5 @@ -VERSION="0.14" +VERSION="0.15" CC = gcc CFLAGS += -Wall -O6 \