jclose_on_end.patch jclose() before exit in the testing programs. jrewind.patch Fix a typo in ansi.c: the "rewind" wrapper should be named "jrewind" jfseek_return.patch Make jfseek() return the same as fseek(), that is, 0 on success and -1 on error. sizeof_in_arith.patch Replace sizeof() in header's memcpy()s for explicit sizes. new_commit.patch Redo the commit procedure to allow true transactions that affect several portions of the file. full_sample.patch Add a more-or-less complete example of how to use the library. open_always_rw.patch We always need to open the file with both read and write access. new_commit_doc.patch Add and update documentation to the new commit procedure. cur-root/Makefile | 2 cur-root/README | 10 cur-root/ansi.c | 8 cur-root/check.c | 260 +++++++++++++ cur-root/common.c | 54 ++ cur-root/common.h | 2 cur-root/doc/guide.lyx | 626 +++++++++++++++++++++++++++++++++ cur-root/doc/guide.txt | 295 +++++++++++++++ cur-root/doc/libjio.3 | 101 ++--- cur-root/doc/libjio.lyx | 47 +- cur-root/doc/libjio.txt | 82 ++-- cur-root/libjio.h | 64 ++- cur-root/samples/build | 1 cur-root/samples/clean | 2 cur-root/samples/full.c | 55 ++ cur-root/samples/jio3.c | 26 - cur-root/tests/performance/paralell.c | 1 cur-root/tests/performance/streaming.c | 1 cur-root/trans.c | 605 +++++++++++-------------------- cur-root/unix.c | 58 +-- cur/doc/big_transactions | 9 cur/doc/jiofsck | 12 cur/doc/threads | 20 - 23 files changed, 1713 insertions(+), 628 deletions(-) unchanged: --- cur/tests/performance/paralell.c~jclose_on_end 2004-06-02 12:51:44.053300680 -0300 +++ cur-root/tests/performance/paralell.c 2004-06-02 12:52:14.070737336 -0300 @@ -116,6 +116,7 @@ int main(int argc, char **argv) pthread_join(*(threads + i), NULL); } + jclose(&fs); return 0; } unchanged: --- cur/tests/performance/streaming.c~jclose_on_end 2004-06-02 12:51:48.409638416 -0300 +++ cur-root/tests/performance/streaming.c 2004-06-02 12:52:32.282968656 -0300 @@ -76,6 +76,7 @@ int main(int argc, char **argv) printf("%d %d %f %f\n", mb, blocksize, seconds, mb_per_sec); + jclose(&fs); return 0; } unchanged: --- cur-root/ansi.c 2004-06-03 00:27:14.417541648 -0300 +++ cur-root/ansi.c 2004-06-03 01:00:58.526830264 -0300 @@ -181,7 +181,11 @@ pos = lseek(stream->fd, offset, whence); pthread_mutex_unlock(&(stream->lock)); - return pos; + /* fseek returns 0 on success, -1 on error */ + if (pos == -1) + return 1; + + return 0; } /* ftell wrapper */ @@ -191,7 +195,7 @@ } /* rewind wrapper */ -void frewind(struct jfs *stream) +void jrewind(struct jfs *stream) { lseek(stream->fd, 0, SEEK_SET); } unchanged: --- cur-root/trans.c 2004-06-19 23:57:20.814099768 -0300 +++ cur-root/trans.c 2004-06-19 23:57:44.355520928 -0300 @@ -22,53 +22,9 @@ #include "common.h" -/* build the journal directory name out of the filename */ -static int get_jdir(const char *filename, char *jdir) -{ - char *base, *baset; - char *dir, *dirt; - - baset = strdup(filename); - if (baset == NULL) - return 0; - base = basename(baset); - - dirt = strdup(filename); - if (dirt == NULL) - return 0; - dir = dirname(dirt); - - snprintf(jdir, PATH_MAX, "%s/.%s.jio", dir, base); - - free(baset); - free(dirt); - - return 1; -} - -/* build the filename of a given transaction */ -static int get_jtfile(const char *filename, int tid, char *jtfile) -{ - char *base, *baset; - char *dir, *dirt; - - baset = strdup(filename); - if (baset == NULL) - return 0; - base = basename(baset); - - dirt = strdup(filename); - if (dirt == NULL) - return 0; - dir = dirname(dirt); - - snprintf(jtfile, PATH_MAX, "%s/.%s.jio/%d", dir, base, tid); - - free(baset); - free(dirt); - - return 1; -} +/* + * helper functions + */ /* gets a new transaction id */ static unsigned int get_tid(struct jfs *fs) @@ -158,53 +114,108 @@ ts->fs = fs; ts->name = NULL; ts->id = 0; - ts->flags = 0; - ts->buf = NULL; - ts->len = 0; - ts->offset = 0; - ts->udata = NULL; - ts->ulen = 0; - ts->pdata = NULL; - ts->plen = 0; + ts->flags = fs->flags; + ts->op = NULL; + ts->numops = 0; + pthread_mutex_init( &(ts->lock), NULL); } + /* free the contents of a transaction structure */ void jtrans_free(struct jtrans *ts) { - /* NOTE: we only really free the name and previous data, which are the - * things _we_ allocate; the user data is caller stuff */ + struct joper *tmpop; + ts->fs = NULL; + if (ts->name) free(ts->name); - if (ts->pdata) - free(ts->pdata); - /* don't free ts itself, it's very common to allocate it in the stack, - * so let the caller take care of it; and, after all, he was the one - * doing the alloc in the first place */ + while (ts->op != NULL) { + tmpop = ts->op->next; + + if (ts->op->buf) + free(ts->op->buf); + if (ts->op->pdata) + free(ts->op->pdata); + free(ts->op); + + ts->op = tmpop; + } +} + + +int jtrans_add(struct jtrans *ts, const void *buf, size_t count, off_t offset) +{ + struct joper *jop, *tmpop; + + /* find the last operation in the transaction and create a new one at + * the end */ + pthread_mutex_lock(&(ts->lock)); + if (ts->op == NULL) { + ts->op = malloc(sizeof(struct joper)); + jop = ts->op; + jop->prev = NULL; + } else { + for (tmpop = ts->op; tmpop->next != NULL; tmpop = tmpop->next) + ; + tmpop->next = malloc(sizeof(struct joper)); + tmpop->next->prev = tmpop; + jop = tmpop->next; + } + pthread_mutex_unlock(&(ts->lock)); + + if (jop == NULL) { + /* malloc() failed */ + return 0; + } + + jop->buf = malloc(count); + if (jop->buf == NULL) { + free(jop); + return 0; + } + + /* we copy the buffer because then the caller can reuse it */ + memcpy(jop->buf, buf, count); + jop->len = count; + jop->offset = offset; + jop->next = NULL; + jop->plen = 0; + jop->pdata = NULL; + jop->locked = 0; + + ts->numops++; + + return 1; } /* commit a transaction */ int jtrans_commit(struct jtrans *ts) { - int id, fd, rv, t; + int id, rv, fd = -1; char *name; unsigned char *buf_init, *bufp; + struct joper *op; + off_t curpos = 0; + size_t written = 0; + + pthread_mutex_lock(&(ts->lock)); name = (char *) malloc(PATH_MAX); if (name == NULL) - return -1; + goto exit; id = get_tid(ts->fs); if (id == 0) - return -1; + goto exit; /* open the transaction file */ if (!get_jtfile(ts->fs->name, id, name)) - return -1; + goto exit; fd = open(name, O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, 0600); if (fd < 0) - return -1; + goto exit; /* and lock it */ plockf(fd, F_LOCK, 0, 0); @@ -212,71 +223,102 @@ ts->id = id; ts->name = name; - /* lock the file region to work on */ - if (!(ts->fs->flags & J_NOLOCK)) - plockf(ts->fs->fd, F_LOCK, ts->offset, ts->len); - - /* read the current content and fill in the transaction structure */ - ts->pdata = malloc(ts->len); - if (ts->pdata == NULL) - goto exit; - - ts->plen = ts->len; - - rv = spread(ts->fs->fd, ts->pdata, ts->len, ts->offset); - if (rv < 0) - goto exit; - if (rv < ts->len) { - /* we are extending the file! use ftruncate() to do it */ - ftruncate(ts->fs->fd, ts->offset + ts->len); - ts->plen = rv; - } - - /* now save the transaction to the file, static data first */ - - buf_init = malloc(J_DISKTFIXSIZE); + /* save the header */ + buf_init = malloc(J_DISKHEADSIZE); if (buf_init == NULL) - return -1; + goto exit; bufp = buf_init; - memcpy(bufp, (void *) &(ts->id), sizeof(ts->id)); + memcpy(bufp, (void *) &(ts->id), 4); bufp += 4; - memcpy(bufp, (void *) &(ts->flags), sizeof(ts->flags)); + memcpy(bufp, (void *) &(ts->flags), 4); bufp += 4; - memcpy(bufp, (void *) &(ts->len), sizeof(ts->len)); + memcpy(bufp, (void *) &(ts->numops), 4); bufp += 4; - memcpy(bufp, (void *) &(ts->plen), sizeof(ts->plen)); - bufp += 4; + rv = spwrite(fd, buf_init, J_DISKHEADSIZE, 0); + if (rv != J_DISKHEADSIZE) { + free(buf_init); + goto exit; + } - memcpy(bufp, (void *) &(ts->ulen), sizeof(ts->ulen)); - bufp += 4; + free(buf_init); - memcpy(bufp, (void *) &(ts->offset), sizeof(ts->offset)); - bufp += 8; + curpos = J_DISKHEADSIZE; - rv = spwrite(fd, buf_init, J_DISKTFIXSIZE, 0); - if (rv != J_DISKTFIXSIZE) - goto exit; + /* first of all lock all the regions we're going to work with; + * otherwise there could be another transaction trying to write the + * same spots and we could end up with interleaved writes, that could + * break atomicity warantees if we need to rollback */ + if (!(ts->flags & J_NOLOCK)) { + for (op = ts->op; op != NULL; op = op->next) { + rv = plockf(ts->fs->fd, F_LOCK, op->offset, op->len); + if (rv == -1) + /* note it can fail with EDEADLK */ + goto exit; + op->locked = 1; + } + } - free(buf_init); + /* save each transacion in the file */ + for (op = ts->op; op != NULL; op = op->next) { + /* read the current content only if it's not there yet, which + * is the normal case, but for rollbacking we fill it + * ourselves */ + if (op->pdata == NULL) { + op->pdata = malloc(op->len); + if (op->pdata == NULL) + goto exit; + + op->plen = op->len; + + rv = spread(ts->fs->fd, op->pdata, op->len, + op->offset); + if (rv < 0) + goto exit; + if (rv < op->len) { + /* we are extending the file! */ + /* ftruncate(ts->fs->fd, op->offset + op->len); */ + op->plen = rv; + } + } + + /* save the operation's header */ + buf_init = malloc(J_DISKOPHEADSIZE); + if (buf_init == NULL) + goto exit; + + bufp = buf_init; + + memcpy(bufp, (void *) &(op->len), 4); + bufp += 4; + memcpy(bufp, (void *) &(op->plen), 4); + bufp += 4; - /* and now the variable data */ + memcpy(bufp, (void *) &(op->offset), 8); + bufp += 8; - if (ts->udata) { - rv = spwrite(fd, ts->udata, ts->ulen, J_DISKTFIXSIZE); - if (rv != ts->ulen) + rv = spwrite(fd, buf_init, J_DISKOPHEADSIZE, curpos); + if (rv != J_DISKOPHEADSIZE) { + free(buf_init); goto exit; - } + } - t = J_DISKTFIXSIZE + ts->ulen; - rv = spwrite(fd, ts->pdata, ts->plen, t); - if (rv != ts->plen) - goto exit; + free(buf_init); + + curpos += J_DISKOPHEADSIZE; + + /* and save it to the disk */ + rv = spwrite(fd, op->pdata, op->plen, curpos); + if (rv != op->plen) + goto exit; + + curpos += op->plen; + } /* this is a simple but efficient optimization: instead of doing * everything O_SYNC, we sync at this point only, this way we avoid @@ -286,9 +328,18 @@ fsync(fd); /* now that we have a safe transaction file, let's apply it */ - rv = spwrite(ts->fs->fd, ts->buf, ts->len, ts->offset); - if (rv != ts->len) - goto exit; + written = 0; + for (op = ts->op; op != NULL; op = op->next) { + rv = spwrite(ts->fs->fd, op->buf, op->len, op->offset); + + plockf(ts->fs->fd, F_ULOCK, op->offset, op->len); + op->locked = 0; + + if (rv != op->len) + goto exit; + + written += rv; + } /* the transaction has been applied, so we cleanup and remove it from * the disk */ @@ -301,13 +352,16 @@ exit: close(fd); + for (op = ts->op; op != NULL; op = op->next) { + if (op->locked) + plockf(ts->fs->fd, F_ULOCK, op->offset, op->len); + } - if (!(ts->fs->flags & J_NOLOCK)) - plockf(ts->fs->fd, F_ULOCK, ts->offset, ts->len); + pthread_mutex_unlock(&(ts->lock)); - /* return the lenght only if it was properly commited */ + /* return the length only if it was properly commited */ if (ts->flags & J_COMMITED) - return ts->len; + return written; else return -1; @@ -316,41 +370,61 @@ /* rollback a transaction */ int jtrans_rollback(struct jtrans *ts) { - int rv; struct jtrans newts; + struct joper *op, *curop, *lop; - /* copy the old transaction to the new one */ - jtrans_init(ts->fs, &newts); + /* FIXME: this looks like a mess! */ - newts.flags = ts->flags; - newts.offset = ts->offset; + if (ts->op == NULL) { + /* we're trying to rollback an empty transaction */ + return 0; + } - newts.buf = ts->pdata; - newts.len = ts->plen; + jtrans_init(ts->fs, &newts); + newts.flags = ts->flags; - if (ts->plen < ts->len) { - /* we extended the data in the previous transaction, so we + /* find the last operation */ + for (op = ts->op; op->next != NULL; op = op->next) + ; + + /* and traverse the list backwards */ + for ( ; op != NULL; op = op->prev) { + /* if we extended the data in the previous transaction, we * should truncate it back */ /* DANGEROUS: this is one of the main reasons why rollbacking * is dangerous and should only be done with extreme caution: * if for some reason, after the previous transacton, we have * extended the file further, this will cut it back to what it * was; read the docs for more detail */ - ftruncate(ts->fs->fd, ts->offset + ts->plen); + if (op->plen < op->len) + ftruncate(ts->fs->fd, op->offset + op->plen); + /* manually add the operation to the new transaction */ + curop = malloc(sizeof(struct joper)); + curop->offset = op->offset; + curop->len = op->plen; + curop->buf = op->pdata; + curop->plen = op->plen; + curop->pdata = op->pdata; + curop->locked = 0; + + /* add the new transaction to the list */ + if (newts.op == NULL) { + newts.op = curop; + curop->prev = NULL; + curop->next = NULL; + } else { + for (lop = newts.op; lop->next != NULL; lop = lop->next) + ; + lop->next = curop; + curop->prev = lop; + curop->next = NULL; + } } - newts.pdata = ts->pdata; - newts.plen = ts->plen; - - newts.udata = ts->udata; - newts.ulen = ts->ulen; - - rv = jtrans_commit(&newts); - return rv; + return jtrans_commit(&newts); } - /* * basic operations */ @@ -363,12 +437,12 @@ char jdir[PATH_MAX], jlockfile[PATH_MAX]; struct stat sinfo; - /* we always need read access, because when we commit a transaction we - * read the current contents before applying */ - if (flags & O_WRONLY) { - flags = flags & ~O_WRONLY; - flags = flags | O_RDWR; - } + /* we always need read and write access, because when we commit a + * transaction we read the current contents before applying, and write + * access is needed for locking with fcntl */ + flags = flags & ~O_WRONLY; + flags = flags & ~O_RDONLY; + flags = flags | O_RDWR; fd = open(name, flags, mode); if (fd < 0) @@ -393,244 +467,3 @@ - pthread_mutex_init( &(fs->lock), NULL); if (!get_jdir(name, jdir)) - -/* - * journal recovery - */ - -/* check the journal and replay the incomplete transactions */ -int jfsck(const char *name, struct jfsck_result *res) -{ - int fd, tfd, rv, i, maxtid; - char jdir[PATH_MAX], jlockfile[PATH_MAX], tname[PATH_MAX]; - unsigned char *buf = NULL; - struct stat sinfo; - struct jfs fs; - struct jtrans *curts; - DIR *dir; - struct dirent *dent; - off_t offset; - - fd = open(name, O_RDWR | O_SYNC | O_LARGEFILE); - if (fd < 0) - return J_ENOENT; - - fs.fd = fd; - fs.name = (char *) name; - - if (!get_jdir(name, jdir)) - return J_ENOMEM; - rv = lstat(jdir, &sinfo); - if (rv < 0 || !S_ISDIR(sinfo.st_mode)) - return J_ENOJOURNAL; - - /* open the lock file, which is only used to complete the jfs - * structure */ - snprintf(jlockfile, PATH_MAX, "%s/%s", jdir, "lock"); - rv = open(jlockfile, O_RDWR | O_CREAT, 0600); - if (rv < 0) - return J_ENOJOURNAL; - fs.jfd = rv; - - dir = opendir(jdir); - if (dir == NULL) - return J_ENOJOURNAL; - - /* loop for each file in the journal directory to find out the greater - * transaction number */ - maxtid = 0; - for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) { - /* see if the file is named like a transaction, ignore - * otherwise; as transactions are named as numbers > 0, a - * simple atoi() is enough testing */ - rv = atoi(dent->d_name); - if (rv <= 0) - continue; - if (rv > maxtid) - maxtid = rv; - } - closedir(dir); - - /* rewrite the lockfile, writing the new maxtid on it, so that when we - * rollback a transaction it doesn't step over existing ones */ - rv = spwrite(fs.jfd, &maxtid, sizeof(maxtid), 0); - if (rv != sizeof(maxtid)) { - return J_ENOMEM; - } - - /* we loop all the way up to the max transaction id */ - for (i = 1; i <= maxtid; i++) { - curts = malloc(sizeof(struct jtrans)); - if (curts == NULL) - return J_ENOMEM; - - jtrans_init(&fs, curts); - curts->id = i; - - /* open the transaction file, using i as its name, so we are - * really looping in order (recovering transaction in a - * different order as they were applied means instant - * corruption) */ - if (!get_jtfile(name, i, tname)) - return J_ENOMEM; - tfd = open(tname, O_RDWR | O_SYNC | O_LARGEFILE, 0600); - if (tfd < 0) { - res->invalid++; - goto loop; - } - - /* try to lock the transaction file, if it's locked then it is - * currently being used so we skip it */ - rv = plockf(tfd, F_TLOCK, 0, 0); - if (rv == -1) { - res->in_progress++; - goto loop; - } - - /* load from disk, header first */ - buf = (unsigned char *) malloc(J_DISKTFIXSIZE); - if (buf == NULL) { - res->load_error++; - goto loop; - } - - rv = read(tfd, buf, J_DISKTFIXSIZE); - if (rv != J_DISKTFIXSIZE) { - res->broken_head++; - free(buf); - goto loop; - } - - curts->flags = *( (uint32_t *) (buf + 4)); - curts->len = *( (uint32_t *) (buf + 8)); - curts->plen = *( (uint32_t *) (buf + 12)); - curts->ulen = *( (uint32_t *) (buf + 16)); - curts->offset = *( (uint64_t *) (buf + 20)); - - free(buf); - - /* if we got here, the transaction was not applied, so we - * check if the transaction file is complete (we only need to - * rollback it) or not (so we can't do anything but ignore it) - */ - - lstat(tname, &sinfo); - rv = J_DISKTFIXSIZE + curts->ulen + curts->plen; - if (sinfo.st_size != rv) { - /* the transaction file is incomplete, some of the - * body is missing */ - res->broken_body++; - goto loop; - } - - /* we have a complete transaction file which commit was not - * successful, so we read it to complete the transaction - * structure and rollback it */ - curts->pdata = malloc(curts->plen); - if (curts->pdata == NULL) { - res->load_error++; - goto loop; - } - - curts->udata = malloc(curts->ulen); - if (curts->udata == NULL) { - res->load_error++; - goto loop; - } - - /* user data */ - offset = J_DISKTFIXSIZE; - rv = spread(tfd, curts->udata, curts->ulen, offset); - if (rv != curts->ulen) { - res->load_error++; - goto loop; - } - - /* previous data */ - offset = J_DISKTFIXSIZE + curts->ulen; - rv = spread(tfd, curts->pdata, curts->plen, offset); - if (rv != curts->plen) { - res->load_error++; - goto loop; - } - - /* rollback */ - rv = jtrans_rollback(curts); - if (rv < 0) { - res->apply_error++; - goto loop; - } - res->rollbacked++; - - /* free the data we just allocated */ - if (curts->plen) { - free(curts->pdata); - curts->pdata = NULL; - } - if (curts->ulen) { - free(curts->udata); - curts->udata = NULL; - } - if (curts->name) { - free(curts->name); - curts->name = NULL; - } - -loop: - if (tfd > 0) - close(tfd); - - free(curts); - - res->total++; - } - - close(fs.fd); - close(fs.jfd); - - return 0; - -} - -/* remove all the files in the journal directory (if any) */ -int jfsck_cleanup(const char *name) -{ - char jdir[PATH_MAX], tfile[PATH_MAX*3]; - DIR *dir; - struct dirent *dent; - - if (!get_jdir(name, jdir)) - return 0; - - dir = opendir(jdir); - if (dir == NULL && errno == ENOENT) - /* it doesn't exist, so it's clean */ - return 1; - else if (dir == NULL) - return 0; - - for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) { - /* we only care about transactions (named as numbers > 0) and - * the lockfile (named "lock"); ignore everything else */ - if (strcmp(dent->d_name, "lock") && atoi(dent->d_name) <= 0) - continue; - - /* build the full path to the transaction file */ - memset(tfile, 0, PATH_MAX * 3); - strcat(tfile, jdir); - strcat(tfile, "/"); - strcat(tfile, dent->d_name); - - /* the full filename is too large */ - if (strlen(tfile) > PATH_MAX) - return 0; - - /* and remove it */ - unlink(tfile); - } - closedir(dir); - - return 1; -} - unchanged: --- cur/libjio.h~new_commit 2004-06-14 12:10:21.000000000 -0300 +++ cur-root/libjio.h 2004-06-16 03:22:20.000000000 -0300 @@ -19,64 +19,77 @@ extern "C" { /* logical structures */ + +/* the main file structure */ struct jfs { int fd; /* main file descriptor */ char *name; /* and its name */ int jfd; /* journal's lock file descriptor */ - int flags; /* journal mode options used in jopen() */ + int flags; /* journal flags */ pthread_mutex_t lock; /* a soft lock used in some operations */ }; +/* a single operation */ +struct joper { + int locked; /* is the region is locked? */ + off_t offset; /* operation's offset */ + size_t len; /* data length */ + void *buf; /* data */ + size_t plen; /* previous data length */ + void *pdata; /* previous data */ + struct joper *prev; + struct joper *next; +}; + +/* a transaction */ struct jtrans { struct jfs *fs; /* journal file structure to operate on */ char *name; /* name of the transaction file */ int id; /* transaction id */ int flags; /* misc flags */ - const void *buf; /* buffer */ - size_t len; /* buffer lenght */ - off_t offset; /* file offset to operate on */ - void *udata; /* user-supplied data */ - size_t ulen; /* udata lenght */ - void *pdata; /* previous data, for rollback */ - size_t plen; /* pdata lenght */ + unsigned int numops; /* quantity of operations in the list */ + pthread_mutex_t lock; /* used to modify the operation list */ + struct joper *op; /* list of operations */ }; struct jfsck_result { int total; /* total transactions files we looked at */ int invalid; /* invalid files in the journal directory */ int in_progress; /* transactions in progress */ - int broken_head; /* transactions broken (header missing) */ - int broken_body; /* transactions broken (body missing) */ - int load_error; /* errors loading the transaction */ + int broken; /* transactions broken */ int apply_error; /* errors applying the transaction */ int rollbacked; /* transactions that were rollbacked */ }; -/* on-disk structure */ -struct disk_trans { - - /* header (fixed lenght, defined below) */ + +/* on-disk structures */ + +/* header (fixed length, defined below) */ +struct disk_header { uint32_t id; /* id */ uint32_t flags; /* flags about this transaction */ - uint32_t len; /* data lenght */ - uint32_t plen; /* previous data lenght */ - uint32_t ulen; /* user-supplied information lenght */ + uint32_t numops; /* number of operations */ +}; + +/* operation */ +struct disk_operation { + uint32_t len; /* data length */ + uint32_t plen; /* previous data length */ uint64_t offset; /* offset relative to the BOF */ - - /* payload (variable lenght) */ - char *udata; /* user-supplied data */ char *prevdata; /* previous data for rollback */ }; -/* core operations */ +/* core functions */ int jopen(struct jfs *fs, const char *name, int flags, int mode, int jflags); void jtrans_init(struct jfs *fs, struct jtrans *ts); +int jtrans_add(struct jtrans *ts, const void *buf, size_t count, off_t offset); int jtrans_commit(struct jtrans *ts); int jtrans_rollback(struct jtrans *ts); void jtrans_free(struct jtrans *ts); int jclose(struct jfs *fs); + /* journal checker */ int jfsck(const char *name, struct jfsck_result *res); int jfsck_cleanup(const char *name); @@ -88,7 +101,7 @@ ssize_t jreadv(struct jfs *fs, struct io ssize_t jwrite(struct jfs *fs, const void *buf, size_t count); ssize_t jpwrite(struct jfs *fs, const void *buf, size_t count, off_t offset); ssize_t jwritev(struct jfs *fs, const struct iovec *vector, int count); -int jtruncate(struct jfs *fs, off_t lenght); +int jtruncate(struct jfs *fs, off_t length); /* ANSI C stdio wrappers */ struct jfs *jfopen(const char *path, const char *mode); @@ -113,8 +126,9 @@ FILE *jfsopen(struct jfs *stream, const #define J_COMMITED 1 /* mark a transaction as commited */ #define J_ROLLBACKED 2 /* mark a transaction as rollbacked */ -/* disk_trans constants */ -#define J_DISKTFIXSIZE 28 /* lenght of disk_trans' header */ +/* disk constants */ +#define J_DISKHEADSIZE 12 /* length of disk_header */ +#define J_DISKOPHEADSIZE 16 /* length of disk_operation header */ /* jfsck constants (return values) */ #define J_ESUCCESS 0 /* success - shouldn't be used */ unchanged: --- /dev/null 2004-04-13 23:59:22.000000000 -0300 +++ cur-root/check.c 2004-06-15 03:03:38.000000000 -0300 @@ -0,0 +1,260 @@ + +/* + * libjio - A library for Journaled I/O + * Alberto Bertogli (albertogli@telpin.com.ar) + * + * Recovery functions + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libjio.h" +#include "common.h" + + +/* fill a transaction structure from a mmapped transaction file */ +static int fill_trans(unsigned char *map, off_t len, struct jtrans *ts) +{ + int i; + unsigned char *p; + struct joper *op, *tmp; + + if (len < J_DISKHEADSIZE) + return 0; + + p = map; + + ts->id = *( (uint32_t *) p); + p += 4; + + ts->flags = *( (uint32_t *) p); + p += 4; + + ts->numops = *( (uint32_t *) p); + p += 4; + + for (i = 0; i < ts->numops; i++) { + if (len < (p - map) + J_DISKOPHEADSIZE) + goto error; + + op = malloc(sizeof(struct joper)); + if (op == NULL) + goto error; + + op->len = *( (uint32_t *) p); + p += 4; + + op->plen = *( (uint32_t *) p); + p += 4; + + op->offset = *( (uint64_t *) p); + p += 8; + + if (len < (p - map) + op->plen) + goto error; + + op->pdata = (void *) p; + p += op->plen; + + if (ts->op == NULL) { + ts->op = op; + op->prev = NULL; + op->next = NULL; + } else { + for(tmp = ts->op; tmp->next != NULL; tmp = tmp->next) + ; + tmp->next = op; + op->prev = tmp; + op->next = NULL; + } + } + + return 1; + +error: + while (ts->op != NULL) { + tmp = ts->op->next; + free(ts->op); + ts->op = tmp; + } + return 0; +} + +/* check the journal and rollback incomplete transactions */ +int jfsck(const char *name, struct jfsck_result *res) +{ + int fd, tfd, rv, i, maxtid; + char jdir[PATH_MAX], jlockfile[PATH_MAX], tname[PATH_MAX]; + struct stat sinfo; + struct jfs fs; + struct jtrans *curts; + DIR *dir; + struct dirent *dent; + void *map; + off_t filelen; + + + fd = open(name, O_RDWR | O_SYNC | O_LARGEFILE); + if (fd < 0) + return J_ENOENT; + + fs.fd = fd; + fs.name = (char *) name; + + if (!get_jdir(name, jdir)) + return J_ENOMEM; + rv = lstat(jdir, &sinfo); + if (rv < 0 || !S_ISDIR(sinfo.st_mode)) + return J_ENOJOURNAL; + + /* open the lock file, which is only used to complete the jfs + * structure */ + snprintf(jlockfile, PATH_MAX, "%s/%s", jdir, "lock"); + rv = open(jlockfile, O_RDWR | O_CREAT, 0600); + if (rv < 0) + return J_ENOJOURNAL; + fs.jfd = rv; + + dir = opendir(jdir); + if (dir == NULL) + return J_ENOJOURNAL; + + /* loop for each file in the journal directory to find out the greater + * transaction number */ + maxtid = 0; + for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) { + /* see if the file is named like a transaction, ignore + * otherwise; as transactions are named as numbers > 0, a + * simple atoi() is enough testing */ + rv = atoi(dent->d_name); + if (rv <= 0) + continue; + if (rv > maxtid) + maxtid = rv; + } + closedir(dir); + + /* rewrite the lockfile, writing the new maxtid on it, so that when we + * rollback a transaction it doesn't step over existing ones */ + rv = spwrite(fs.jfd, &maxtid, sizeof(maxtid), 0); + if (rv != sizeof(maxtid)) { + return J_ENOMEM; + } + + /* we loop all the way up to the max transaction id */ + for (i = 1; i <= maxtid; i++) { + curts = malloc(sizeof(struct jtrans)); + if (curts == NULL) + return J_ENOMEM; + + jtrans_init(&fs, curts); + curts->id = i; + + /* open the transaction file, using i as its name, so we are + * really looping in order (recovering transaction in a + * different order as they were applied means instant + * corruption) */ + if (!get_jtfile(name, i, tname)) + return J_ENOMEM; + tfd = open(tname, O_RDWR | O_SYNC | O_LARGEFILE, 0600); + if (tfd < 0) { + res->invalid++; + goto loop; + } + + /* try to lock the transaction file, if it's locked then it is + * currently being used so we skip it */ + rv = plockf(tfd, F_TLOCK, 0, 0); + if (rv == -1) { + res->in_progress++; + goto loop; + } + + filelen = lseek(tfd, 0, SEEK_END); + map = mmap(0, filelen, PROT_READ, MAP_SHARED, tfd, 0); + rv = fill_trans((unsigned char *) map, filelen, curts); + if (rv != 1) { + res->broken++; + goto loop; + } + + rv = jtrans_rollback(curts); + + munmap(map, filelen); + + if (rv < 0) { + res->apply_error++; + goto loop; + } + res->rollbacked++; + + +loop: + if (tfd >= 0) { + close(tfd); + tfd = -1; + } + + free(curts); + + res->total++; + } + + close(fs.fd); + close(fs.jfd); + + return 0; + +} + +/* remove all the files in the journal directory (if any) */ +int jfsck_cleanup(const char *name) +{ + char jdir[PATH_MAX], tfile[PATH_MAX*3]; + DIR *dir; + struct dirent *dent; + + if (!get_jdir(name, jdir)) + return 0; + + dir = opendir(jdir); + if (dir == NULL && errno == ENOENT) + /* it doesn't exist, so it's clean */ + return 1; + else if (dir == NULL) + return 0; + + for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) { + /* we only care about transactions (named as numbers > 0) and + * the lockfile (named "lock"); ignore everything else */ + if (strcmp(dent->d_name, "lock") && atoi(dent->d_name) <= 0) + continue; + + /* build the full path to the transaction file */ + memset(tfile, 0, PATH_MAX * 3); + strcat(tfile, jdir); + strcat(tfile, "/"); + strcat(tfile, dent->d_name); + + /* the full filename is too large */ + if (strlen(tfile) > PATH_MAX) + return 0; + + /* and remove it */ + unlink(tfile); + } + closedir(dir); + + return 1; +} + unchanged: --- cur/common.c~new_commit 2004-06-14 16:40:42.000000000 -0300 +++ cur-root/common.c 2004-06-14 17:41:24.000000000 -0300 @@ -9,6 +9,11 @@ #include #include #include +#include +#include +#include +#include +#include #include "common.h" @@ -90,3 +95,52 @@ ssize_t spwrite(int fd, const void *buf, return count; } +/* build the journal directory name out of the filename */ +int get_jdir(const char *filename, char *jdir) +{ + char *base, *baset; + char *dir, *dirt; + + baset = strdup(filename); + if (baset == NULL) + return 0; + base = basename(baset); + + dirt = strdup(filename); + if (dirt == NULL) + return 0; + dir = dirname(dirt); + + snprintf(jdir, PATH_MAX, "%s/.%s.jio", dir, base); + + free(baset); + free(dirt); + + return 1; +} + +/* build the filename of a given transaction */ +int get_jtfile(const char *filename, int tid, char *jtfile) +{ + char *base, *baset; + char *dir, *dirt; + + baset = strdup(filename); + if (baset == NULL) + return 0; + base = basename(baset); + + dirt = strdup(filename); + if (dirt == NULL) + return 0; + dir = dirname(dirt); + + snprintf(jtfile, PATH_MAX, "%s/.%s.jio/%d", dir, base, tid); + + free(baset); + free(dirt); + + return 1; +} + + unchanged: --- cur/Makefile~new_commit 2004-06-14 16:42:47.000000000 -0300 +++ cur-root/Makefile 2004-06-14 16:43:00.000000000 -0300 @@ -3,7 +3,7 @@ include Make.conf # objects to build -OBJS = common.o trans.o unix.o ansi.o +OBJS = common.o trans.o check.o unix.o ansi.o # rules default: all unchanged: --- cur/common.h~new_commit 2004-06-14 16:48:07.000000000 -0300 +++ cur-root/common.h 2004-06-14 16:48:11.000000000 -0300 @@ -12,6 +12,8 @@ off_t plockf(int fd, int cmd, off_t offset, off_t len); ssize_t spread(int fd, void *buf, size_t count, off_t offset); ssize_t spwrite(int fd, const void *buf, size_t count, off_t offset); +int get_jdir(const char *filename, char *jdir); +int get_jtfile(const char *filename, int tid, char *jtfile); #endif unchanged: --- cur/unix.c~new_commit 2004-06-14 17:49:33.000000000 -0300 +++ cur-root/unix.c 2004-06-16 03:22:44.000000000 -0300 @@ -88,10 +88,7 @@ ssize_t jwrite(struct jfs *fs, const voi jtrans_init(fs, &ts); pos = lseek(fs->fd, 0, SEEK_CUR); - ts.offset = pos; - - ts.buf = buf; - ts.len = count; + jtrans_add(&ts, buf, count, pos); rv = jtrans_commit(&ts); @@ -114,10 +111,7 @@ ssize_t jpwrite(struct jfs *fs, const vo struct jtrans ts; jtrans_init(fs, &ts); - ts.offset = offset; - - ts.buf = buf; - ts.len = count; + jtrans_add(&ts, buf, count, offset); rv = jtrans_commit(&ts); @@ -129,43 +123,29 @@ ssize_t jpwrite(struct jfs *fs, const vo /* writev wrapper */ ssize_t jwritev(struct jfs *fs, const struct iovec *vector, int count) { - int rv, i, bufp; - ssize_t sum; - char *buf; - off_t pos; + int rv, i; + size_t sum; + off_t ipos, t; struct jtrans ts; - sum = 0; - for (i = 0; i < count; i++) - sum += vector[i].iov_len; - - /* unify the buffers into one big chunk to commit */ - /* FIXME: can't we do this more efficient? It ruins the whole purpose - * of using writev()! maybe we should do one transaction per vector */ - buf = malloc(sum); - if (buf == NULL) - return -1; - bufp = 0; - - for (i = 0; i < count; i++) { - memcpy(buf + bufp, vector[i].iov_base, vector[i].iov_len); - bufp += vector[i].iov_len; - } - pthread_mutex_lock(&(fs->lock)); jtrans_init(fs, &ts); - pos = lseek(fs->fd, 0, SEEK_CUR); - ts.offset = pos; + ipos = lseek(fs->fd, 0, SEEK_CUR); + t = ipos; - ts.buf = buf; - ts.len = sum; + sum = 0; + for (i = 0; i < count; i++) { + jtrans_add(&ts, vector[i].iov_base, vector[i].iov_len, t); + sum += vector[i].iov_len; + t += vector[i].iov_len; + } rv = jtrans_commit(&ts); if (rv >= 0) { /* if success, advance the file pointer */ - lseek(fs->fd, count, SEEK_CUR); + lseek(fs->fd, sum, SEEK_CUR); } pthread_mutex_unlock(&(fs->lock)); @@ -177,14 +157,14 @@ ssize_t jwritev(struct jfs *fs, const st } /* truncate a file - be careful with this */ -int jtruncate(struct jfs *fs, off_t lenght) +int jtruncate(struct jfs *fs, off_t length) { int rv; - /* lock from lenght to the end of file */ - plockf(fs->fd, F_LOCK, lenght, 0); - rv = ftruncate(fs->fd, lenght); - plockf(fs->fd, F_ULOCK, lenght, 0); + /* lock from length to the end of file */ + plockf(fs->fd, F_LOCK, length, 0); + rv = ftruncate(fs->fd, length); + plockf(fs->fd, F_ULOCK, length, 0); return rv; } unchanged: --- cur/samples/jio3.c~new_commit 2004-06-14 17:54:09.000000000 -0300 +++ cur-root/samples/jio3.c 2004-06-15 02:28:22.000000000 -0300 @@ -16,28 +16,34 @@ int main(int argc, char **argv) struct jfs fs; struct jtrans ts; - fd = jopen(&fs, "test1", O_RDWR | O_CREAT | O_TRUNC | O_SYNC, 0660, 0); + fd = jopen(&fs, "test1", O_RDWR | O_CREAT | O_SYNC, 0660, 0); if (fd < 0) perror("OPEN"); -#define str "ROLLBACKTEST!\n" - jtrans_init(&fs, &ts); - ts.offset = 0; - ts.buf = str; - ts.len = strlen(str); - +#define str1 "1ROLLBACKTEST1!\n" + jtrans_add(&ts, str1, strlen(str1), 0); + +#define str2 "2ROLLBACKTEST2!\n" + jtrans_add(&ts, str2, strlen(str2), strlen(str1)); + +#define str3 "3ROLLBACKTEST3!\n" + jtrans_add(&ts, str3, strlen(str3), strlen(str1) + strlen(str2)); + + rv = jtrans_commit(&ts); - if (rv != strlen(str)) + if (rv != strlen(str1) + strlen(str2) + strlen(str3)) perror("COMMIT"); + printf("COMMIT OK: %d\n", rv); + rv = jtrans_rollback(&ts); - if (rv != 0) + if (rv < 0) perror("ROLLBACK"); + printf("ROLLBACK OK: %d\n", rv); return 0; } - unchanged: --- /dev/null 2004-04-13 23:59:22.000000000 -0300 +++ cur-root/samples/full.c 2004-06-19 23:57:39.906197328 -0300 @@ -0,0 +1,55 @@ + +#include +#include +#include +#include +#include +#include + +#define FILENAME "test1" +#define TEXT "Hello world!" + +int main(void) +{ + int r; + struct jfs file; + struct jtrans trans; + struct jfsck_result result; + + /* check the file is OK */ + jfsck(FILENAME, &result); + jfsck_cleanup(FILENAME); + + /* and open it */ + r = jopen(&file, FILENAME, O_SYNC | O_CREAT | O_TRUNC, 0600, 0); + if (r < 0) { + perror("jopen"); + return 1; + } + + /* write two "Hello world"s next to each other */ + jtrans_init(&file, &trans); + jtrans_add(&trans, TEXT, strlen(TEXT), 0); + jtrans_add(&trans, TEXT, strlen(TEXT), strlen(TEXT) - 1); + r = jtrans_commit(&trans); + if (r < 0) { + perror("jtrans_commit"); + return 1; + } + + /* at this point the file has "Hello world!Hello world!" */ + + /* now we rollback */ + r = jtrans_rollback(&trans); + if (r < 0) { + perror("jtrans_rollback"); + return 1; + } + + /* and now the file is empty! */ + + jtrans_free(&trans); + jclose(&file); + return 0; +} + unchanged: --- cur/samples/build~full_sample 2004-06-19 23:57:39.903197784 -0300 +++ cur-root/samples/build 2004-06-19 23:57:39.906197328 -0300 @@ -1,4 +1,5 @@ gcc -Wall -O6 -ljio -lpthread jio1.c -o jio1 gcc -Wall -O6 -ljio -lpthread jio2.c -o jio2 gcc -Wall -O6 -ljio -lpthread jio3.c -o jio3 +gcc -Wall -O6 -ljio full.c -o full unchanged: --- cur/samples/clean~full_sample 2004-06-19 23:57:39.905197480 -0300 +++ cur-root/samples/clean 2004-06-19 23:57:39.906197328 -0300 @@ -1,2 +1,2 @@ -rm -rf jio1 jio2 jio3 test1 .test1.jio +rm -rf jio1 jio2 jio3 full test1 .test1.jio only in patch2: unchanged: --- /dev/null 2004-04-13 23:59:22.000000000 -0300 +++ cur-root/doc/guide.lyx 2004-06-19 23:57:46.000000000 -0300 @@ -0,0 +1,626 @@ +#LyX 1.3 created this file. For more info see http://www.lyx.org/ +\lyxformat 221 +\textclass article +\language english +\inputencoding auto +\fontscheme default +\graphics default +\paperfontsize default +\papersize Default +\paperpackage a4 +\use_geometry 0 +\use_amsmath 0 +\use_natbib 0 +\use_numerical_citations 0 +\paperorientation portrait +\secnumdepth 3 +\tocdepth 3 +\paragraph_separation indent +\defskip medskip +\quotes_language english +\quotes_times 2 +\papercolumns 1 +\papersides 1 +\paperpagestyle default + +\layout Title + +libjio Programmer's Guide +\layout Author + +Alberto Bertogli (albertogli@telpin.com.ar) +\layout Standard + + +\begin_inset LatexCommand \tableofcontents{} + +\end_inset + + +\layout Section + +Introduction +\layout Standard + +This small document attempts serve as a guide to the programmer who wants + to make use of the library. + It's not a replacement for the man page or reading the code; but it's a + good starting point for everyone who wants to get involved with it. +\layout Standard + +The library is not complex to use at all, and the interfaces were designed + to be as intuitive as possible, so the text is structured as a guide to + present the reader all the common structures and functions the way they're + normally used. +\layout Section + +Definitions +\layout Standard + +This is a library which provides a journaled transaction-oriented I/O API. + You've probably read this a hundred times already in the documents, and + if you haven't wondered yet what on earth does this mean you should be + reading something else! +\layout Standard + +We say this is a transaction-oriented API because we make transactions the + center of our operations, and journaled because we use a journal (which + takes the form of a directory with files on it) to guarantee coherency + even after a crash at any point. +\layout Standard + +Here we think a transaction as a list of +\emph on +(buffer, length, offset) +\emph default + to be applied to a file. + That triple is called an +\emph on +operation +\emph default +, so we can say that a transaction represent an ordered group of operations + on the same file +\emph on +. +\layout Standard + +The act of +\emph on +committing +\emph default + a transaction means writing all the elements of the list; and +\emph on +rollbacking +\emph default + means to undo a previous commit, and leave the data just as it was before + doing the commit. +\begin_inset Foot +collapsed false + +\layout Standard + +While all this definitions may seem obvious to some people, it requires + special attention because there are a lot of different definitions, and + it's not that common to see +\begin_inset Quotes eld +\end_inset + +transaction +\begin_inset Quotes erd +\end_inset + + applied to file I/O (it's a term used mostly on database stuff), so it's + important to clarify before continuing. +\end_inset + + +\layout Standard + +It's important to note that the library not only provides a convenient and + easy API to perform this kind of operations, but provides a lot of guarantees + while doing this. + The most relevant and useful is that at any point of time, even if we crash + horribly, a transaction will be either fully applied or not applied at + all. + You should not ever see partial transactions or any kind of data corruption. +\layout Standard + +To achieve this, the library uses what is called a +\emph on +journal +\emph default +, a very vague (and fashionable) term we use to describe a set of auxiliary + files that get created to store temporary data at several stages. + The proper definition and how we use them is outside the scope of this + document, and you as a programmer shouldn't need to deal with it. + In case you're curious, it's described in a bit more detail in another + text which talks about how the library works internally. + Now let's get real. +\layout Section + +The data types +\layout Standard + +To understand any library, it's essential to be confident in the knowledge + of their data structures and how they relate each other. + In libjio we have two basic structures which have a very strong relationship, + and represent the essential objects we deal with. + Note that you normally don't manipulate them directly, because they have + their own initializer functions, but they are the building blocks for the + rest of the text, which, once this is understood, is obvious and self-evident. +\layout Standard + +The first structure we face is +\family typewriter +struct\SpecialChar ~ +jfs +\family default +, called the +\emph on +file structure +\emph default +, and it represents an open file, just like a regular file descriptor or + a +\family typewriter +FILE\SpecialChar ~ +* +\family default +. +\layout Standard + +Then you find +\family typewriter +struct\SpecialChar ~ +jtrans +\family default +, called the +\emph on +transaction structure +\emph default +, which represents a single transaction. + You can have as many transactions as you want, and operate on all of them + simultaneously without problems; the library is entirely thread safe so + there's no need to worry about that. +\layout Section + +The basic functions +\layout Standard + +Now that we've described our data types, let's see how we can really operate + with the library. + +\layout Standard + +First of all, as with regular I/O, you need to open your files. + This is done with +\family typewriter +jopen() +\family default +, which looks a lot like +\family typewriter +open() +\family default + but takes a file structure instead of a file descriptor (this will be very + common among all the functions), and adds a new parameter +\emph on +jflags +\emph default + that can be used to modify some subtle library behaviour we'll see later, + and it's normally not used. +\layout Standard + +We have a happy file structure open now, and the next thing to do would + be to create a transaction. + This is what +\family typewriter +jtrans_init() +\family default + is for: it takes a file structure and a transaction structure and initializes + the latter, leaving it ready to use. +\layout Standard + +So we have our transaction, let's add a write operation to it; to do this + we use +\family typewriter +jtrans_add() +\family default +. + We could keep on adding operations to the transaction by keep on calling + +\family typewriter +jtrans_add() +\family default + as many times as we want. +\layout Standard + +Finally, we decide to apply our transaction to the file, that is, write + all the operations we've added. + And this is the easiest part: we call +\family typewriter +jtrans_commit() +\family default +, and that's it! +\layout Standard + +When we're done using the file, we call +\family typewriter +jclose() +\family default +, just like we call +\family typewriter +close() +\family default +. +\layout Standard + +Let's put it all together and code a nice +\begin_inset Quotes eld +\end_inset + +hello world +\begin_inset Quotes erd +\end_inset + + program (return values are ignored for simplicity): +\layout LyX-Code + +char buf[] = "Hello world!"; +\layout LyX-Code + +struct jfs file; +\layout LyX-Code + +struct jtrans trans; +\newline + +\newline + +\layout LyX-Code + +jopen(&file, "filename", O_RDWR | O_CREAT, 0600, 0); +\layout LyX-Code + +jtrans_init(&file, &trans); +\newline + +\newline + +\layout LyX-Code + +jtrans_add(&trans, buf, strlen(buf), 0); +\newline + +\newline + +\layout LyX-Code + +jtrans_commit(&trans); +\newline + +\newline + +\layout LyX-Code + +jclose(&file); +\layout Standard + +As we've seen, we open the file and initialize the structure with +\family typewriter +jopen() +\family default + (with the parameter +\emph on +jflags +\emph default + being the last 0)and +\family typewriter +jtrans_init() +\family default +, then add an operation with +\family typewriter +jtrans_add() +\family default + (the last 0 is the offset, in this case the beginning of the file), commit + the transaction with +\family typewriter +jtrans_commit() +\family default +, and finally close the file with +\family typewriter +jclose() +\family default +. +\layout Section + +Advanced functions +\layout Subsection + +Interaction with reads +\begin_inset LatexCommand \label{sub:Interaction-with-reads} + +\end_inset + + +\layout Standard + +So far we've seen how to use the library to perform writes, but what about + reads? The only and main issue with reads is that, because we provide transacti +on atomicity, a read must never be able to +\begin_inset Quotes eld +\end_inset + +see +\begin_inset Quotes erd +\end_inset + + a transaction partially applied. + This is achieved internally by using fine-grained file locks; but you shouldn't + mind about it if you use the functions the library gives you because they + take care of all the locking. +\layout Standard + +This set of functions are very similar to the UNIX ones ( +\family typewriter +read() +\family default +, +\family typewriter +readv() +\family default +, etc.); and in fact are named after them: they're called +\family typewriter +jread() +\family default +, +\family typewriter +jreadv() +\family default + and +\family typewriter +jpread() +\family default +; and have the same parameters except for the first one, which instead of + a file descriptor is a file structure +\begin_inset Foot +collapsed false + +\layout Standard + +In fact, this set of functions is a part of what is called the +\begin_inset Quotes eld +\end_inset + +UNIX API +\begin_inset Quotes erd +\end_inset + +, which is described below. +\end_inset + +. + Bear in mind that transactions are only visible by reads +\emph on +after +\emph default + you commit them with +\family typewriter +jtrans_commit() +\family default +. +\layout Subsection + +Rollback +\layout Standard + +There is a very nice and important feature in transactions, that allow them + to be +\begin_inset Quotes eld +\end_inset + +undone +\begin_inset Quotes erd +\end_inset + +, which means that you can undo a transaction and leave the file just as + it was the moment before applying it. + The action of undoing it is called to +\emph on +rollback +\emph default +, and the function is called +\family typewriter +jtrans_rollback() +\family default +, which takes the transaction as the only parameter. +\layout Standard + +Be aware that rollbacking a transaction can be dangerous if you're not careful + and cause you a lot of troubles. + For instance, consider you have two transactions (let's call them 1 and + 2, and assume they were applied in that order) that modify the same offset, + and you rollback transaction 1; then 2 would be lost. + It is not an dangerous operation itself, but its use requires care and + thought. +\layout Subsection + +Integrity checking and recovery +\layout Standard + +An essential part of the library is taking care of recovering from crashes + and be able to assure a file is consistent. + When you're working with the file, this is taking care of; but what when + you first open it? To answer that question, the library provides you with + a function named +\family typewriter +jfsck() +\family default +, which checks the integrity of a file and makes sure that everything is + consistent. + It must be called +\begin_inset Quotes eld +\end_inset + +offline +\begin_inset Quotes erd +\end_inset + +, that is when you are not actively committing and rollbacking; it is normally + done before calling +\family typewriter +jopen() +\family default +. + Another good practise is call jfsck_cleanup() after calling jfsck() to + make sure we're starting up with a fresh clean journal. + After both calls, it is safe to assume that the file is and ready to use. +\layout Standard + +You can also do this manually with an utility named +\emph on +jiofsck +\emph default +, which can be used from the shell to perform the checking and cleanup. +\layout Section + +Disk layout +\layout Standard + +The library creates a single directory for each file opened, named after + it. + So if we open a file +\begin_inset Quotes eld +\end_inset + +output +\begin_inset Quotes erd +\end_inset + +, a directory named +\begin_inset Quotes eld +\end_inset + +.output.jio +\begin_inset Quotes erd +\end_inset + + will be created. + We call it the +\emph on +journal directory +\emph default +, and it's used internally by the library to save temporary data; you shouldn't + modify any of the files that are inside it, or move it while it's in use. + It doesn't grow much (it only uses space for transactions that are in the + process of committing) and gets automatically cleaned while working with + it so you can (and should) ignore it. + Besides that, the file you work with has no special modification and is + just like any other file, all the internal stuff is kept isolated on the + journal directory. +\layout Section + +Other APIs +\layout Standard + +We're all used to do things our way, and when we learn something new it's + often better if it looks alike what we already know. + With this in mind, the library comes with two sets of APIs that look a + lot like traditional, well known ones. + Bear in mind that they are not as powerful as the transaction API that + is described above, and they can't provide the same functionality in a + lot of cases; however for a lot of common and simple use patterns they + are good enough. +\layout Subsection + +UNIX API +\layout Standard + +There is a set of functions that emulate the UNIX API ( +\family typewriter +read() +\family default +, +\family typewriter +write() +\family default +, and so on) which make each operation a transaction. + This can be useful if you don't need to have the full power of the transactions + but only to provide guarantees between the different functions. + They are a lot like the normal UNIX functions, but instead of getting a + file descriptor as their first parameter, they get a file structure. + You can check out the manual page to see the details, but they work just + like their UNIX version, only that they preserve atomicity and thread-safety + +\emph on +within each call +\emph default +. +\layout Standard + +In particular, the group of functions related to reading (which was described + above in +\begin_inset LatexCommand \ref{sub:Interaction-with-reads} + +\end_inset + +) are extremely useful because they take care of the locking needed for + the library proper behaviour. + You should use them instead of the regular calls. +\layout Standard + +The full function list is available on the man page and I won't reproduce + it here; however the naming is quite simple: just prepend a 'j' to all + the names: +\family typewriter +jread() +\family default +, +\family typewriter +jwrite() +\family default +, etc. +\layout Subsection + +ANSI C API +\layout Standard + +Besides the UNIX API you can find an ANSI C API, which emulates the traditional + +\family typewriter +fread() +\family default +, +\family typewriter +fwrite() +\family default +, etc. + They're still in development and has not been tested carefully, so I won't + spend time documenting them. + Let me know if you need them. +\layout Section + +Where to go from here +\layout Standard + +If you're still interested in learning more, you can find some small and + clean samples are in the +\begin_inset Quotes eld +\end_inset + +samples +\begin_inset Quotes erd +\end_inset + + directory (full.c is a simple and complete one), other more advanced examples + can be found in the web page, as well as modifications to well known software + to make use of the library. + For more information about the inner workings of the library, you can read + the +\begin_inset Quotes eld +\end_inset + +libjio +\begin_inset Quotes erd +\end_inset + + document, and the source code. +\the_end only in patch2: unchanged: --- cur/doc/libjio.lyx~new_commit_doc 2004-06-19 23:57:46.000000000 -0300 +++ cur-root/doc/libjio.lyx 2004-06-19 23:57:46.000000000 -0300 @@ -56,6 +56,8 @@ This document explains the design of the ways, it provides (or at least tries to =) an insight view on how the library performs its job, which can be very valuable knowledge when working with it. + It assumes that there is some basic knowledge about how the library is + used, which can be found in the manpage or in the programmer's guide. \layout Standard To the user, libjio provides two groups of functions, one UNIX-alike that @@ -83,7 +85,7 @@ write() The following sections describe different concepts and procedures that the library bases its work on. It's not intended to be a replace to reading the source code: please do - so if you have any doubts, it's not big at all (less than 800 lines, including + so if you have any doubts, it's not big at all (less than 1500 lines, including comments) and I hope it's readable enough. If you think that's not the case, please let me know and I'll try to give you a hand. @@ -131,18 +133,12 @@ The transaction file is composed of two \layout Standard The header holds basic information about the transaction itself, including - the ID, some flags, the offset to commit to and the lenght of the data. - The payload holds the data, in three parts: user-defined data, previous - data, and real data. -\layout Standard - -User-defined data is not used by the library itself, but it's a space where - the user can save private data that can be useful later. - Previous data is saved by the library prior applying the commit, so transaction -s can be rollbacked. - Real data is just the data to save to the disk, and it is saved because - if a crash occurs when while we are applying the transaction we can recover - gracefuly. + the ID, some flags, and the amount of operations it includes. + Then the payload has all the operations one after the other, divided in + two parts: the first one includes static information about the operation + (the lenght of the data, the offset of the file where it should be applied, + etc.) and the data itself, which is saved by the library prior applying + the commit, so transactions can be rollbacked. \layout Section The commit procedure @@ -165,7 +161,7 @@ safely \emph default , means that after a commit has been done we can assume the data will not get lost and can be retrieved, unless of course some major event happens - (like a hardware failure). + (like a physical hard disk crash). For us, this means that the data was effectively written to the disk and if a crash occurs after the commit operation has returned, the operation will be complete and data will be available from the file. @@ -175,7 +171,7 @@ The latter, \emph on atomically \emph default -, warantees that the operation is either completely done, or not done at +, guarantees that the operation is either completely done, or not done at all. This is a really common word, specially if you have worked with multiprocessing , and should be quite familiar. @@ -193,7 +189,7 @@ jtrans_commit() : \layout Itemize -Lock the section where the commit takes place +Lock the file offsets where the commit takes place \layout Itemize Open the transaction file @@ -202,10 +198,7 @@ Open the transaction file Write the header \layout Itemize -Write the user data (if any) -\layout Itemize - -Read the previous data from the file +Read all the previous data from the file \layout Itemize Write the previous data in the transaction @@ -220,7 +213,7 @@ Mark the transaction as commited by sett Unlink the transaction file \layout Itemize -Unlock the section where the commit takes place +Unlock the offsets where the commit takes place \layout Standard This may look as a lot of steps, but they're not as much as it looks like @@ -245,8 +238,8 @@ undo and straightforward. \layout Standard -In the previous section we said that each transaction held, besides the - data to commit to the disk, the data that was on it before commiting. +In the previous section we said that each transaction held the data that + was on it before commiting. That data is saved precisely to be able to rollback. So, to rollback a transaction all that has to be done is recover that \begin_inset Quotes eld @@ -332,15 +325,15 @@ In any case, after making the recovery y transaction atomicity was preserved. \layout Section -High-level functions +UNIX API \layout Standard We call \emph on -high-level functions +UNIX API \emph default - to the ones provided by the library that emulate the good old unix file - manipulation calls. + to the functions provided by the library that emulate the good old UNIX + file manipulation calls. Most of them are just wrappers around commits, and implement proper locking when operating in order to allow simultaneous operations (either across threads or processes). only in patch2: unchanged: --- cur/doc/libjio.3~new_commit_doc 2004-06-19 23:57:46.000000000 -0300 +++ cur-root/doc/libjio.3 2004-06-20 00:02:46.000000000 -0300 @@ -24,10 +24,12 @@ libjio - A library for Journaled I/O .BI "int jclose(struct jfs *" fs " ); -.BI "void jtrans_init(struct jfs *" fs " , struct jtrans *" ts " ); +.BI "void jtrans_init(struct jfs *" fs " ,struct jtrans *" ts " ); .BI "int jtrans_commit(struct jtrans *" ts " ); +.BI "int jtrans_add(struct jtrans *" ts ", const void * " buf ", size_t " count ", off_t " offset " ); + .BI "int jtrans_rollback(struct jtrans *" ts " ); .BI "void jtrans_free(struct jtrans *" ts " ); @@ -40,56 +42,47 @@ libjio - A library for Journaled I/O .PP .br .nf -.in 10 +.in +2n struct jfs { -.in 14 -int fd; /* main file descriptor */ -char *name; /* and its name */ -int jfd; /* journal's lock file descriptor */ -int flags; /* journal mode options used in jopen() */ -pthread_mutex_t lock; /* a soft lock used in some operations */ -.in 10 + int fd; /* main file descriptor */ + char *name; /* and its name */ + int jfd; /* journal's lock file descriptor */ + int flags; /* journal mode options used in jopen() */ + pthread_mutex_t lock; /* a soft lock used in some operations */ + ... }; .FI +.in -2n .PP .br .nf -.in 10 +.in +2n struct jtrans { -.in 14 -struct jfs *fs; /* journal file structure to operate on */ -char *name; /* name of the transaction file */ -int id; /* transaction id */ -int flags; /* misc flags */ -void *buf; /* buffer */ -size_t len; /* buffer lenght */ -off_t offset; /* file offset to operate on */ -void *udata; /* user-supplied data */ -size_t ulen; /* udata lenght */ -void *pdata; /* previous data, for rollback */ -size_t plen; /* pdata lenght */ -.in 10 + struct jfs *fs; /* journal file structure to operate on */ + char *name; /* name of the transaction file */ + int id; /* transaction id */ + int flags; /* misc flags */ + ... }; .FI +.in -2n .PP .br .nf -.in 10 +.in +2n struct jfsck_result { -.in 14 -int total; /* total transactions files we looked at */ -int invalid; /* invalid files in the journal directory */ -int in_progress; /* transactions in progress */ -int broken_head; /* transactions broken (header missing) */ -int broken_body; /* transactions broken (body missing) */ -int load_error; /* errors loading the transaction */ -int apply_error; /* errors applying the transaction */ -int rollbacked; /* transactions that were rollbacked */ -.in 10 + int total; /* total transactions files we looked at */ + int invalid; /* invalid files in the journal directory */ + int in_progress; /* transactions in progress */ + int broken; /* transactions broken */ + int apply_error; /* errors applying the transaction */ + int rollbacked; /* transactions that were rollbacked */ + ... }; .FI +.in -2n .SH DESCRIPTION @@ -98,15 +91,21 @@ describes it's C API very briefly, furth documentation that comes along with the library itself, or on the web at http://auriga.wearlab.de/~alb/libjio. -We can group the functions into three groups: a common one, with functions -common to the other two; low-level one, which consists of jtrans_commit and -jtrans_rollback. They provide a method for manipulating transactions, which -are defined in the jtrans structure (described above). +We can group the functions into three groups: the common functions, the basic +functions and the UNIX API. + +The common functions provide functionality common to the other two: jopen to +open files to use them with the library, and jfsck and jfsck_cleanup to +provide integrity checking. + +The basic functions consists of jtrans_commit, jtrans_add and jtrans_rollback. +They provide a method for manipulating transactions, which are defined in the +jtrans structure (described above). The second group mimics somehow the traditional UNIX API by providing similar interfaces to read(), write(), and their friends. -.SH COMMON API +.SH Common functions Most functions reference somehow the structures described avobe, specially struct jfs and struct jtrans. They represent a file to operate on and a single @@ -133,9 +132,9 @@ longer be needed, so by cleaning up the starting over with a clean journal. It returns 0 if there was an error, or 1 if it succeeded. -.SH HIGH LEVEL API +.SH UNIX API -The high level API, as explained before, consists of the functions jread(), +The UNIX API, as explained before, consists of the functions jread(), jpread(), jreadv(), jwrite(), jpwrite(), jwritev(), jtruncate(). In most cases you will only need to use this, because they're simple and familiar. @@ -146,10 +145,10 @@ like jopen() and jclose()). Again, I wil these functions, just refer to the regular UNIX versions to see how to use them, they all have the same semantics and behave the same way. -.SH LOW LEVEL API +.SH Basic functions -The low level functions are the ones which manipulate transactions directly; -they are four: jtrans_init(), jtrans_commit(), jtrans_rollback() and +The basic functions are the ones which manipulate transactions directly; they +are five: jtrans_init(), jtrans_add(), jtrans_commit(), jtrans_rollback() and jtrans_free(). These are intended to be use in special situations where your application needs direct control over the transactions. @@ -160,10 +159,16 @@ only frees the pointers that were previo disk operations are performed by the other two functions. They have no return value. -jtrans_commit() is in charge of commiting the given transaction (which data -was completed by you, and is described in the STRUCTURES section), and after -its return the data has been saved to the disk atomically. It returns the -number of bytes written or -1 if there was an error. +jtrans_add() is used to add operations to a transaction, and it takes the same +parameters as the pwrite() call. It gets a buffer, it's lenght and the offset +where it should be applied, and adds it to the transaction. You can add +multiple operations to a transaction, and they will be applied in order. +Operation within the same transaction must not overlap; if they do, commiting +the transaction will fail. + +jtrans_commit() is in charge of commiting the given transaction, and after its +return the data has been saved to the disk atomically. It returns the number +of bytes written or -1 if there was an error. jtrans_rollback() reverses a transaction that was applied with jtrans_commit(), and leaves the file as it was before applying it. Be very only in patch2: unchanged: --- cur/doc/big_transactions +++ /dev/null 2004-04-13 23:59:22.000000000 -0300 @@ -1,9 +0,0 @@ - -If you have to create big transactions, instead of creating a huge buffer you -can mmap a temporary file and periodically sync it; and when you're done, just -jtrans_commit() the whole thing. - -This would be a quite efficient way, without any performance penalty and a -very simple approach; I originally thought of doing this on the journal, but -it had many drawbacks that made it much expensive, slower and complex. - only in patch2: unchanged: --- cur/doc/jiofsck +++ /dev/null 2004-04-13 23:59:22.000000000 -0300 @@ -1,12 +0,0 @@ - -Note that jfsck does not warantee that all the transactions are fully -completed, it can only do so if you run it without any other process accessing -the journal. - -If you want to see this, you can take a look at the struct jfsck_result. It -include a field named in_progress which tell the number of transactions that -were in progress at the moment of checking, and as such weren't checked. - -Be aware that the counter is not atomic, as two checkers can be running at the -same time. - only in patch2: unchanged: --- cur/doc/threads +++ /dev/null 2004-04-13 23:59:22.000000000 -0300 @@ -1,20 +0,0 @@ - -The library is entirely threadsafe. - -This will make some people who worked with threads a bit concerned, because -everybody knows that if a file descriptor is shared among threads, and two -threads decide to read/write/perform any op that moves the file pointer, a -mess is waiting to happen. And almost operations do touch the file pointer. - -But don't worry, the library is _truly_ threadsafe: it uses pread/pwrite, -which do not touch the file pointer, and allows working on the same file -simultaneously without concerns. Besides, it slightly improves performance by -having less locking, less system calls, lower overhead and less calculation to -perform the operation. - - -Still, bear in mind that if you decide to work on the file outside libjio you -need to lockf() the sections you're going to work on, because libjio relies on -lockf() locking to warantee atomicity. - - only in patch2: unchanged: --- cur/README~new_commit_doc 2004-06-20 00:08:58.000000000 -0300 +++ cur-root/README 2004-06-20 00:12:11.000000000 -0300 @@ -16,10 +16,12 @@ special directory is created to store in This allows both simple file manipulation, recovery and debugging because everything is isolated. -There's a more detailed document about the library itself in doc/libjio.txt -which is based on doc/libjio.lyx (there are HTML, Postscript, PDF and TXT -versions in the website, which are not included here for space reasons), and a -manpage where you will find the API reference. +There are more detailed documents: a programming guide, a brief introduction +to the design and inner workings, and the manpage; all in the doc/ directory. + +The first two, called 'guide' and 'libjio' respectively, are both in txt and +lyx formats, with HTML, Postscript and PDF versions in the website, which are +not included in the package for space reasons. To see how to install it, please read the INSTALL file. only in patch2: unchanged: --- cur/doc/libjio.txt~new_commit_doc 2004-06-20 00:17:28.402257568 -0300 +++ cur-root/doc/libjio.txt 2004-06-20 00:18:06.834414992 -0300 @@ -10,7 +10,7 @@ Table of Contents 3 The commit procedure 4 The rollback procedure 5 The recovery procedure -6 High-level functions +6 UNIX API 7 ACID (or How does libjio fit into theory) 8 Working from outside @@ -28,7 +28,9 @@ should read it even if you don't plan to library in strange ways, it provides (or at least tries to =) an insight view on how the library performs its job, which can be very valuable knowledge when working -with it. +with it. It assumes that there is some basic knowledge +about how the library is used, which can be found in +the manpage or in the programmer's guide. To the user, libjio provides two groups of functions, one UNIX-alike that implements the journaled versions @@ -46,7 +48,7 @@ The following sections describe differen procedures that the library bases its work on. It's not intended to be a replace to reading the source code: please do so if you have any doubts, it's not big at -all (less than 800 lines, including comments) and I +all (less than 1500 lines, including comments) and I hope it's readable enough. If you think that's not the case, please let me know and I'll try to give you a hand. @@ -88,19 +90,14 @@ The transaction file is composed of two header and the payload. The header holds basic information about the -transaction itself, including the ID, some flags, the -offset to commit to and the lenght of the data. The -payload holds the data, in three parts: user-defined -data, previous data, and real data. - -User-defined data is not used by the library itself, -but it's a space where the user can save private data -that can be useful later. Previous data is saved by the -library prior applying the commit, so transactions can -be rollbacked. Real data is just the data to save to -the disk, and it is saved because if a crash occurs -when while we are applying the transaction we can -recover gracefuly. +transaction itself, including the ID, some flags, and +the amount of operations it includes. Then the payload +has all the operations one after the other, divided in +two parts: the first one includes static information +about the operation (the lenght of the data, the offset +of the file where it should be applied, etc.) and the +data itself, which is saved by the library prior +applying the commit, so transactions can be rollbacked. 3 The commit procedure @@ -110,15 +107,15 @@ write some given data to the disk. The former, safely, means that after a commit has been done we can assume the data will not get lost and can be retrieved, unless of course some major event happens -(like a hardware failure). For us, this means that the -data was effectively written to the disk and if a crash -occurs after the commit operation has returned, the -operation will be complete and data will be available -from the file. - -The latter, atomically, warantees that the operation is -either completely done, or not done at all. This is a -really common word, specially if you have worked with +(like a physical hard disk crash). For us, this means +that the data was effectively written to the disk and +if a crash occurs after the commit operation has +returned, the operation will be complete and data will +be available from the file. + +The latter, atomically, guarantees that the operation +is either completely done, or not done at all. This is +a really common word, specially if you have worked with multiprocessing, and should be quite familiar. We implement atomicity by combining fine-grained locks and journaling, which can assure us both to be able to @@ -130,15 +127,13 @@ Well, so much for talking, now let's get applies commits in a very simple and straightforward way, inside jtrans_commit(): -* Lock the section where the commit takes place +* Lock the file offsets where the commit takes place * Open the transaction file * Write the header -* Write the user data (if any) - -* Read the previous data from the file +* Read all the previous data from the file * Write the previous data in the transaction @@ -149,7 +144,7 @@ way, inside jtrans_commit(): * Unlink the transaction file -* Unlock the section where the commit takes place +* Unlock the offsets where the commit takes place This may look as a lot of steps, but they're not as much as it looks like inside the code, and allows a @@ -164,17 +159,16 @@ commit was applied. Due to the way we ha doing this operation becomes quite simple and straightforward. In the previous section we said that each transaction -held, besides the data to commit to the disk, the data -that was on it before commiting. That data is saved -precisely to be able to rollback. So, to rollback a -transaction all that has to be done is recover that " -previous data" from the transaction we want to rollback, -and save it to the disk. In the end, this ends up being -a new transaction with the previous data as the new -one, so we do that: create a new transaction structure, -fill in the data from the transaction we want to -rollback, and commit it. All this is performed by -jtrans_rollback(). +held the data that was on it before commiting. That +data is saved precisely to be able to rollback. So, to +rollback a transaction all that has to be done is +recover that "previous data" from the transaction we want +to rollback, and save it to the disk. In the end, this +ends up being a new transaction with the previous data +as the new one, so we do that: create a new transaction +structure, fill in the data from the transaction we +want to rollback, and commit it. All this is performed +by jtrans_rollback(). By doing this we can provide the same warranties a commit has, it's really fast, eases the recovery, and @@ -220,10 +214,10 @@ remove the journal entirely and let the a new one, and you can be sure that transaction atomicity was preserved. -6 High-level functions +6 UNIX API -We call high-level functions to the ones provided by -the library that emulate the good old unix file +We call UNIX API to the functions provided by the +library that emulate the good old UNIX file manipulation calls. Most of them are just wrappers around commits, and implement proper locking when operating in order to allow simultaneous operations only in patch2: unchanged: --- /dev/null 2004-04-13 23:59:22.000000000 -0300 +++ cur-root/doc/guide.txt 2004-06-20 00:21:59.026116488 -0300 @@ -0,0 +1,295 @@ +libjio Programmer's Guide + +Alberto Bertogli (albertogli@telpin.com.ar) + +Table of Contents + +1 Introduction +2 Definitions +3 The data types +4 The basic functions +5 Advanced functions + 5.1 Interaction with reads + 5.2 Rollback + 5.3 Integrity checking and recovery +6 Disk layout +7 Other APIs + 7.1 UNIX API + 7.2 ANSI C API +8 Where to go from here + + + +1 Introduction + +This small document attempts serve as a guide to the +programmer who wants to make use of the library. It's +not a replacement for the man page or reading the code; +but it's a good starting point for everyone who wants +to get involved with it. + +The library is not complex to use at all, and the +interfaces were designed to be as intuitive as +possible, so the text is structured as a guide to +present the reader all the common structures and +functions the way they're normally used. + +2 Definitions + +This is a library which provides a journaled +transaction-oriented I/O API. You've probably read this +a hundred times already in the documents, and if you +haven't wondered yet what on earth does this mean you +should be reading something else! + +We say this is a transaction-oriented API because we +make transactions the center of our operations, and +journaled because we use a journal (which takes the +form of a directory with files on it) to guarantee +coherency even after a crash at any point. + +Here we think a transaction as a list of (buffer, +length, offset) to be applied to a file. That triple is +called an operation, so we can say that a transaction +represent an ordered group of operations on the same file. + +The act of committing a transaction means writing all +the elements of the list; and rollbacking means to undo +a previous commit, and leave the data just as it was +before doing the commit (while all this definitions may +seem obvious to some people, it requires special attention +because there are a lot of different definitions, and it's +not that common to see "transaction" applied to file I/O +(it's a term used mostly on database stuff), so it's +important to clarify before continuing). + +It's important to note that the library not only +provides a convenient and easy API to perform this kind +of operations, but provides a lot of guarantees while +doing this. The most relevant and useful is that at any +point of time, even if we crash horribly, a transaction +will be either fully applied or not applied at all. You +should not ever see partial transactions or any kind of +data corruption. + +To achieve this, the library uses what is called a +journal, a very vague (and fashionable) term we use to +describe a set of auxiliary files that get created to +store temporary data at several stages. The proper +definition and how we use them is outside the scope of +this document, and you as a programmer shouldn't need +to deal with it. In case you're curious, it's described +in a bit more detail in another text which talks about +how the library works internally. Now let's get real. + +3 The data types + +To understand any library, it's essential to be +confident in the knowledge of their data structures and +how they relate each other. In libjio we have two basic +structures which have a very strong relationship, and +represent the essential objects we deal with. Note that +you normally don't manipulate them directly, because +they have their own initializer functions, but they are +the building blocks for the rest of the text, which, +once this is understood, is obvious and self-evident. + +The first structure we face is struct jfs, called the +file structure, and it represents an open file, just +like a regular file descriptor or a FILE *. + +Then you find struct jtrans, called the transaction +structure, which represents a single transaction. You +can have as many transactions as you want, and operate +on all of them simultaneously without problems; the +library is entirely thread safe so there's no need to +worry about that. + +4 The basic functions + +Now that we've described our data types, let's see how +we can really operate with the library. + +First of all, as with regular I/O, you need to open +your files. This is done with jopen(), which looks a +lot like open() but takes a file structure instead of a +file descriptor (this will be very common among all the +functions), and adds a new parameter jflags that can be +used to modify some subtle library behaviour we'll see +later, and it's normally not used. + +We have a happy file structure open now, and the next +thing to do would be to create a transaction. This is +what jtrans_init() is for: it takes a file structure +and a transaction structure and initializes the latter, +leaving it ready to use. + +So we have our transaction, let's add a write operation +to it; to do this we use jtrans_add(). We could keep on +adding operations to the transaction by keep on calling +jtrans_add() as many times as we want. + +Finally, we decide to apply our transaction to the +file, that is, write all the operations we've added. +And this is the easiest part: we call jtrans_commit(), +and that's it! + +When we're done using the file, we call jclose(), just +like we call close(). + +Let's put it all together and code a nice "hello world" +program (return values are ignored for simplicity): + +char buf[] = "Hello world!"; +struct jfs file; +struct jtrans trans; + +jopen(&file, "filename", O_RDWR | O_CREAT, 0600, 0); +jtrans_init(&file, &trans); +jtrans_add(&trans, buf, strlen(buf), 0); + +jtrans_commit(&trans); + +jclose(&file); + +As we've seen, we open the file and initialize the +structure with jopen() (with the parameter jflags being +the last 0)and jtrans_init(), then add an operation +with jtrans_add() (the last 0 is the offset, in this +case the beginning of the file), commit the transaction +with jtrans_commit(), and finally close the file with jclose(). + +5 Advanced functions + +5.1 Interaction with reads + +So far we've seen how to use the library to perform +writes, but what about reads? The only and main issue +with reads is that, because we provide transaction +atomicity, a read must never be able to "see" a +transaction partially applied. This is achieved +internally by using fine-grained file locks; but you +shouldn't mind about it if you use the functions the +library gives you because they take care of all the locking. + +This set of functions are very similar to the UNIX ones +(read(), readv(), etc.); and in fact are named after +them: they're called jread(), jreadv() and jpread(); +and have the same parameters except for the first one, +which instead of a file descriptor is a file structure (in +fact, this set of functions is a part of what is called +the "UNIX API", which is described below). Bear in mind +that transactions are only visible by reads after you +commit them with jtrans_commit(). + +5.2 Rollback + +There is a very nice and important feature in +transactions, that allow them to be "undone", which means +that you can undo a transaction and leave the file just +as it was the moment before applying it. The action of +undoing it is called to rollback, and the function is +called jtrans_rollback(), which takes the transaction +as the only parameter. + +Be aware that rollbacking a transaction can be +dangerous if you're not careful and cause you a lot of +troubles. For instance, consider you have two +transactions (let's call them 1 and 2, and assume they +were applied in that order) that modify the same +offset, and you rollback transaction 1; then 2 would be +lost. It is not an dangerous operation itself, but its +use requires care and thought. + +5.3 Integrity checking and recovery + +An essential part of the library is taking care of +recovering from crashes and be able to assure a file is +consistent. When you're working with the file, this is +taking care of; but what when you first open it? To +answer that question, the library provides you with a +function named jfsck(), which checks the integrity of a +file and makes sure that everything is consistent. It +must be called "offline", that is when you are not +actively committing and rollbacking; it is normally +done before calling jopen(). Another good practise is +call jfsck_cleanup() after calling jfsck() to make sure +we're starting up with a fresh clean journal. After +both calls, it is safe to assume that the file is and +ready to use. + +You can also do this manually with an utility named +jiofsck, which can be used from the shell to perform +the checking and cleanup. + +6 Disk layout + +The library creates a single directory for each file +opened, named after it. So if we open a file "output", a +directory named ".output.jio" will be created. We call it +the journal directory, and it's used internally by the +library to save temporary data; you shouldn't modify +any of the files that are inside it, or move it while +it's in use. It doesn't grow much (it only uses space +for transactions that are in the process of committing) +and gets automatically cleaned while working with it so +you can (and should) ignore it. Besides that, the file +you work with has no special modification and is just +like any other file, all the internal stuff is kept +isolated on the journal directory. + +7 Other APIs + +We're all used to do things our way, and when we learn +something new it's often better if it looks alike what +we already know. With this in mind, the library comes +with two sets of APIs that look a lot like traditional, +well known ones. Bear in mind that they are not as +powerful as the transaction API that is described +above, and they can't provide the same functionality in +a lot of cases; however for a lot of common and simple +use patterns they are good enough. + +7.1 UNIX API + +There is a set of functions that emulate the UNIX API +(read(), write(), and so on) which make each operation +a transaction. This can be useful if you don't need to +have the full power of the transactions but only to +provide guarantees between the different functions. +They are a lot like the normal UNIX functions, but +instead of getting a file descriptor as their first +parameter, they get a file structure. You can check out +the manual page to see the details, but they work just +like their UNIX version, only that they preserve +atomicity and thread-safety within each call. + +In particular, the group of functions related to reading +(which was described above in "Interaction with reads") +are extremely useful because they take care of the locking +needed for the library proper behaviour. You should use +them instead of the regular calls. + +The full function list is available on the man page and +I won't reproduce it here; however the naming is quite +simple: just prepend a 'j' to all the names: jread(), +jwrite(), etc. + +7.2 ANSI C API + +Besides the UNIX API you can find an ANSI C API, which +emulates the traditional fread(), fwrite(), etc. +They're still in development and has not been tested +carefully, so I won't spend time documenting them. Let +me know if you need them. + +8 Where to go from here + +If you're still interested in learning more, you can +find some small and clean samples are in the "samples" +directory (full.c is a simple and complete one), other +more advanced examples can be found in the web page, as +well as modifications to well known software to make +use of the library. For more information about the +inner workings of the library, you can read the "libjio" +document, and the source code.