Commit 50e88538 authored by Per Cederqvist's avatar Per Cederqvist
Browse files

Write dbfiles atomically, renaming them once complete.

Added support for renaming a datafile once it has been completely
written, and to write it using a temporary extension.

* src/server/ram-io.h (struct dbfile): Added fields fn,
tmp_extension and tmp_fn.  Added documentation.
(dbfile_open_write): New argument: tmp_extension.
(dbfile_rename): New function.
(dbfile_abort): New function.
* src/server/ram-io.c (dbfile_new): Initialize fn, tmp_extension
and tmp_fn.
(current_fn): New static function.  Return tmp_fn if non-NULL,
otherwise fn.
(dbfile_delete): Free the new fields fn, tmp_extension and
tmp_fn.
(dbfile_open_write): New argument: tmp_extension.  Store a copy of
the supplied filename.  If tmp_extension is non-NULL, store a copy
of it, and compute the actual file name to open as the filname
followed by the tmp_extension.
(fsync_dirname): New static function.
(dbfile_rename): New function.  Rename the file from tmp_fn to the
wanted final filename, and sync the directory.
(dbfile_abort): New function.
* src/server/dbck-cache.c (cache_sync_all): Pass a NULL pointer as
tmp_extension of dbfile_open_write(), to get the old behavior.
* src/server/simple-cache.c (pre_sync): Ditto.
parent 0cb622ac
dbfile_open_read-check-magic sstrdup
From: Per Cederqvist <ceder@lysator.liu.se> From: Per Cederqvist <ceder@lysator.liu.se>
Subject: Created the utility function sstrdup. Subject: Write dbfiles atomically, renaming them once complete.
* src/server/ram-smalloc.c, src/include/server/smalloc.h: Added support for renaming a datafile once it has been completely
(sstrdup): New function. written, and to write it using a temporary extension.
* src/server/ram-io.h (struct dbfile): Added fields fn,
tmp_extension and tmp_fn. Added documentation.
(dbfile_open_write): New argument: tmp_extension.
(dbfile_rename): New function.
(dbfile_abort): New function.
* src/server/ram-io.c (dbfile_new): Initialize fn, tmp_extension
and tmp_fn.
(current_fn): New static function. Return tmp_fn if non-NULL,
otherwise fn.
(dbfile_delete): Free the new fields fn, tmp_extension and
tmp_fn.
(dbfile_open_write): New argument: tmp_extension. Store a copy of
the supplied filename. If tmp_extension is non-NULL, store a copy
of it, and compute the actual file name to open as the filname
followed by the tmp_extension.
(fsync_dirname): New static function.
(dbfile_rename): New function. Rename the file from tmp_fn to the
wanted final filename, and sync the directory.
(dbfile_abort): New function.
* src/server/dbck-cache.c (cache_sync_all): Pass a NULL pointer as
tmp_extension of dbfile_open_write(), to get the old behavior.
* src/server/simple-cache.c (pre_sync): Ditto.
...@@ -42,6 +42,31 @@ ...@@ -42,6 +42,31 @@
   
2010-04-01 Per Cederqvist <ceder@lysator.liu.se> 2010-04-01 Per Cederqvist <ceder@lysator.liu.se>
   
Added support for renaming a datafile once it has been completely
written, and to write it using a temporary extension.
* src/server/ram-io.h (struct dbfile): Added fields fn,
tmp_extension and tmp_fn. Added documentation.
(dbfile_open_write): New argument: tmp_extension.
(dbfile_rename): New function.
(dbfile_abort): New function.
* src/server/ram-io.c (dbfile_new): Initialize fn, tmp_extension
and tmp_fn.
(current_fn): New static function. Return tmp_fn if non-NULL,
otherwise fn.
(dbfile_delete): Free the new fields fn, tmp_extension and
tmp_fn.
(dbfile_open_write): New argument: tmp_extension. Store a copy of
the supplied filename. If tmp_extension is non-NULL, store a copy
of it, and compute the actual file name to open as the filname
followed by the tmp_extension.
(fsync_dirname): New static function.
(dbfile_rename): New function. Rename the file from tmp_fn to the
wanted final filename, and sync the directory.
(dbfile_abort): New function.
* src/server/dbck-cache.c (cache_sync_all): Pass a NULL pointer as
tmp_extension of dbfile_open_write(), to get the old behavior.
* src/server/simple-cache.c (pre_sync): Ditto.
Created the utility function sstrdup. Created the utility function sstrdup.
* src/server/ram-smalloc.c, src/include/server/smalloc.h: * src/server/ram-smalloc.c, src/include/server/smalloc.h:
(sstrdup): New function. (sstrdup): New function.
......
...@@ -581,7 +581,7 @@ cache_sync_all(void) ...@@ -581,7 +581,7 @@ cache_sync_all(void)
else else
kom_log("cache_sync_all: datafile not clean. No backup taken.\n"); kom_log("cache_sync_all: datafile not clean. No backup taken.\n");
if ((fp = dbfile_open_write(param.datafile_name, "DIRTY")) == NULL) if ((fp = dbfile_open_write(param.datafile_name, "DIRTY", NULL)) == NULL)
{ {
kom_log("WARNING: cache_sync_all: can't open file to save in.\n"); kom_log("WARNING: cache_sync_all: can't open file to save in.\n");
return; return;
......
...@@ -26,10 +26,13 @@ ...@@ -26,10 +26,13 @@
# include <config.h> # include <config.h>
#endif #endif
#include <fcntl.h>
#include <errno.h> #include <errno.h>
#include <stdio.h> #include <stdio.h>
#include <assert.h> #include <assert.h>
#include <unistd.h>
#include <string.h> #include <string.h>
#include <stddef.h>
#include "timewrap.h" #include "timewrap.h"
#include "server/smalloc.h" #include "server/smalloc.h"
...@@ -51,13 +54,27 @@ static struct dbfile * ...@@ -51,13 +54,27 @@ static struct dbfile *
dbfile_new(void) dbfile_new(void)
{ {
struct dbfile *res = smalloc(sizeof(struct dbfile)); struct dbfile *res = smalloc(sizeof(struct dbfile));
res->fp = NULL;
res->fp = NULL;
res->fn = NULL;
res->tmp_extension = NULL;
res->tmp_fn = NULL;
res->format = output_format; res->format = output_format;
++nr_dbfile_objects; ++nr_dbfile_objects;
return res; return res;
} }
static const char *
current_fn(struct dbfile *obj)
{
if (obj->tmp_fn != NULL)
return obj->tmp_fn;
return obj->fn;
}
int int
dbfile_delete(struct dbfile *obj) dbfile_delete(struct dbfile *obj)
{ {
...@@ -71,6 +88,10 @@ dbfile_delete(struct dbfile *obj) ...@@ -71,6 +88,10 @@ dbfile_delete(struct dbfile *obj)
--nr_dbfile_files; --nr_dbfile_files;
} }
sfree(obj->fn);
sfree(obj->tmp_extension);
sfree(obj->tmp_fn);
sfree(obj); sfree(obj);
errno = errno_save; errno = errno_save;
--nr_dbfile_objects; --nr_dbfile_objects;
...@@ -163,13 +184,27 @@ dbfile_open_read(const char *filename, ...@@ -163,13 +184,27 @@ dbfile_open_read(const char *filename,
struct dbfile * struct dbfile *
dbfile_open_write(const char *filename, dbfile_open_write(const char *filename,
const char *magic) const char *magic,
const char *tmp_extension)
{ {
struct dbfile *res = dbfile_new(); struct dbfile *res = dbfile_new();
if (res == NULL) if (res == NULL)
return NULL; return NULL;
if ((res->fp = i_fopen(filename, "wb")) == NULL) res->fn = sstrdup(filename);
if (tmp_extension == NULL)
{
res->tmp_extension = NULL;
res->tmp_fn = NULL;
}
else
{
res->tmp_extension = sstrdup(tmp_extension);
res->tmp_fn = smalloc(strlen(filename) + strlen(tmp_extension) + 1);
sprintf(res->tmp_fn, "%s%s", filename, tmp_extension);
}
if ((res->fp = i_fopen(current_fn(res), "wb")) == NULL)
{ {
int saved_errno = errno; int saved_errno = errno;
dbfile_delete(res); dbfile_delete(res);
...@@ -218,6 +253,88 @@ dbfile_change_magic(struct dbfile *fp, ...@@ -218,6 +253,88 @@ dbfile_change_magic(struct dbfile *fp,
return OK; return OK;
} }
/* Call fsync on the parent directory of file name fn. */
static Success
fsync_dirname(const char *fn)
{
const char *tail = strrchr(fn, '/');
char *dir;
ptrdiff_t len;
int fd;
if (tail == NULL)
restart_kom("The file name %s contains no slash.\n", fn);
len = tail - fn;
dir = smalloc(len + 1);
strncpy(dir, fn, len);
dir[len] = '\0';
fd = open(dir, O_RDONLY);
if (fd < 0)
{
kom_log("Failed to open directory %s: %s.\n", dir, strerror(errno));
sfree(dir);
return FAILURE;
}
if (fsync(fd) != 0)
{
kom_log("fsync(%s) failed: %s.\n", dir, strerror(errno));
close(fd);
sfree(dir);
return FAILURE;
}
close(fd);
sfree(dir);
return OK;
}
Success
dbfile_rename(struct dbfile *fp)
{
if (fp->tmp_fn == NULL)
restart_kom("dbfile_rename called at the wrong time for %s.\n", fp->fn);
if (rename(fp->tmp_fn, fp->fn) != 0)
{
kom_log("Failed to rename %s to %s: %s.\n", fp->tmp_fn, fp->fn,
strerror(errno));
return FAILURE;
}
sfree(fp->tmp_fn);
fp->tmp_fn = NULL;
/* Since the rename succeeded, there really is no point in failing
just because we cannot sync the directory. Log a message (to
catch any systematic problems) and continue and hope for the
best. */
if (fsync_dirname(fp->fn) != OK)
kom_log("Ignoring failed attempt to sync directory of %s.\n", fp->fn);
return OK;
}
extern void
dbfile_abort(struct dbfile *fp)
{
const char *fn = current_fn(fp);
if (fclose(fp->fp) != 0)
kom_log("dbfile_abort: fclose of %s failed: %s (ignored).\n",
fn, strerror(errno));
fp->fp = NULL;
if (remove(fn) != 0)
kom_log("dbfile_abort: failed to remove file %s: %s (ignored).\n",
fn, strerror(errno));
}
void void
dump_dbfile_stats(FILE *fp) dump_dbfile_stats(FILE *fp)
......
...@@ -22,12 +22,27 @@ ...@@ -22,12 +22,27 @@
* Please report bugs at http://bugzilla.lysator.liu.se/. * Please report bugs at http://bugzilla.lysator.liu.se/.
*/ */
/* Store state needed to access a database status file. There can be /* Store state needed to access a database file. */
several files opened at the same time, with different formats. This
struct keeps track of the FILE* and format version. */
struct dbfile { struct dbfile {
/* The file name. */
char *fn;
/* The temporary extension used when writing a file that is
renamed once it has been completely written. NULL for files
opened for reading, and for files that are not renamed. */
char *tmp_extension;
/* Only used for files opened for writing, with a non-NULL
_extension. While the file is opened under its temporary
name, it is stored here. In all other cases, this is NULL. */
char *tmp_fn;
/* The FILE pointer. */
FILE *fp; FILE *fp;
/* The format version used for this file. */
int format; int format;
}; };
...@@ -42,10 +57,35 @@ void set_output_format(int new_format); ...@@ -42,10 +57,35 @@ void set_output_format(int new_format);
/* Open the file for writing, and write a header that contains the /* Open the file for writing, and write a header that contains the
specified magic marker. The magic string must be 5 characters specified magic marker. The magic string must be 5 characters
long. */ long.
If tmp_extension is non-NULL, it is appended to filename before the
file is opened for writing. Use dbfile_change_magic() to update
the magic cookie and dbfile_rename() to rename the file once the
entire file has been written, or dbfile_abort_write() if the file
should be removed. It is an error to call dbfile_delete() before
calling dbfile_rename() or dbfile_abort_write() if tmp_extension is
non-NULL. tmp_extension should not contain slashes, and should
typically start with a dot. */
extern struct dbfile * extern struct dbfile *
dbfile_open_write(const char *filename, dbfile_open_write(const char *filename,
const char *magic); const char *magic,
const char *tmp_extension);
/* Rename the file by removing the tmp_extension (specified in the
call to dbfile_open_write()) from the file name. This function
should only be called on file handles returned by
dbfile_open_write(). Typically, dbfile_change_magic() should be
used before this function. */
extern Success
dbfile_rename(struct dbfile *fp);
/* Close the file and remove it.
Note: You must still use dbfile_delete to remove the in-core data
structures. */
extern void
dbfile_abort(struct dbfile *fp);
/* Change the magic cookie in the file. This is typically done to /* Change the magic cookie in the file. This is typically done to
mark the file as clean once it is completely written. The magic mark the file as clean once it is completely written. The magic
......
...@@ -1909,7 +1909,8 @@ pre_sync(void) ...@@ -1909,7 +1909,8 @@ pre_sync(void)
dbfile_delete(file_b); dbfile_delete(file_b);
} }
if ((file_b = dbfile_open_write(param.datafile_name, "DIRTY")) == NULL) if ((file_b = dbfile_open_write(param.datafile_name,
"DIRTY", NULL)) == NULL)
{ {
kom_log("WARNING: pre_sync: can't open file %s to save in: %s.\n", kom_log("WARNING: pre_sync: can't open file %s to save in: %s.\n",
param.datafile_name, strerror(errno)); param.datafile_name, strerror(errno));
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment