From cc8bdba328bae20c2d71bd974d99ec413bd3f025 Mon Sep 17 00:00:00 2001 From: Fredrik Noring <noring@nocrew.org> Date: Sat, 23 May 1998 21:51:26 +0200 Subject: [PATCH] The Yabu database is now included in Pike. Rev: lib/modules/Yabu.pmod/module.pmod:1.1 --- .gitattributes | 1 + lib/modules/Yabu.pmod/module.pmod | 943 ++++++++++++++++++++++++++++++ 2 files changed, 944 insertions(+) create mode 100644 lib/modules/Yabu.pmod/module.pmod diff --git a/.gitattributes b/.gitattributes index d1a52fb07c..8a12202105 100644 --- a/.gitattributes +++ b/.gitattributes @@ -70,6 +70,7 @@ testfont binary /lib/modules/Sql.pmod/sql.pike foreign_ident /lib/modules/Sql.pmod/sql_result.pike foreign_ident /lib/modules/Standards.pmod/ASN1.pmod/decode.pike foreign_ident +/lib/modules/Yabu.pmod/module.pmod foreign_ident /lib/modules/error.pmod foreign_ident /man/hilfe.1 foreign_ident /man/pike.1 foreign_ident diff --git a/lib/modules/Yabu.pmod/module.pmod b/lib/modules/Yabu.pmod/module.pmod new file mode 100644 index 0000000000..d4523d9722 --- /dev/null +++ b/lib/modules/Yabu.pmod/module.pmod @@ -0,0 +1,943 @@ +// Yabu by Fredrik Noring +// $Id: module.pmod,v 1.1 1998/05/23 19:51:26 noring Exp $ + +#if constant(thread_create) +#define THREAD_SAFE +#define LOCK() do { object key; catch(key=lock()) +#define UNLOCK() key=0; } while(0) +#else +#undef THREAD_SAFE +#define LOCK() do { +#define UNLOCK() } while(0) +#endif + +#define ERR(msg) throw(({ "(Yabu) "+msg+"\n", backtrace() })) +#define WARN(msg) werror(msg) +#define DEB(msg) /* werror(msg) */ +#define CHECKSUM(s) (hash(s) & 0xffffffff) + +static private class FileIO { + static private inherit Stdio.File:file; + + static private void seek(int offset) + { + if(file::seek(offset) == -1) + ERR("seek failed"); + } + + string read_at(int offset, int|void size) + { + seek(offset); + mixed s = size?file::read(size):file::read(); + if(!stringp(s)) + ERR("read failed"); + return s; + } + + void write_at(int offset, string s) + { + seek(offset); + while(sizeof(s)) { + int n = file::write(s); + if(n < 0 && !(<11,12,16,24,28,49>)[file::errno()]) + ERR(strerror(file::errno())); + if(n == sizeof(s)) + break; + s = s[n..]; + if(n<0) + WARN(strerror(file::errno())+" (sleeping)"); + else + WARN("disk seems to be full. (sleeping)"); + sleep(1); + } + } + + void create(string filename, string mode) + { + file::create(); + if(!file::open(filename, mode)) + ERR(strerror(file::errno())); + } +} + +class Chunk { + static private inherit FileIO:file; + + static private object parent; + static private int magic, compress, write; + static private string start_time, filename; + + static private int eof = 0; + static private mapping frees = ([]), keys = ([]); + + static private string escape(string s) + { + return replace(s, ({ "\n", "%" }), ({ "%n", "%p" })); + } + + static private string descape(string s) + { + return replace(s, ({ "%n", "%p" }), ({ "\n", "%" })); + } + + static private string encode_num(int x) + { + return sprintf("%08x", x); + } + + static private int decode_num(string s, int offset) + { + int x; + if(sizeof(s) < offset+8) + ERR("cunk short"); + if(sscanf(s[offset..offset+7], "%x", x) != 1) + ERR("cunk number decode error"); + return x; + } + + static private string encode_key(int offset, int type) + { + return encode_num(offset)+":"+encode_num(type); + } + +#define DECODE_KEY(key, offset, type) \ + { if(sizeof(key) != 17 || sscanf(key, "%x:%x", offset, type) != 2) \ + ERR("Key not valid"); } + + static private string next_magic() + { + return start_time + encode_num(magic++); + } + + static private int find_nearest_2x(int num) + { + for(int b=4;b<32;b++) if((1<<b) >= num) return (1<<b); + } + + static private mapping null_chunk(int t) + { + string entry = ""; + string magic = next_magic(); + string type = encode_num(t); + string checksum = encode_num(CHECKSUM(entry)); + string size = encode_num(sizeof(entry)); + + return ([ "type":t, + "entry":("\n"+magic+type+checksum+size+entry+"%m"+magic)]); + } + + static private mapping encode_chunk(mixed x) + { + string entry = encode_value(x); +#if constant(Gz.inflate) + if(compress) + catch { entry = Gz.deflate()->deflate(entry); }; +#endif + string entry = escape(entry); + string magic = next_magic(); + string checksum = encode_num(CHECKSUM(entry)); + string size = encode_num(sizeof(entry)); + + entry = checksum+size+entry+"%m"+magic; + int t = find_nearest_2x(1+16+8+sizeof(entry)); + string type = encode_num(t); + entry = "\n"+magic+type+entry; + return ([ "type":t, "entry":entry ]); + } + + static private mapping decode_chunk(string chunk) + { + mapping m = ([]); + + if(chunk[0] != '\n') + ERR("No chunk start"); + chunk = chunk[1..]; + m->magic = chunk[0..15]; + m->type = decode_num(chunk, 16); + m->checksum = decode_num(chunk, 24); + m->size = decode_num(chunk, 32); + chunk = chunk[40..]; + + m->entry = chunk[0..m->size-1]; + if(sizeof(m->entry) < m->size) + ERR("Chunk too small"); + if(CHECKSUM(m->entry) != m->checksum) + ERR("Chunk checksum error"); + if(chunk[m->size..m->size+1] != "%m") + ERR("No magic"); + if(m->magic != chunk[m->size+2..m->size+17]) + ERR("Magic diff"); + + if(m->size) { + m->entry = descape(m->entry); +#if constant(Gz.inflate) + catch { m->entry = Gz.inflate()->inflate(m->entry); }; +#endif + m->entry = decode_value(m->entry); + } + + return m; + } + + static private int allocate_chunk(int type, mapping m) + { + array f; + if(f = frees[type]) { + if(sizeof(f) > 1) + frees[type] = f[1..]; + else + m_delete(frees, type); + return f[0]; + } + int x = eof; + eof += type; + return x; + } + + array(string) list_keys() + { + return indices(keys); + } + + string set(mixed x) + { + mapping m = encode_chunk(x); + int offset = allocate_chunk(m->type, m); + string key = encode_key(offset, m->type); + file::write_at(offset, m->entry); + keys[key] = offset; + return key; + } + + mixed get(string key, void|int attributes) + { + if(!attributes && zero_type(keys[key])) + ERR(sprintf("Unknown key '%O'", key)); + + int offset, type; + DECODE_KEY(key, offset, type); + + mapping m = decode_chunk(file::read_at(offset, type)); + if(m->size == 0) + ERR("Cannot decode free chunk!"); + if(attributes) + return m; + return m->entry; + } + + void free(string key) + { + if(zero_type(keys[key])) + ERR(sprintf("Unknown key '%O'", key)); + + int offset, type; + DECODE_KEY(key, offset, type); + + m_delete(keys, key); + file::write_at(offset, null_chunk(type)->entry); + frees[type] = (frees[type]||({})) | ({ offset }); + } + + mapping state(array|void almost_free) + { + mapping m = copy_value(frees); + foreach(almost_free||({}), string key) { + int offset, type; + DECODE_KEY(key, offset, type); + m[type] = (m[type]||({})) | ({ offset }); + } + return copy_value(([ "eof":eof, "keys":keys, "frees":m ])); + } + + void purge() + { + if(!write) + ERR("Cannot purge in read mode"); + rm(filename); + destruct(this_object()); + } + + void destroy() + { + if(parent) + destruct(parent); + } + + void create(string filename_in, string mode, + object|void parent_in, mapping|void m) + { + magic = 0; + filename = filename_in; + parent = parent_in; + start_time = encode_num(time()); + + if(search(mode, "C")+1) + compress = 1; + if(search(mode, "w")+1) { + write = 1; + file::create(filename, "cwr"); + } else + file::create(filename, "r"); + + if(m) { + eof = m->eof; + keys = m->keys||([]); + frees = m->frees||([]); + } else { + string s; + array offsets = ({}); + int n, offset = 0; + while(sizeof(s = file::read_at(offset, 4096))) { + n = -1; + while((n = search(s, "\n", n+1)) >= 0) + offsets += ({ offset+n }); + offset += 4096; + } + + eof = 0; + for(int i = 0; i < sizeof(offsets); i++) { + int offset = offsets[i]; + int size = (i+1 < sizeof(offsets)?offsets[i+1]-offset:0); + string key = encode_key(offset, size); + if(!size || size == find_nearest_2x(size)) { + mapping m; + if(catch(m = get(key, 1))) { + if(size) { + frees[size] = (frees[size]||({})) + ({ offset }); + eof = offset+size; + } + } else { + keys[encode_key(offset, m->type)] = offset; + eof = offset+m->type; + } + } + } + } + } +} + +class Transaction { + static private int id; + static private object table, keep_ref; + + void sync() + { + table->sync(); + } + + void destroy() + { + if(table) + table->t_destroy(id); + } + + mixed set(string handle, mixed x) + { + return table->t_set(id, handle, x); + } + + mixed get(string handle) + { + return table->t_get(id, handle); + } + + mixed delete(string handle) + { + return table->t_delete(id, handle); + } + + array list_keys() + { + return table->t_list_keys(id); + } + + void commit() + { + table->t_commit(id); + } + + void rollback() + { + table->t_rollback(id); + } + + mixed `[]=(string handle, mixed x) + { + return set(handle, x); + } + + mixed `[](string handle) + { + return get(handle); + } + + array _indices() + { + return list_keys(); + } + + void create(object table_in, int id_in, object keep_ref_in) + { + table = table_in; + id = id_in; + keep_ref = keep_ref_in; + } +} + +class Table { +#ifdef THREAD_SAFE + static inherit Thread.Mutex; +#endif + static private object index, db; + + static private string filename; + static private mapping handles, new, changes; + static private mapping t_start, t_changes, t_handles, t_deleted; + static private int sync_timeout, write, dirty, magic, id = 0x314159; + + void sync() + { + if(!write || !dirty) return; + + LOCK(); + new = ([]); + array almost_free = db->list_keys() - values(handles); + string key = index->set(([ "db":db->state(almost_free), + "handles":handles ])); + + foreach(index->list_keys() - ({ key }), string k) + index->free(k); + + array working = ({}); + if(t_handles) + foreach(indices(t_handles), int id) + working += values(t_handles[id]); + foreach(almost_free - working, string k) + db->free(k); + + dirty = sizeof(working); + UNLOCK(); + } + + static private int next_magic() + { + return magic++; + } + + static private void modified() + { + dirty++; + if(sync_timeout && dirty >= sync_timeout) + sync(); + } + + static private mixed _set(string handle, mixed x, + mapping handles, mapping new) + { + if(!write) ERR("Cannot set in read mode"); + + string key = handles[handle]; + if(key && new[key]) { + m_delete(new, key); + db->free(key); + } + handles[handle] = key = db->set(([ "handle":handle, "entry":x ])); + new[key] = 1; + modified(); + return x; + } + + static private mixed _get(string handle, mapping handles) + { + if(!handles[handle]) + return 0; + return db->get(handles[handle])->entry; + } + + void delete(string handle) + { + if(!write) ERR("Cannot delete in read mode"); + + LOCK(); + if(!handles[handle]) ERR(sprintf("Unknown handle '%O'", handle)); + + string key = handles[handle]; + m_delete(handles, handle); + if(changes) + changes[handle] = next_magic(); + // m_delete(changes, handle); + if(new[key]) { + m_delete(new, key); + db->free(key); + } + modified(); + UNLOCK(); + } + + mixed set(string handle, mixed x) + { + LOCK(); + if(changes) + changes[handle] = next_magic(); + return _set(handle, x, handles, new); + UNLOCK(); + } + + mixed get(string handle) + { + LOCK(); + return _get(handle, handles); + UNLOCK(); + } + + // + // Transactions + // + mixed t_set(int id, string handle, mixed x) + { + if(!t_handles[id]) ERR("Unknown transaction id"); + + LOCK(); + t_changes[id][handle] = t_start[id]; + mapping t_new = mkmapping(values(t_handles[id]), indices(t_handles[id])); + return _set(handle, x, t_handles[id], t_new); + UNLOCK(); + } + + mixed t_get(int id, string handle) + { + if(!t_handles[id]) ERR("Unknown transaction id"); + + LOCK(); + t_changes[id][handle] = t_start[id]; + if(t_deleted[id][handle]) + return 0; + return _get(handle, t_handles[id][handle]?t_handles[id]:handles); + UNLOCK(); + } + + void t_delete(int id, string handle) + { + if(!t_handles[id]) ERR("Unknown transaction id"); + + LOCK(); + t_deleted[id][handle] = 1; + t_changes[id][handle] = t_start[id]; + if(t_handles[id][handle]) { + db->free(t_handles[id][handle]); + m_delete(t_handles[id], handle); + } + UNLOCK(); + } + + void t_commit(int id) + { + if(!write) ERR("Cannot commit in read mode"); + if(!t_handles[id]) ERR("Unknown transaction id"); + + LOCK(); + foreach(indices(t_changes[id]), string handle) + if(t_changes[id][handle] < changes[handle]) + ERR("Transaction conflict"); + + mapping tmp_t_changes = copy_value(t_changes); + mapping tmp_t_handles = copy_value(t_handles); + t_start[id] = next_magic(); + t_changes[id] = ([]); + t_handles[id] = ([]); + t_deleted[id] = ([]); + + array obsolete = ({}); + foreach(indices(tmp_t_changes[id]), string handle) { + if(new[handles[handle]]) + obsolete += ({ handles[handle] }); + changes[handle] = next_magic(); + handles[handle] = tmp_t_handles[id][handle]; + } + + foreach(obsolete, string key) { + m_delete(new, key); + db->free(key); + } + UNLOCK(); + } + + void t_rollback(int id) + { + if(!t_handles[id]) ERR("Unknown transaction id"); + + LOCK(); + array keys = values(t_handles[id]); + t_start[id] = next_magic(); + t_changes[id] = ([]); + t_handles[id] = ([]); + t_deleted[id] = ([]); + foreach(keys, string key) + db->free(key); + UNLOCK(); + } + + array t_list_keys(int id) + { + if(!t_handles[id]) ERR("Unknown transaction id"); + + LOCK(); + return Array.uniq(indices(handles) + indices(t_handles[id])); + UNLOCK(); + } + + void t_destroy(int id) + { + if(!t_handles[id]) ERR("Unknown transaction id"); + + t_rollback(id); + LOCK(); + m_delete(t_start, id); + m_delete(t_changes, id); + m_delete(t_handles, id); + m_delete(t_deleted, id); + UNLOCK(); + } + + object transaction(object|void keep_ref) + { + if(!changes) ERR("Transactions are not enabled"); + + LOCK(); + id++; + t_start[id] = next_magic(); + t_changes[id] = ([]); + t_handles[id] = ([]); + t_deleted[id] = ([]); + UNLOCK(); + return Transaction(this_object(), id, keep_ref); + } + // + // + // + + array list_keys() + { + LOCK(); + return indices(handles); + UNLOCK(); + } + + // + // Interface functions + // + + void sync_schedule() + { + sync(); + call_out(sync_schedule, 120); + } + + mixed `[]=(string handle, mixed x) + { + return set(handle, x); + } + + mixed `[](string handle) + { + return get(handle); + } + + void destroy() + { + sync(); + } + + void _destroy() + { + write = 0; + destruct(this_object()); + } + + void close() + { + sync(); + _destroy(); + } + + void purge() + { + if(!write) ERR("Cannot purge in read mode"); + + LOCK(); + write = 0; + rm(filename+".inx"); + rm(filename+".chk"); + destruct(this_object()); + UNLOCK(); + } + + array _indices() + { + return list_keys(); + } + + void create(string filename_in, string mode) + { + filename = filename_in; + + if(search(mode, "w")+1) + write = 1; + if(search(mode, "t")+1) { + changes = ([]); + t_start = ([]); + t_changes = ([]); + t_handles = ([]); + t_deleted = ([]); + } + + index = Chunk(filename+".inx", mode, this_object()); + mapping m = ([]); + if(sizeof(index->list_keys())) + m = index->get(sort(index->list_keys())[-1]); + handles = m->handles||([]); + new = ([]); + db = Chunk(filename+".chk", mode, this_object(), m->db||([])); + + if(write && search(mode, "s")+1) { + sync_timeout = 32; + sync_schedule(); + } + } +} + +class _Table { + static string handle; + static object table, parent; + + void sync() { table->sync(); } + void delete(string handle) { table->delete(handle); } + mixed set(string handle, mixed x) { return table->set(handle, x); } + mixed get(string handle) { return table->get(handle); } + array list_keys() { return table->list_keys(); } + object transaction() { return table->transaction(this_object()); } + void close() { destruct(this_object()); } + void purge() { table->purge(); } + + mixed `[]=(string handle, mixed x) + { + return set(handle, x); + } + + mixed `[](string handle) + { + return get(handle); + } + + array _indices() + { + return list_keys(); + } + + void destroy() + { + if(parent) + parent->_table_destroyed(handle); + } + + void create(string _handle, object _table, object _parent) + { + handle = _handle; + table = _table; + parent = _parent; + } +} + +class db { +#ifdef THREAD_SAFE + static inherit Thread.Mutex; +#endif + + static string dir, mode; + static mapping tables = ([]), table_refs = ([]); + static int write, id; + + void sync() + { + LOCK(); + foreach(values(tables), object o) + if(o) + o->sync(); + UNLOCK(); + } + + object table(string handle) + { + LOCK(); + if(!tables[handle]) + tables[handle] = Table(dir+"/"+handle, mode); + table_refs[handle]++; + // DEB(sprintf("### refs[%s]++ = %d\n", handle, table_refs[handle])); + DEB(sprintf("# tables '%s': %d (%s)\n", + reverse(reverse(dir/"/")[0..1])*"/", sizeof(tables), handle)); + return _Table(handle, tables[handle], this_object()); + UNLOCK(); + } + + void _table_destroyed(string handle) + { + LOCK(); + table_refs[handle]--; + // DEB(sprintf("### refs[%s]-- = %d\n", handle, table_refs[handle])); + DEB(sprintf("! tables '%s': %d (%s)\n", + reverse(reverse(dir/"/")[0..1])*"/", sizeof(tables), handle)); + if(!table_refs[handle]) { + // DEB(sprintf("### zonking[%s]\n", handle)); + destruct(tables[handle]); + m_delete(tables, handle); + m_delete(table_refs, handle); + } + UNLOCK(); + } + + mixed `[](string handle) + { + return table(handle); + } + + array list_tables() + { + LOCK(); + return Array.map(glob("*.chk", get_dir(dir)||({})), + lambda(string s) { return s[0..sizeof(s)-5]; }); + UNLOCK(); + } + + array _indices() + { + return list_tables(); + } + + // Remove maximum one level of directories and files + static private void level2_rm(string f) + { + if(sizeof(f) > 1 && f[-1] == '/') + f = f[0..sizeof(f)-2]; // remove /'s + if((file_stat(f)||({0,0}))[1] == -2) // directory + foreach(get_dir(f)||({}), string file) + rm(f+"/"+file); // delete file + rm(f); // delete file/directory + } + + void purge() + { + LOCK(); + foreach(values(tables), object o) + if(o) + destruct(o); + level2_rm(dir); + destruct(this_object()); + UNLOCK(); + } + + static void mkdirhier(string from) + { + string a, b; + array f; + + f=(from/"/"); + b=""; + + foreach(f[0..sizeof(f)-2], a) + { + mkdir(b+a); + b+=a+"/"; + } + } + + void create(string dir_in, string mode_in) + { + dir = dir_in; + mode = mode_in; + + if(search(mode, "w")+1) { + write = 1; + mkdirhier(dir+"/"); + } + } +} + +class LookupTable { +#ifdef THREAD_SAFE + static inherit Thread.Mutex; +#endif + static private int minx; + static private object table; + + static private string h(string s) + { + return (string)(hash(s) & minx); + } + + mixed get(string handle) + { + LOCK(); + return (table->get(h(handle))||([]))[handle]; + UNLOCK(); + } + + mixed set(string handle, mixed x) + { + LOCK(); + mapping m = table->get(h(handle))||([]); + m[handle] = x; + return table->set(h(handle), m)[handle]; + UNLOCK(); + } + + mixed `[]=(string handle, mixed x) + { + return set(handle, x); + } + + mixed `[](string handle) + { + return get(handle); + } + + void purge() + { + LOCK(); + table->purge(); + UNLOCK(); + } + + void sync() + { + LOCK(); + table->sync(); + UNLOCK(); + } + + void create(string filename, string mode, int _minx) + { + minx = _minx; + table = Table(filename, mode); + } +} + +class lookup { + inherit db; + static private int minx; + + object table(string handle) + { + LOCK(); + return (tables[handle] = + tables[handle]||LookupTable(dir+"/"+handle, mode, minx)); + UNLOCK(); + } + + void create(string dir, string mode, mapping|void opt) + { + ::create(dir, (mode-"t")); + minx = (opt||([]))->index_size || 0x7ff; + if(!sscanf(Stdio.read_bytes(dir+"/hs")||"", "%4c", minx)) + Stdio.write_file(dir+"/hs", sprintf("%4c", minx)); + } +} -- GitLab