initial checkin

src/server/cache-database.c

+/*
+ * cache-database.c
+ *
+ * cache: Database low-level driver
+ *
+ * Pontus Hagland 1993-10-07
+ *
+ */
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+#include <stdlib.h>
+#include <time.h>
+#include "cache-database.h"
+
+/* misc. defines *********************************************/
+
+#define smalloc malloc
+#define sfree free
+#define srealloc realloc
+
+#define TIME (time(NULL))
+#define SAFESTORAGETIME 30 /* seconds to regen blocks */
+#define cb_addsafestoragetime(x) ((x)+SAFESTORAGETIME)
+
+
+#define MAXRECORDSINBLOCK 32
+   /* no of records in a block is max 32 */
+
+
+#define DATASIZE(cb) ((cb)->header.blocksize- \
+		      sizeof(struct cb_blockheader)- \
+		      DEFEATSIZE)
+	/* total size of records in a block */
+
+#define REFRESHTRIGGER(cb) (DATASIZE(cb)/3)
+	/* trigg refresh of block when unused has reached 1/3 */
+
+#define BLOCKPOSITION(cb,no) \
+        (((no)-0)*(cb)->header.blocksize+ \
+	 /*sizeof(struct cachebaseheader)+DEFEATSIZE*/0) /* let's block-align */
+
+/* misc. declarations ****************************************/
+
+struct cb_blockheader
+{
+   unsigned long generation; /* 0 if clear, othervise counting upwards */
+   time_t ts_created;
+   time_t ts_written;
+   unsigned long index[MAXRECORDSINBLOCK];
+};
+
+struct cb_recordheader
+{
+   unsigned long no;
+   unsigned long generation; /* 0 if clear, othervise counting upwards,
+				serial following record number */
+   time_t ts_created;
+   unsigned int size;
+};
+
+struct cb_blocklist
+{
+   unsigned long no;
+   time_t early;
+   unsigned long used;
+   unsigned long records;
+   struct cb_blocklist*next;
+};
+
+static void cb_addtorefreshlist(struct cachebase *cb,unsigned long no,
+		     time_t since,unsigned long used,unsigned long records);
+static void cb_addtofreelist(struct cachebase *cb,unsigned long no);
+static unsigned long
+   cb_search(struct cachebase *cb,unsigned long no);
+static void
+   cb_abandon(struct cachebase *cb,long bn,long no);
+static unsigned long
+   cb_getgeneration(struct cachebase *cb,unsigned long bn,
+		    unsigned long no);
+
+/* ugly we-don't-link-with-others-right-now functions ********/
+
+void log(char *format,...)
+{
+   va_list va;
+   time_t t;
+   struct tm *tm;
+   t=TIME;
+   tm=localtime(&t);
+   printf("%02d-%02d-%02d %2d:%02d:%02d ",
+	  tm->tm_year,tm->tm_mon+1,tm->tm_mday,
+	  tm->tm_hour,tm->tm_min,tm->tm_sec);
+   va_start(va,format);
+   vprintf(format,va);
+   va_end(va);
+   fflush(stdout);
+}
+
+void restart_kom(char *format,...)
+{
+   va_list va;
+   time_t t;
+   struct tm*tm;
+   t=TIME;
+   tm=localtime(&t);
+   printf("%02d-%02d-%02d %2d:%02d:%02d ",
+	  tm->tm_year,tm->tm_mon+1,tm->tm_mday,
+	  tm->tm_hour,tm->tm_min,tm->tm_sec);
+   va_start(va,format);
+   vprintf(format,va);
+   va_end(va);
+   printf("(exit! exit! exit! abandon! alien intruder detected! self destructing!)\n");
+   exit(0);
+}
+
+/* internal functions ****************************************/
+
+static unsigned long cb_sizeoflist(struct cb_blocklist *cbb)
+{
+   unsigned long i;
+   i=0;
+   while (cbb) { i++; cbb=cbb->next; }
+   return i;
+}
+
+static struct cachebase *
+   cb_recoverbase(char *filename,char *indexname)
+{
+   struct cachebase *cb;
+
+   unsigned long bn;
+   unsigned long busy;
+   unsigned long index;
+   struct cb_blockheader bh;
+   struct cb_recordheader rh;
+   int i,c;
+   unsigned long records,brecords,totsz,erecords;
+   unsigned long rbn,foo;
+   
+   cb=smalloc(sizeof(struct cachebase));
+   if (!cb) { restart_kom("cb_recoverbase: out of memory\n"); return NULL; }
+   
+   cb->fp=fopen(filename,"r+b");
+   if (!cb->fp) { sfree(cb); return NULL; }
+   
+   log("cache: recovering database '%s' (recreating index '%s'):\n",filename,indexname);
+
+   fseek(cb->fp,0,0);
+   fread(&(cb->header),sizeof(cb->header),1,cb->fp);
+
+   if (cb->header.hex01234567!=0x01234567||
+       cb->header.hex89abcdef!=0x89abcdef||
+       cb->header.sizeofint!=sizeof(int)||
+       cb->header.sizeoflong!=sizeof(long))
+   {
+      log("cache: can't use database '%s': incompatible architecture "
+	  "(x%lx,x%lx,x%lx,x%lx!=x%lx,x%lx,x%lx,x%lx)\n",
+	  filename,cb->header.hex01234567,cb->header.hex89abcdef,
+	  cb->header.sizeofint,cb->header.sizeoflong,
+	  0x01234567,0x89abcdef,sizeof(int),sizeof(long));
+      sfree(cb);
+      return NULL;
+   }
+   
+   cb->filename=filename;
+   cb->indexname=indexname;
+   cb->freelist=NULL;
+   cb->regenlist=NULL;
+   cb->writeonblock=0;
+   
+   cb->ifp=fopen(indexname,"w+b");
+   if (!cb->ifp)
+   {
+      log("cache: !failure, can't create index '%s'\n",indexname);
+      fclose(cb->fp);
+      sfree(cb);
+      return NULL;
+   }
+
+   records=0;
+   brecords=0;
+   erecords=0;
+   totsz=0;
+   for (bn=1; ;bn++)
+   {
+      fseek(cb->fp,BLOCKPOSITION(cb,bn),0);
+      if (!fread(&bh,sizeof(struct cb_blockheader),1,cb->fp)) break;
+      if (bh.generation==0)
+      {
+	 cb_addtofreelist(cb,bn);
+      }
+      else
+      {
+	 busy=0;
+	 index=sizeof(struct cb_blockheader);
+	 for (i=c=0; bh.index[i]&&i<MAXRECORDSINBLOCK&&
+	             index<cb->header.blocksize; i++)
+	 {
+	    fseek(cb->fp,BLOCKPOSITION(cb,bn)+index,0);
+	    fread(&rh,sizeof(struct cb_recordheader),1,cb->fp);
+	    if (bh.index[i]!=~0&&bh.index[i]==rh.no)
+	    {
+	       if (rbn=cb_search(cb,rh.no))
+	       {
+		  if (cb_getgeneration(cb,rbn,rh.no)<rh.generation)
+		  {
+		     cb_abandon(cb,rbn,rh.no);
+		     fseek(cb->ifp,rh.no*sizeof(unsigned long),0);
+		     fwrite(&bn,sizeof(unsigned long),1,cb->ifp);
+		     brecords++; records--;
+		  }
+	       }
+	       else
+	       {
+		  fseek(cb->ifp,rh.no*sizeof(unsigned long),0);
+		  fwrite(&bn,sizeof(unsigned long),1,cb->ifp);
+	       }
+	       busy+=rh.size;
+	       if (rh.size==0) erecords++;
+	       records++;
+	       c++;
+	       if (cb->header.highnumber<rh.no)
+	          cb->header.highnumber=rh.no;
+	    }
+	    index+=rh.size+sizeof(struct cb_recordheader);
+	    if (bh.index[i]!=rh.no&&bh.index[i]!=~0)
+	    {
+	       log("        block %lu found trashed\n",bn);
+	       for (;i<MAXRECORDSINBLOCK;i++)
+	          bh.index[i]=~0;
+	       fseek(cb->fp,BLOCKPOSITION(cb,bn),0);
+	       if (!fwrite(&bh,sizeof(struct cb_blockheader),1,cb->fp))
+	       {
+		  log("cache: !can't update trashed block (%lu)\n",bn);
+		  fclose(cb->fp); fclose(cb->ifp); sfree(cb);
+		  return NULL;
+	       }
+	    }
+	 }
+	 totsz+=busy;
+	 if (busy<REFRESHTRIGGER(cb))
+	   cb_addtorefreshlist(cb,bn,0,busy,c);
+      }
+   }
+   cb->header.highblock=bn-1;
+   cb->active_records=records;
+
+   log("       database recovering finished:\n");
+   log("        high block.........%-5lu  used blocks.........%-5lu\n",
+       cb->header.highblock,
+       cb->header.highblock+1-cb_sizeoflist(cb->freelist));
+   log("        blocks to refresh..%-5lu  free blocks.........%-5lu\n",
+       cb_sizeoflist(cb->regenlist),
+       cb_sizeoflist(cb->freelist));
+   log("        high record........%-5lu  used records........%-5lu\n",
+       cb->header.highnumber,records);
+   log("        empty records......%-5lu  refreshed records...%-5lu\n",
+       erecords,brecords);
+   log("        database size:%lub  size of data:%lub\n",
+       BLOCKPOSITION(cb,bn),totsz);
+   return cb;
+}
+
+static struct cachebase *
+   cb_createbase(char *filename,char *indexname,unsigned int blocksize)
+{
+   struct cachebase *cb;
+   log("cache: creating missing database '%s':\n",filename);
+   cb=smalloc(sizeof(struct cachebase));
+   if (!cb) { restart_kom("cache: cb_createbase: out of memory\n"); return NULL; }
+
+   cb->fp=fopen(filename,"rb");
+   if (cb->fp)
+   {
+      log("cache: !database do exists, refusing to recreate ('%s')\n",
+	  filename);
+      fclose(cb->fp);
+      sfree(cb);
+      return NULL;
+   }
+
+   cb->ifp=fopen(indexname,"w+b");
+   if (!cb->ifp)
+   {
+      log("cache: !failure: can't create index file '%s'\n",indexname);
+      sfree(cb);
+      return NULL;
+   }
+   log("cache: index '%s' created, ok\n",indexname);
+
+   cb->fp=fopen(filename,"w+b");
+   if (!cb->fp)
+   {
+      log("cache: !failure: can't create database file\n");
+      fclose(cb->ifp);
+      sfree(cb);
+      return NULL;
+   }
+   cb->filename=filename;
+   cb->indexname=indexname;
+   cb->freelist=NULL;
+   cb->regenlist=NULL;
+   cb->writeonblock=0;
+   cb->active_records=0;
+
+   memset(&cb->header,0,sizeof(struct cachebaseheader));
+
+   cb->header.blocksize=blocksize;
+   cb->header.highblock=0;
+   cb->header.highnumber=0;
+   cb->header.ts_written=
+     cb->header.ts_created=TIME;
+   cb->header.blockgeneration=0;
+
+   cb->header.hex01234567=0x01234567;
+   cb->header.hex89abcdef=0x89abcdef;
+   cb->header.sizeofint=sizeof(int);
+   cb->header.sizeoflong=sizeof(long);
+
+   fseek(cb->fp,0,0);
+   if (fwrite(&(cb->header),sizeof(struct cachebaseheader),1,cb->fp)<1)
+   {
+      log("cache: !failure: can't write header\n");
+      fclose(cb->ifp);
+      fclose(cb->fp);
+      sfree(cb);
+      return NULL;
+   }
+   fwrite(DEFEATSTRING,DEFEATSIZE,1,cb->fp);
+   log("       initialized with 0 block, high number=0, ok\n");
+   return cb;
+}
+
+static void cb_addtorefreshlist(struct cachebase *cb,unsigned long no,
+		     time_t since,unsigned long used,unsigned long records)
+{
+   struct cb_blocklist *bl;
+   if (records>MAXRECORDSINBLOCK-5) return;
+   bl=cb->regenlist;
+   while (bl)
+   {
+      if (bl->no==no)
+      {
+	 bl->used=used;
+	 bl->records=records;
+   log("debug: block %lu updated in regenlist (used: %lu, records: %lu)\n",
+       no,used,records);
+	 return;
+      }
+      bl=bl->next;
+   }
+   bl=smalloc(sizeof(struct cb_blocklist));
+   if (!bl) { restart_kom("cache: cb_addtorefreshlist: out of memory\n"); return; }
+   bl->no=no;
+   if (!since) bl->early=TIME; /* may be changed if time_t != integer */
+   else bl->early=cb_addsafestoragetime(since);
+   bl->used=used;
+   bl->records=records;
+   bl->next=cb->regenlist;
+   cb->regenlist=bl;
+   log("debug: block %lu added to regenlist (used: %lu, records: %lu)\n",
+       no,used,records);
+}
+
+static void cb_addtofreelist(struct cachebase *cb,unsigned long no)
+{
+   struct cb_blocklist *bl;
+   bl=cb->freelist;
+   while (bl)
+   {
+      if (bl->no==no) return;
+      bl=bl->next;
+   }
+   bl=smalloc(sizeof(struct cb_blocklist));
+   bl->no=no;
+   bl->next=cb->freelist;
+   cb->freelist=bl;
+}
+
+static unsigned long
+   cb_search(struct cachebase *cb,unsigned long no)
+{
+   unsigned long pos;
+   pos=0;
+   if (no>cb->header.highnumber) return 0;
+   fseek(cb->ifp,sizeof(unsigned long)*no,0);
+   fread(&pos,sizeof(unsigned long),1,cb->ifp);
+   return pos;
+}
+
+static unsigned long
+   cb_getnewblock(struct cachebase *cb,struct cb_blockheader *bh)
+{
+   struct cb_blocklist *bl;
+   unsigned long n;
+   int i;
+   
+   if (cb->freelist)
+   {
+      bl=cb->freelist;
+      n=bl->no;
+      cb->freelist=bl->next;
+      sfree(bl);
+   }
+   else
+   {
+      n=++(cb->header.highblock);
+   }
+   bh->generation=++cb->header.blockgeneration;
+   bh->ts_created=bh->ts_created=TIME;
+   for (i=0; i<MAXRECORDSINBLOCK; i++) bh->index[i]=0;
+   
+   fseek(cb->fp,0,0);
+   cb->header.ts_written=TIME;
+   if (!fwrite(&cb->header,sizeof(struct cachebaseheader),1,cb->fp))
+   {
+      restart_kom("cache: !write to base header failed ('%s')\n",cb->filename);
+      return 0;
+   }
+   cb->writeonblock=n;
+   return n;
+}
+
+static unsigned long
+   cb_writenew(struct cachebase *cb,unsigned long no,void *buffert,
+	       unsigned int size,unsigned long lastgeneration)
+{
+   struct cb_blockheader bh;
+   struct cb_recordheader rh;
+   unsigned long bn,pos;
+   unsigned long space,index;
+   int i;
+   
+   /* if (size+sizeof(struct cb_recordheader)>DATASIZE(fp)) ... */
+
+   bn=cb->writeonblock;
+   if (bn)
+   {
+      fseek(cb->fp,BLOCKPOSITION(cb,bn),0);
+      fread(&bh,sizeof(struct cb_blockheader),1,cb->fp);
+      space=DATASIZE(cb);
+      index=sizeof(struct cb_blockheader);
+
+      for (i=0; i<MAXRECORDSINBLOCK&&bh.index[i]; i++)
+      {
+	 fseek(cb->fp,BLOCKPOSITION(cb,bn)+index,0);
+	 /* optimera fseek'en till att ska framt */
+	 fread(&rh,sizeof(struct cb_recordheader),1,cb->fp);
+	 space-=rh.size+sizeof(struct cb_recordheader);
+	 index+=rh.size+sizeof(struct cb_recordheader);
+      }
+      if (i==MAXRECORDSINBLOCK) bn=0;
+      else if (space<sizeof(struct cb_recordheader)+size)
+      {
+	 if (DATASIZE(cb)-space<REFRESHTRIGGER(cb))
+	    cb_addtorefreshlist(cb,bn,bh.ts_written,DATASIZE(cb)-space,i);
+	 bn=0;
+      }
+
+      for (i=0; i<MAXRECORDSINBLOCK&&bh.index[i]; i++)
+         if (bh.index[i]==no) { bn=0; break; }
+      /* do not store the same record twice in a block */ 
+   }
+   if (!bn)
+   if (bn=cb_getnewblock(cb,&bh))
+   {
+      i=0; /* next record==0 */
+      index=sizeof(struct cb_blockheader); /* write at this position */
+   }
+   else return 0; /* error! (probably restart_kom already done) */
+   rh.generation=lastgeneration+1;
+   rh.no=no;
+   rh.size=size;
+   rh.ts_created=TIME;
+   fseek(cb->fp,BLOCKPOSITION(cb,bn)+index,0);
+   if (!fwrite(&rh,sizeof(struct cb_recordheader),1,cb->fp))
+      { restart_kom("cache: Can't write recordheader, r%ib%i, base='%s'\n",no,bn,cb->filename); return 0; }
+
+   if (size)
+    if (!fwrite(buffert,size,1,cb->fp))
+      { restart_kom("cache: Can't write record data, r%ib%i, base='%s'\n",no,bn,cb->filename); return 0; }
+   fseek(cb->fp,BLOCKPOSITION(cb,bn),0);
+   bh.index[i]=no;
+   bh.ts_written=TIME;
+   if (!fwrite(&bh,sizeof(struct cb_blockheader),1,cb->fp))
+      { restart_kom("cache: Can't write block header, r%i to b%i, base='%s'\n",no,bn,cb->filename); return 0; }
+
+   fseek(cb->ifp,sizeof(unsigned long)*no,0);
+   if (!fwrite(&bn,sizeof(unsigned long),1,cb->ifp))
+      { restart_kom("cache: Can't write record index, r%i to b%i, base='%s', index='%s'\n",no,bn,cb->filename,cb->indexname); return 0; }
+
+log("debug: W: rec %lu as #%lu in bl %lu (pos %lu)\n",no,i,bn,BLOCKPOSITION(cb,bn)+sizeof(struct cb_recordheader)+index);
+
+   if (no>=cb->header.highnumber)
+   {
+      cb->header.highnumber=no;
+log("debug: highnumber set to %lu\n",no);
+      fseek(cb->fp,0,0);
+      cb->header.ts_written=TIME;
+      if (!fwrite(&cb->header,sizeof(struct cachebaseheader),1,cb->fp))
+      {
+	 restart_kom("cache: !write to base header failed ('%s')\n",cb->filename);
+	 return bn;
+      }
+   }
+   return bn;
+}
+
+static void
+   cb_abandon(struct cachebase *cb,long bn,long no)
+{
+   struct cb_blockheader bh;
+   struct cb_recordheader rh;
+   int i,c;
+   unsigned long index;
+   unsigned long busy;
+
+   log("debug: cb_abandon rec %lu from block %lu\n",no,bn);
+   
+   fseek(cb->fp,BLOCKPOSITION(cb,bn),0);
+   fread(&bh,sizeof(struct cb_blockheader),1,cb->fp);
+   for (i=0; i<MAXRECORDSINBLOCK; i++)
+      if (bh.index[i]==no) bh.index[i]=~0;
+   fseek(cb->fp,BLOCKPOSITION(cb,bn),0);
+   if (!fwrite(&bh,sizeof(struct cb_blockheader),1,cb->fp))
+   {
+      restart_kom("cache: cb_abandon: failed to write blockheader (b%lu)\n",bn);
+      return;
+   }
+
+   index=sizeof(struct cb_blockheader);
+   
+   for (i=c=0; i<MAXRECORDSINBLOCK&&bh.index[i]; i++)
+   {
+      fseek(cb->fp,BLOCKPOSITION(cb,bn)+index,0);
+      /* optimera fseek'en till att ska framt? */
+      fread(&rh,sizeof(struct cb_recordheader),1,cb->fp);
+      if ((bh.index[i]!=~0)&&(rh.no!=bh.index[i]))
+      {
+	 restart_kom("cache: !record %lu in '%s', index(%lu)!=recordno(%lu) => corrupt\n",bn,cb->filename,bh.index[i],rh.no);
+	 return;
+      }
+      index+=rh.size+sizeof(struct cb_recordheader);
+      if (bh.index[i]!=~0)
+         busy+=rh.size+sizeof(struct cb_recordheader),c++;
+         
+   }
+   if (busy<REFRESHTRIGGER(cb))
+      cb_addtorefreshlist(cb,bn,TIME,busy,c);
+}
+
+static unsigned long
+   cb_getgeneration(struct cachebase *cb,unsigned long bn,
+		    unsigned long no)
+{
+   struct cb_blockheader bh;
+   struct cb_recordheader rh;
+   int i;
+   unsigned long index;
+   
+   if (bn<1)
+      { log("cache: cb_getgeneration: warning; bn<1\n"); return 0; }
+   if (no<1)
+      { log("cache: cb_getgeneration: warning; no<1\n"); return 0; }
+
+   fseek(cb->fp,BLOCKPOSITION(cb,bn),0);
+   fread(&bh,sizeof(struct cb_blockheader),1,cb->fp);
+   index=sizeof(struct cb_blockheader);
+   
+   for (i=0; i<MAXRECORDSINBLOCK&&bh.index[i]; i++)
+   {
+      fseek(cb->fp,BLOCKPOSITION(cb,bn)+index,0);
+      /* optimera fseek'en till att ska framt? */
+      fread(&rh,sizeof(struct cb_recordheader),1,cb->fp);
+      if ((bh.index[i]!=~0)&&(rh.no!=bh.index[i]))
+      {
+	 restart_kom("cache: !record %lu in '%s', index(%lu)!=recordno(%lu) => corrupt\n",bn,cb->filename,bh.index[i],rh.no);
+	 return 0;
+      }
+      if (rh.no==no) return rh.generation;
+      index+=rh.size+sizeof(struct cb_recordheader);
+   }
+   restart_kom("cache: !record %lu in '%s', missing record %lu => corrupt",bn,cb->filename,rh.no);
+   log("cache: cb_getgeneration: warning, missing record %ld in block %ld.\n",
+       no,bn);
+   return 0;
+}
+
+void *
+   cb_readrecord(struct cachebase *cb,unsigned long bn,
+		 unsigned long no,unsigned long *size,
+		 unsigned long *gen)
+{
+   struct cb_blockheader bh;
+   struct cb_recordheader rh;
+   int i;
+   unsigned long index;
+   void *buffert;
+   
+   if (no<1)
+      { log("cache: cb_readrecord: warning; no<1\n"); return 0; }
+
+   fseek(cb->fp,BLOCKPOSITION(cb,bn),0);
+   fread(&bh,sizeof(struct cb_blockheader),1,cb->fp);
+   index=sizeof(struct cb_blockheader);
+   
+   for (i=0; i<MAXRECORDSINBLOCK&&bh.index[i]; i++)
+   {