/* $Id: buffindexed.c 6854 2004-05-20 22:23:28Z rra $
**
** Overview buffer and index method.
*/
#include "config.h"
#include "clibrary.h"
#include "portable/mmap.h"
#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#if HAVE_LIMITS_H
# include <limits.h>
#endif
#include <syslog.h>
#include <sys/stat.h>
#include <time.h>
#include "inn/innconf.h"
#include "libinn.h"
#include "ov.h"
#include "paths.h"
#include "ovinterface.h"
#include "storage.h"
#include "buffindexed.h"
#define OVBUFF_MAGIC "ovbuff"
/* ovbuff header */
#define OVBUFFMASIZ 8
#define OVBUFFNASIZ 16
#define OVBUFFLASIZ 16
#define OVBUFFPASIZ 64
#define OVMAXCYCBUFFNAME 8
#define OV_HDR_PAGESIZE 16384
#define OV_BLOCKSIZE 8192
#define OV_BEFOREBITF (1 * OV_BLOCKSIZE)
#define OV_FUDGE 1024
/* ovblock pointer */
typedef struct _OV {
unsigned int blocknum;
short index;
} OV;
/* ovbuff header */
typedef struct {
char magic[OVBUFFMASIZ];
char path[OVBUFFPASIZ];
char indexa[OVBUFFLASIZ]; /* ASCII version of index */
char lena[OVBUFFLASIZ]; /* ASCII version of len */
char totala[OVBUFFLASIZ]; /* ASCII version of total */
char useda[OVBUFFLASIZ]; /* ASCII version of used */
char freea[OVBUFFLASIZ]; /* ASCII version of free */
char updateda[OVBUFFLASIZ]; /* ASCII version of updated */
} OVBUFFHEAD;
/* ovbuff info */
typedef struct _OVBUFF {
unsigned int index; /* ovbuff index */
char path[OVBUFFPASIZ]; /* Path to file */
int magicver; /* Magic version number */
int fd; /* file descriptor for this
ovbuff */
off_t len; /* Length of writable area, in
bytes */
off_t base; /* Offset (relative to byte
0 of file) to base block */
unsigned int freeblk; /* next free block number no
freeblk left if equals
totalblk */
unsigned int totalblk; /* number of total blocks */
unsigned int usedblk; /* number of used blocks */
time_t updated; /* Time of last update to
header */
void * bitfield; /* Bitfield for ovbuff block in
use */
bool needflush; /* true if OVBUFFHEAD is needed
to be flushed */
struct _OVBUFF *next; /* next ovbuff */
int nextchunk; /* next chunk */
#ifdef OV_DEBUG
struct ov_trace_array *trace;
#endif /* OV_DEBUG */
} OVBUFF;
typedef struct _OVINDEXHEAD {
OV next; /* next block */
ARTNUM low; /* lowest article number in the index */
ARTNUM high; /* highest article number in the index */
} OVINDEXHEAD;
typedef struct _OVINDEX {
ARTNUM artnum; /* article number */
unsigned int blocknum; /* overview data block number */
short index; /* overview data block index */
TOKEN token; /* token for this article */
off_t offset; /* offset from the top in the block */
int len; /* length of the data */
time_t arrived; /* arrived time of article */
time_t expires; /* expire time of article */
} OVINDEX;
#define OVINDEXMAX ((OV_BLOCKSIZE-sizeof(OVINDEXHEAD))/sizeof(OVINDEX))
typedef struct _OVBLOCK {
OVINDEXHEAD ovindexhead; /* overview index header */
OVINDEX ovindex[OVINDEXMAX]; /* overview index */
} OVBLOCK;
typedef struct _OVBLKS {
OVBLOCK *ovblock;
void * addr;
int len;
OV indexov;
} OVBLKS;
/* Data structure for specifying a location in the group index */
typedef struct {
int recno; /* Record number in group index */
} GROUPLOC;
#ifdef OV_DEBUG
struct ov_trace {
time_t occupied;
time_t freed;
GROUPLOC gloc;
};
#define OV_TRACENUM 10
struct ov_trace_array {
int max;
int cur;
struct ov_trace *ov_trace;
};
struct ov_name_table {
char *name;
int recno;
struct ov_name_table *next;
};
static struct ov_name_table *name_table = NULL;
#endif /* OV_DEBUG */
#define GROUPHEADERHASHSIZE (16 * 1024)
#define GROUPHEADERMAGIC (~(0xf1f0f33d))
typedef struct {
int magic;
GROUPLOC hash[GROUPHEADERHASHSIZE];
GROUPLOC freelist;
} GROUPHEADER;
/* The group is matched based on the MD5 of the grouname. This may prove to
be inadequate in the future, if so, the right thing to do is to is
probably just to add a SHA1 hash into here also. We get a really nice
benefit from this being fixed length, we should try to keep it that way.
*/
typedef struct {
HASH hash; /* MD5 hash of the group name */
HASH alias; /* If not empty then this is the hash of the
group that this group is an alias for */
ARTNUM high; /* High water mark in group */
ARTNUM low; /* Low water mark in group */
int count; /* Number of articles in group */
int flag; /* Posting/Moderation Status */
time_t expired; /* When last expiry */
time_t deleted; /* When this was deleted, 0 otherwise */
GROUPLOC next; /* Next block in this chain */
OV baseindex; /* base index buff */
OV curindex; /* current index buff */
int curindexoffset; /* current index offset for this ovbuff */
ARTNUM curhigh; /* High water mark in group */
ARTNUM curlow; /* Low water mark in group */
OV curdata; /* current offset for this ovbuff */
off_t curoffset; /* current offset for this ovbuff */
} GROUPENTRY;
typedef struct _GIBLIST {
OV ov;
struct _GIBLIST *next;
} GIBLIST;
typedef struct _GDB {
OV datablk;
void * addr;
void * data;
int len;
bool mmapped;
struct _GDB *next;
} GROUPDATABLOCK;
typedef struct {
char *group;
int lo;
int hi;
int cur;
bool needov;
GROUPLOC gloc;
int count;
GROUPDATABLOCK gdb; /* used for caching current block */
} OVSEARCH;
#define GROUPDATAHASHSIZE 25
static GROUPDATABLOCK *groupdatablock[GROUPDATAHASHSIZE];
typedef enum {PREPEND_BLK, APPEND_BLK} ADDINDEX;
typedef enum {SRCH_FRWD, SRCH_BKWD} SRCH;
#define _PATH_OVBUFFCONFIG "buffindexed.conf"
static char LocalLogName[] = "buffindexed";
static long pagesize = 0;
static OVBUFF *ovbufftab;
static int GROUPfd;
static GROUPHEADER *GROUPheader = NULL;
static GROUPENTRY *GROUPentries = NULL;
static int GROUPcount = 0;
static GROUPLOC GROUPemptyloc = { -1 };
#define NULLINDEX (-1)
static OV ovnull = { 0, NULLINDEX };
typedef unsigned long ULONG;
static ULONG onarray[64], offarray[64];
static int longsize = sizeof(long);
static bool Nospace;
static bool Needunlink;
static bool Cutofflow;
static bool Cache;
static OVSEARCH *Cachesearch;
static int ovbuffmode;
static GROUPLOC GROUPnewnode(void);
static bool GROUPremapifneeded(GROUPLOC loc);
static void GROUPLOCclear(GROUPLOC *loc);
static bool GROUPLOCempty(GROUPLOC loc);
static bool GROUPlockhash(enum inn_locktype type);
static bool GROUPlock(GROUPLOC gloc, enum inn_locktype type);
static off_t GROUPfilesize(int count);
static bool GROUPexpand(int mode);
static void *ovopensearch(char *group, int low, int high, bool needov);
static void ovclosesearch(void *handle, bool freeblock);
static OVINDEX *Gib;
static GIBLIST *Giblist;
static int Gibcount;
#ifdef MMAP_MISSES_WRITES
/* With HP/UX, you definitely do not want to mix mmap-accesses of
a file with read()s and write()s of the same file */
static off_t mmapwrite(int fd, void *buf, off_t nbyte, off_t offset) {
int pagefudge, len;
off_t mmapoffset;
void * addr;
pagefudge = offset % pagesize;
mmapoffset = offset - pagefudge;
len = pagefudge + nbyte;
if ((addr = mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_SHARED, fd, mmapoffset)) == MAP_FAILED) {
return -1;
}
memcpy(addr+pagefudge, buf, nbyte);
munmap(addr, len);
return nbyte;
}
#endif /* MMAP_MISSES_WRITES */
static bool ovparse_part_line(char *l) {
char *p;
struct stat sb;
off_t len, base;
int tonextblock;
OVBUFF *ovbuff, *tmp = ovbufftab;
/* ovbuff partition name */
if ((p = strchr(l, ':')) == NULL || p - l <= 0 || p - l > OVMAXCYCBUFFNAME - 1) {
syslog(L_ERROR, "%s: bad index in line '%s'", LocalLogName, l);
return false;
}
*p = '\0';
ovbuff = xmalloc(sizeof(OVBUFF));
ovbuff->index = strtoul(l, NULL, 10);
for (; tmp != (OVBUFF *)NULL; tmp = tmp->next) {
if (tmp->index == ovbuff->index) {
syslog(L_ERROR, "%s: dupulicate index in line '%s'", LocalLogName, l);
free(ovbuff);
return false;
}
}
l = ++p;
/* Path to ovbuff partition */
if ((p = strchr(l, ':')) == NULL || p - l <= 0 || p - l > OVBUFFPASIZ - 1) {
syslog(L_ERROR, "%s: bad pathname in line '%s'", LocalLogName, l);
free(ovbuff);
return false;
}
*p = '\0';
memset(ovbuff->path, '\0', OVBUFFPASIZ);
strlcpy(ovbuff->path, l, OVBUFFPASIZ);
if (stat(ovbuff->path, &sb) < 0) {
syslog(L_ERROR, "%s: file '%s' does not exist, ignoring '%d'",
LocalLogName, ovbuff->path, ovbuff->index);
free(ovbuff);
return false;
}
l = ++p;
/* Length/size of symbolic partition */
len = strtoul(l, NULL, 10) * 1024; /* This value in KB in decimal */
/*
** The minimum article offset will be the size of the bitfield itself,
** len / (blocksize * 8), plus however many additional blocks the OVBUFFHEAD
** external header occupies ... then round up to the next block.
*/
base = len / (OV_BLOCKSIZE * 8) + OV_BEFOREBITF;
tonextblock = OV_HDR_PAGESIZE - (base & (OV_HDR_PAGESIZE - 1));
ovbuff->base = base + tonextblock;
if (S_ISREG(sb.st_mode) && (len != sb.st_size || ovbuff->base > sb.st_size)) {
if (len != sb.st_size)
syslog(L_NOTICE, "%s: length mismatch '%lu' for index '%d' (%lu bytes)",
LocalLogName, (unsigned long) len, ovbuff->index,
(unsigned long) sb.st_size);
if (ovbuff->base > sb.st_size)
syslog(L_NOTICE, "%s: length must be at least '%lu' for index '%d' (%lu bytes)",
LocalLogName, (unsigned long) ovbuff->base, ovbuff->index,
(unsigned long) sb.st_size);
free(ovbuff);
return false;
}
ovbuff->len = len;
ovbuff->fd = -1;
ovbuff->next = (OVBUFF *)NULL;
ovbuff->needflush = false;
ovbuff->bitfield = NULL;
ovbuff->nextchunk = 1;
if (ovbufftab == (OVBUFF *)NULL)
ovbufftab = ovbuff;
else {
for (tmp = ovbufftab; tmp->next != (OVBUFF *)NULL; tmp = tmp->next);
tmp->next = ovbuff;
}
return true;
}
/*
** ovbuffread_config() -- Read the overview partition/file configuration file.
*/
static bool ovbuffread_config(void) {
char *path, *config, *from, *to, **ctab = (char **)NULL;
int ctab_free = 0; /* Index to next free slot in ctab */
int ctab_i;
path = concatpath(innconf->pathetc, _PATH_OVBUFFCONFIG);
config = ReadInFile(path, NULL);
if (config == NULL) {
syslog(L_ERROR, "%s: cannot read %s", LocalLogName, path);
free(config);
free(path);
return false;
}
free(path);
for (from = to = config; *from; ) {
if (*from == '#') { /* Comment line? */
while (*from && *from != '\n')
from++; /* Skip past it */
from++;
continue; /* Back to top of loop */
}
if (*from == '\n') { /* End or just a blank line? */
from++;
continue; /* Back to top of loop */
}
if (ctab_free == 0)
ctab = xmalloc(sizeof(char *));
else
ctab = xrealloc(ctab, (ctab_free + 1) * sizeof(char *));
/* If we're here, we've got the beginning of a real entry */
ctab[ctab_free++] = to = from;
while (1) {
if (*from && *from == '\\' && *(from + 1) == '\n') {
from += 2; /* Skip past backslash+newline */
while (*from && isspace((int)*from))
from++;
continue;
}
if (*from && *from != '\n')
*to++ = *from++;
if (*from == '\n') {
*to++ = '\0';
from++;
break;
}
if (! *from)
break;
}
}
for (ctab_i = 0; ctab_i < ctab_free; ctab_i++) {
if (!ovparse_part_line(ctab[ctab_i])) {
free(config);
free(ctab);
return false;
}
}
free(config);
free(ctab);
if (ovbufftab == (OVBUFF *)NULL) {
syslog(L_ERROR, "%s: no buffindexed defined", LocalLogName);
return false;
}
return true;
}
static char hextbl[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'a', 'b', 'c', 'd', 'e', 'f'};
static char *offt2hex(off_t offset, bool leadingzeros) {
static char buf[24];
char *p;
if (sizeof(off_t) <= 4) {
snprintf(buf, sizeof(buf), (leadingzeros) ? "%016lx" : "%lx", offset);
} else {
int i;
for (i = 0; i < OVBUFFLASIZ; i++)
buf[i] = '0'; /* Pad with zeros to start */
for (i = OVBUFFLASIZ - 1; i >= 0; i--) {
buf[i] = hextbl[offset & 0xf];
offset >>= 4;
}
}
if (!leadingzeros) {
for (p = buf; *p == '0'; p++)
;
if (*p != '\0')
return p;
else
return p - 1; /* We converted a "0" and then bypassed all the zeros */
} else
return buf;
}
static off_t hex2offt(char *hex) {
if (sizeof(off_t) <= 4) {
unsigned long rpofft;
sscanf(hex, "%lx", &rpofft);
return rpofft;
} else {
char diff;
off_t n = 0;
for (; *hex != '\0'; hex++) {
if (*hex >= '0' && *hex <= '9')
diff = '0';
else if (*hex >= 'a' && *hex <= 'f')
diff = 'a' - 10;
else if (*hex >= 'A' && *hex <= 'F')
diff = 'A' - 10;
else {
/*
** We used to have a syslog() message here, but the case
** where we land here because of a ":" happens, er, often.
*/
break;
}
n += (*hex - diff);
if (isalnum((int)*(hex + 1)))
n <<= 4;
}
return n;
}
}
static void ovreadhead(OVBUFF *ovbuff) {
OVBUFFHEAD rpx;
char buff[OVBUFFLASIZ+1];
memcpy(&rpx, ovbuff->bitfield, sizeof(OVBUFFHEAD));
strncpy(buff, rpx.useda, OVBUFFLASIZ);
buff[OVBUFFLASIZ] = '\0';
ovbuff->usedblk = (unsigned int)hex2offt((char *)buff);
strncpy(buff, rpx.freea, OVBUFFLASIZ);
buff[OVBUFFLASIZ] = '\0';
ovbuff->freeblk = (unsigned int)hex2offt((char *)buff);
return;
}
static void ovflushhead(OVBUFF *ovbuff) {
OVBUFFHEAD rpx;
if (!ovbuff->needflush)
return;
memset(&rpx, 0, sizeof(OVBUFFHEAD));
ovbuff->updated = time(NULL);
strncpy(rpx.magic, OVBUFF_MAGIC, strlen(OVBUFF_MAGIC));
strncpy(rpx.path, ovbuff->path, OVBUFFPASIZ);
/* Don't use sprintf() directly ... the terminating '\0' causes grief */
strncpy(rpx.indexa, offt2hex(ovbuff->index, true), OVBUFFLASIZ);
strncpy(rpx.lena, offt2hex(ovbuff->len, true), OVBUFFLASIZ);
strncpy(rpx.totala, offt2hex(ovbuff->totalblk, true), OVBUFFLASIZ);
strncpy(rpx.useda, offt2hex(ovbuff->usedblk, true), OVBUFFLASIZ);
strncpy(rpx.freea, offt2hex(ovbuff->freeblk, true), OVBUFFLASIZ);
strncpy(rpx.updateda, offt2hex(ovbuff->updated, true), OVBUFFLASIZ);
memcpy(ovbuff->bitfield, &rpx, sizeof(OVBUFFHEAD));
mmap_flush(ovbuff->bitfield, ovbuff->base);
ovbuff->needflush = false;
return;
}
static bool ovlock(OVBUFF *ovbuff, enum inn_locktype type) {
return inn_lock_range(ovbuff->fd, type, true, 0, sizeof(OVBUFFHEAD));
}
static bool ovbuffinit_disks(void) {
OVBUFF *ovbuff = ovbufftab;
char buf[64];
OVBUFFHEAD *rpx;
int i, fd;
off_t tmpo;
/*
** Discover the state of our ovbuffs. If any of them are in icky shape,
** duck shamelessly & return false.
*/
for (; ovbuff != (OVBUFF *)NULL; ovbuff = ovbuff->next) {
if (ovbuff->fd < 0) {
if ((fd = open(ovbuff->path, ovbuffmode & OV_WRITE ? O_RDWR : O_RDONLY)) < 0) {
syslog(L_ERROR, "%s: ERROR opening '%s' : %m", LocalLogName, ovbuff->path);
return false;
} else {
close_on_exec(fd, true);
ovbuff->fd = fd;
}
}
if ((ovbuff->bitfield =
mmap(NULL, ovbuff->base, ovbuffmode & OV_WRITE ? (PROT_READ | PROT_WRITE) : PROT_READ,
MAP_SHARED, ovbuff->fd, (off_t) 0)) == MAP_FAILED) {
syslog(L_ERROR,
"%s: ovinitdisks: mmap for %s offset %d len %lu failed: %m",
LocalLogName, ovbuff->path, 0, (unsigned long) ovbuff->base);
return false;
}
rpx = (OVBUFFHEAD *)ovbuff->bitfield;
ovlock(ovbuff, INN_LOCK_WRITE);
if (strncmp(rpx->magic, OVBUFF_MAGIC, strlen(OVBUFF_MAGIC)) == 0) {
ovbuff->magicver = 1;
if (strncmp(rpx->path, ovbuff->path, OVBUFFPASIZ) != 0) {
syslog(L_ERROR, "%s: Path mismatch: read %s for buffindexed %s",
LocalLogName, rpx->path, ovbuff->path);
ovbuff->needflush = true;
}
strncpy(buf, rpx->indexa, OVBUFFLASIZ);
buf[OVBUFFLASIZ] = '\0';
i = hex2offt(buf);
if (i != ovbuff->index) {
syslog(L_ERROR, "%s: Mismatch: index '%d' for buffindexed %s",
LocalLogName, i, ovbuff->path);
ovlock(ovbuff, INN_LOCK_UNLOCK);
return false;
}
strncpy(buf, rpx->lena, OVBUFFLASIZ);
buf[OVBUFFLASIZ] = '\0';
tmpo = hex2offt(buf);
if (tmpo != ovbuff->len) {
syslog(L_ERROR, "%s: Mismatch: read 0x%s length for buffindexed %s",
LocalLogName, offt2hex(tmpo, false), ovbuff->path);
ovlock(ovbuff, INN_LOCK_UNLOCK);
return false;
}
strncpy(buf, rpx->totala, OVBUFFLASIZ);
buf[OVBUFFLASIZ] = '\0';
ovbuff->totalblk = hex2offt(buf);
strncpy(buf, rpx->useda, OVBUFFLASIZ);
buf[OVBUFFLASIZ] = '\0';
ovbuff->usedblk = hex2offt(buf);
strncpy(buf, rpx->freea, OVBUFFLASIZ);
buf[OVBUFFLASIZ] = '\0';
ovbuff->freeblk = hex2offt(buf);
ovflushhead(ovbuff);
Needunlink = false;
} else {
ovbuff->totalblk = (ovbuff->len - ovbuff->base)/OV_BLOCKSIZE;
if (ovbuff->totalblk < 1) {
syslog(L_ERROR, "%s: too small length '%lu' for buffindexed %s",
LocalLogName, (unsigned long) ovbuff->len, ovbuff->path);
ovlock(ovbuff, INN_LOCK_UNLOCK);
return false;
}
ovbuff->magicver = 1;
ovbuff->usedblk = 0;
ovbuff->freeblk = 0;
ovbuff->updated = 0;
ovbuff->needflush = true;
syslog(L_NOTICE,
"%s: No magic cookie found for buffindexed %d, initializing",
LocalLogName, ovbuff->index);
ovflushhead(ovbuff);
}
#ifdef OV_DEBUG
ovbuff->trace = xcalloc(ovbuff->totalblk, sizeof(ov_trace_array));
#endif /* OV_DEBUG */
ovlock(ovbuff, INN_LOCK_UNLOCK);
}
return true;
}
static int ovusedblock(OVBUFF *ovbuff, int blocknum, bool set_operation, bool setbitvalue) {
off_t longoffset;
int bitoffset; /* From the 'left' side of the long */
ULONG bitlong, mask;
longoffset = blocknum / (sizeof(long) * 8);
bitoffset = blocknum % (sizeof(long) * 8);
bitlong = *((ULONG *) ovbuff->bitfield + (OV_BEFOREBITF / sizeof(long))
+ longoffset);
if (set_operation) {
if (setbitvalue) {
mask = onarray[bitoffset];
bitlong |= mask;
} else {
mask = offarray[bitoffset];
bitlong &= mask;
}
*((ULONG *) ovbuff->bitfield + (OV_BEFOREBITF / sizeof(long))
+ longoffset) = bitlong;
return 2; /* XXX Clean up return semantics */
}
/* It's a read operation */
mask = onarray[bitoffset];
/* return bitlong & mask; doesn't work if sizeof(ulong) > sizeof(int) */
if ( bitlong & mask ) return 1; else return 0;
}
static void ovnextblock(OVBUFF *ovbuff) {
int i, j, last, lastbit, left;
ULONG mask = 0x80000000;
ULONG *table;
last = ovbuff->totalblk/(sizeof(long) * 8);
if ((left = ovbuff->totalblk % (sizeof(long) * 8)) != 0) {
last++;
}
table = ((ULONG *) ovbuff->bitfield + (OV_BEFOREBITF / sizeof(long)));
for (i = ovbuff->nextchunk ; i < last ; i++) {
if (i == last - 1 && left != 0) {
for (j = 1 ; j < left ; j++) {
mask |= mask >> 1;
}
if ((table[i] & mask) != mask)
break;
} else {
if ((table[i] ^ ~0) != 0)
break;
}
}
if (i == last) {
for (i = 0 ; i < ovbuff->nextchunk ; i++) {
if ((table[i] ^ ~0) != 0)
break;
}
if (i == ovbuff->nextchunk) {
ovbuff->freeblk = ovbuff->totalblk;
return;
}
}
if ((i - 1) >= 0 && (last - 1 == i) && left != 0) {
lastbit = left;
} else {
lastbit = sizeof(long) * 8;
}
for (j = 0 ; j < lastbit ; j++) {
if ((table[i] & onarray[j]) == 0)
break;
}
if (j == lastbit) {
ovbuff->freeblk = ovbuff->totalblk;
return;
}
ovbuff->freeblk = i * sizeof(long) * 8 + j;
ovbuff->nextchunk = i + 1;
if (i == last)
ovbuff->nextchunk = 0;
return;
}
static OVBUFF *getovbuff(OV ov) {
OVBUFF *ovbuff = ovbufftab;
for (; ovbuff != (OVBUFF *)NULL; ovbuff = ovbuff->next) {
if (ovbuff->index == ov.index)
return ovbuff;
}
return NULL;
}
#ifdef OV_DEBUG
static OV ovblocknew(GROUPENTRY *ge) {
#else
static OV ovblocknew(void) {
#endif /* OV_DEBUG */
static OVBUFF *ovbuffnext = NULL;
OVBUFF *ovbuff;
OV ov;
#ifdef OV_DEBUG
int recno;
struct ov_trace_array *trace;
#endif /* OV_DEBUG */
if (ovbuffnext == NULL)
ovbuffnext = ovbufftab;
for (ovbuff = ovbuffnext ; ovbuff != (OVBUFF *)NULL ; ovbuff = ovbuff->next) {
ovlock(ovbuff, INN_LOCK_WRITE);
ovreadhead(ovbuff);
if (ovbuff->totalblk != ovbuff->usedblk && ovbuff->freeblk == ovbuff->totalblk) {
ovnextblock(ovbuff);
}
if (ovbuff->totalblk == ovbuff->usedblk || ovbuff->freeblk == ovbuff->totalblk) {
/* no space left for this ovbuff */
ovlock(ovbuff, INN_LOCK_UNLOCK);
continue;
}
break;
}
if (ovbuff == NULL) {
for (ovbuff = ovbufftab ; ovbuff != ovbuffnext ; ovbuff = ovbuff->next) {
ovlock(ovbuff, INN_LOCK_WRITE);
ovreadhead(ovbuff);
if (ovbuff->totalblk == ovbuff->usedblk || ovbuff->freeblk == ovbuff->totalblk) {
/* no space left for this ovbuff */
ovlock(ovbuff, INN_LOCK_UNLOCK);
continue;
}
break;
}
if (ovbuff == ovbuffnext) {
Nospace = true;
return ovnull;
}
}
#ifdef OV_DEBUG
recno = ((char *)ge - (char *)&GROUPentries[0])/sizeof(GROUPENTRY);
if (ovusedblock(ovbuff, ovbuff->freeblk, false, true)) {
syslog(L_FATAL, "%s: 0x%08x trying to occupy new block(%d, %d), but already occupied", LocalLogName, recno, ovbuff->index, ovbuff->freeblk);
buffindexed_close();
abort();
}
trace = &ovbuff->trace[ovbuff->freeblk];
if (trace->ov_trace == NULL) {
trace->ov_trace = xcalloc(OV_TRACENUM, sizeof(struct ov_trace));
trace->max = OV_TRACENUM;
} else if (trace->cur + 1 == trace->max) {
trace->max += OV_TRACENUM;
trace->ov_trace = xrealloc(trace->ov_trace, trace->max * sizeof(struct ov_trace));
memset(&trace->ov_trace[trace->cur], '\0', sizeof(struct ov_trace) * (trace->max - trace->cur));
}
if (trace->ov_trace[trace->cur].occupied != 0) {
trace->cur++;
}
trace->ov_trace[trace->cur].gloc.recno = recno;
trace->ov_trace[trace->cur].occupied = time(NULL);
#endif /* OV_DEBUG */
ov.index = ovbuff->index;
ov.blocknum = ovbuff->freeblk;
ovusedblock(ovbuff, ov.blocknum, true, true);
ovnextblock(ovbuff);
ovbuff->usedblk++;
ovbuff->needflush = true;
ovflushhead(ovbuff);
ovlock(ovbuff, INN_LOCK_UNLOCK);
ovbuffnext = ovbuff->next;
if (ovbuffnext == NULL)
ovbuffnext = ovbufftab;
return ov;
}
#ifdef OV_DEBUG
static void ovblockfree(OV ov, GROUPENTRY *ge) {
#else
static void ovblockfree(OV ov) {
#endif /* OV_DEBUG */
OVBUFF *ovbuff;
#ifdef OV_DEBUG
int recno;
struct ov_trace_array *trace;
#endif /* OV_DEBUG */
if (ov.index == NULLINDEX)
return;
if ((ovbuff = getovbuff(ov)) == NULL)
return;
ovlock(ovbuff, INN_LOCK_WRITE);
#ifdef OV_DEBUG
recno = ((char *)ge - (char *)&GROUPentries[0])/sizeof(GROUPENTRY);
if (!ovusedblock(ovbuff, ov.blocknum, false, false)) {
syslog(L_FATAL, "%s: 0x%08x trying to free block(%d, %d), but already freed", LocalLogName, recno, ov.index, ov.blocknum);
buffindexed_close();
abort();
}
trace = &ovbuff->trace[ov.blocknum];
if (trace->ov_trace == NULL) {
trace->ov_trace = xcalloc(OV_TRACENUM, sizeof(struct ov_trace));
trace->max = OV_TRACENUM;
} else if (trace->cur + 1 == trace->max) {
trace->max += OV_TRACENUM;
trace->ov_trace = xrealloc(trace->ov_trace, trace->max * sizeof(struct ov_trace));
memset(&trace->ov_trace[trace->cur], '\0', sizeof(struct ov_trace) * (trace->max - trace->cur));
}
if (trace->ov_trace[trace->cur].freed != 0) {
trace->cur++;
}
trace->ov_trace[trace->cur].freed = time(NULL);
trace->ov_trace[trace->cur].gloc.recno = recno;
trace->cur++;
#endif /* OV_DEBUG */
ovusedblock(ovbuff, ov.blocknum, true, false);
ovreadhead(ovbuff);
if (ovbuff->freeblk == ovbuff->totalblk)
ovbuff->freeblk = ov.blocknum;
ovbuff->usedblk--;
ovbuff->needflush = true;
ovflushhead(ovbuff);
ovlock(ovbuff, INN_LOCK_UNLOCK);
return;
}
bool buffindexed_open(int mode) {
char *groupfn;
struct stat sb;
int i, flag = 0;
static int uninitialized = 1;
ULONG on, off;
if (uninitialized) {
on = 1;
off = on;
off ^= ULONG_MAX;
for (i = (longsize * 8) - 1; i >= 0; i--) {
onarray[i] = on;
offarray[i] = off;
on <<= 1;
off = on;
off ^= ULONG_MAX;
}
uninitialized = 0;
}
ovbuffmode = mode;
if (pagesize == 0) {
pagesize = getpagesize();
if (pagesize == -1) {
syslog(L_ERROR, "%s: getpagesize failed: %m", LocalLogName);
pagesize = 0;
return false;
}
if ((pagesize > OV_HDR_PAGESIZE) || (OV_HDR_PAGESIZE % pagesize)) {
syslog(L_ERROR, "%s: OV_HDR_PAGESIZE (%d) is not a multiple of pagesize (%ld)", LocalLogName, OV_HDR_PAGESIZE, pagesize);
return false;
}
}
memset(&groupdatablock, '\0', sizeof(groupdatablock));
if (!ovbuffread_config()) {
return false;
}
Needunlink = true;
if (!ovbuffinit_disks()) {
return false;
}
groupfn = concatpath(innconf->pathdb, "group.index");
if (Needunlink && unlink(groupfn) == 0) {
syslog(L_NOTICE, "%s: all buffers are brandnew, unlink '%s'", LocalLogName, groupfn);
}
GROUPfd = open(groupfn, ovbuffmode & OV_WRITE ? O_RDWR | O_CREAT : O_RDONLY, 0660);
if (GROUPfd < 0) {
syslog(L_FATAL, "%s: Could not create %s: %m", LocalLogName, groupfn);
free(groupfn);
return false;
}
if (fstat(GROUPfd, &sb) < 0) {
syslog(L_FATAL, "%s: Could not fstat %s: %m", LocalLogName, groupfn);
free(groupfn);
close(GROUPfd);
return false;
}
if (sb.st_size > sizeof(GROUPHEADER)) {
if (mode & OV_READ)
flag |= PROT_READ;
if (mode & OV_WRITE) {
/*
* Note: below mapping of groupheader won't work unless we have
* both PROT_READ and PROT_WRITE perms.
*/
flag |= PROT_WRITE|PROT_READ;
}
GROUPcount = (sb.st_size - sizeof(GROUPHEADER)) / sizeof(GROUPENTRY);
if ((GROUPheader = (GROUPHEADER *)mmap(0, GROUPfilesize(GROUPcount), flag,
MAP_SHARED, GROUPfd, 0)) == (GROUPHEADER *) -1) {
syslog(L_FATAL, "%s: Could not mmap %s in buffindexed_open: %m", LocalLogName, groupfn);
free(groupfn);
close(GROUPfd);
return false;
}
GROUPentries = (GROUPENTRY *)((char *)GROUPheader + sizeof(GROUPHEADER));
} else {
GROUPcount = 0;
if (!GROUPexpand(mode)) {
free(groupfn);
close(GROUPfd);
return false;
}
}
close_on_exec(GROUPfd, true);
free(groupfn);
Cutofflow = false;
return true;
}
static GROUPLOC GROUPfind(char *group, bool Ignoredeleted) {
HASH grouphash;
unsigned int i;
GROUPLOC loc;
grouphash = Hash(group, strlen(group));
memcpy(&i, &grouphash, sizeof(i));
loc = GROUPheader->hash[i % GROUPHEADERHASHSIZE];
GROUPremapifneeded(loc);
while (!GROUPLOCempty(loc)) {
if (GROUPentries[loc.recno].deleted == 0 || Ignoredeleted) {
if (memcmp(&grouphash, &GROUPentries[loc.recno].hash, sizeof(HASH)) == 0) {
return loc;
}
}
loc = GROUPentries[loc.recno].next;
}
return GROUPemptyloc;
}
bool buffindexed_groupstats(char *group, int *lo, int *hi, int *count, int *flag) {
GROUPLOC gloc;
gloc = GROUPfind(group, false);
if (GROUPLOCempty(gloc)) {
return false;
}
GROUPlock(gloc, INN_LOCK_READ);
if (lo != NULL)
*lo = GROUPentries[gloc.recno].low;
if (hi != NULL)
*hi = GROUPentries[gloc.recno].high;
if (count != NULL)
*count = GROUPentries[gloc.recno].count;
if (flag != NULL)
*flag = GROUPentries[gloc.recno].flag;
GROUPlock(gloc, INN_LOCK_UNLOCK);
return true;
}
static void setinitialge(GROUPENTRY *ge, HASH grouphash, char *flag, GROUPLOC next, ARTNUM lo, ARTNUM hi) {
ge->hash = grouphash;
if (lo != 0)
ge->low = lo;
ge->high = hi;
ge->expired = ge->deleted = ge->count = 0;
ge->flag = *flag;
ge->baseindex = ge->curindex = ge->curdata = ovnull;
ge->curindexoffset = ge->curoffset = 0;
ge->next = next;
}
bool buffindexed_groupadd(char *group, ARTNUM lo, ARTNUM hi, char *flag) {
unsigned int i;
HASH grouphash;
GROUPLOC gloc;
GROUPENTRY *ge;
#ifdef OV_DEBUG
struct ov_name_table *ntp;
#endif /* OV_DEBUG */
gloc = GROUPfind(group, true);
if (!GROUPLOCempty(gloc)) {
ge = &GROUPentries[gloc.recno];
if (GROUPentries[gloc.recno].deleted != 0) {
grouphash = Hash(group, strlen(group));
setinitialge(ge, grouphash, flag, ge->next, lo, hi);
} else {
ge->flag = *flag;
}
return true;
}
grouphash = Hash(group, strlen(group));
memcpy(&i, &grouphash, sizeof(i));
i = i % GROUPHEADERHASHSIZE;
GROUPlockhash(INN_LOCK_WRITE);
gloc = GROUPnewnode();
ge = &GROUPentries[gloc.recno];
setinitialge(ge, grouphash, flag, GROUPheader->hash[i], lo, hi);
GROUPheader->hash[i] = gloc;
#ifdef OV_DEBUG
ntp = xmalloc(sizeof(struct ov_name_table));
memset(ntp, '\0', sizeof(struct ov_name_table));
ntp->name = xstrdup(group);
ntp->recno = gloc.recno;
if (name_table == NULL)
name_table = ntp;
else {
ntp->next = name_table;
name_table = ntp;
}
#endif /* OV_DEBUG */
GROUPlockhash(INN_LOCK_UNLOCK);
return true;
}
static off_t GROUPfilesize(int count) {
return ((off_t) count * sizeof(GROUPENTRY)) + sizeof(GROUPHEADER);
}
/* Check if the given GROUPLOC refers to GROUPENTRY that we don't have mmap'ed,
** if so then see if the file has been grown by another writer and remmap
*/
static bool GROUPremapifneeded(GROUPLOC loc) {
struct stat sb;
if (loc.recno < GROUPcount)
return true;
if (fstat(GROUPfd, &sb) < 0)
return false;
if (GROUPfilesize(GROUPcount) >= sb.st_size)
return true;
if (GROUPheader) {
if (munmap((void *)GROUPheader, GROUPfilesize(GROUPcount)) < 0) {
syslog(L_FATAL, "%s: Could not munmap group.index in GROUPremapifneeded: %m", LocalLogName);
return false;
}
}
GROUPcount = (sb.st_size - sizeof(GROUPHEADER)) / sizeof(GROUPENTRY);
GROUPheader = (GROUPHEADER *)mmap(0, GROUPfilesize(GROUPcount),
PROT_READ | PROT_WRITE, MAP_SHARED, GROUPfd, 0);
if (GROUPheader == (GROUPHEADER *) -1) {
syslog(L_FATAL, "%s: Could not mmap group.index in GROUPremapifneeded: %m", LocalLogName);
return false;
}
GROUPentries = (GROUPENTRY *)((char *)GROUPheader + sizeof(GROUPHEADER));
return true;
}
/* This function does not need to lock because it's callers are expected to do so */
static bool GROUPexpand(int mode) {
int i;
int flag = 0;
if (GROUPheader) {
if (munmap((void *)GROUPheader, GROUPfilesize(GROUPcount)) < 0) {
syslog(L_FATAL, "%s: Could not munmap group.index in GROUPexpand: %m", LocalLogName);
return false;
}
}
GROUPcount += 1024;
if (ftruncate(GROUPfd, GROUPfilesize(GROUPcount)) < 0) {
syslog(L_FATAL, "%s: Could not extend group.index: %m", LocalLogName);
return false;
}
if (mode & OV_READ)
flag |= PROT_READ;
if (mode & OV_WRITE) {
/*
* Note: below check of magic won't work unless we have both PROT_READ
* and PROT_WRITE perms.
*/
flag |= PROT_WRITE|PROT_READ;
}
GROUPheader = (GROUPHEADER *)mmap(0, GROUPfilesize(GROUPcount),
flag, MAP_SHARED, GROUPfd, 0);
if (GROUPheader == (GROUPHEADER *) -1) {
syslog(L_FATAL, "%s: Could not mmap group.index in GROUPexpand: %m", LocalLogName);
return false;
}
GROUPentries = (GROUPENTRY *)((char *)GROUPheader + sizeof(GROUPHEADER));
if (GROUPheader->magic != GROUPHEADERMAGIC) {
GROUPheader->magic = GROUPHEADERMAGIC;
GROUPLOCclear(&GROUPheader->freelist);
for (i = 0; i < GROUPHEADERHASHSIZE; i++)
GROUPLOCclear(&GROUPheader->hash[i]);
}
/* Walk the new entries from the back to the front, adding them to the freelist */
for (i = GROUPcount - 1; (GROUPcount - 1024) <= i; i--) {
GROUPentries[i].next = GROUPheader->freelist;
GROUPheader->freelist.recno = i;
}
return true;
}
static GROUPLOC GROUPnewnode(void) {
GROUPLOC loc;
/* If we didn't find any free space, then make some */
if (GROUPLOCempty(GROUPheader->freelist)) {
if (!GROUPexpand(ovbuffmode)) {
return GROUPemptyloc;
}
}
assert(!GROUPLOCempty(GROUPheader->freelist));
loc = GROUPheader->freelist;
GROUPheader->freelist = GROUPentries[GROUPheader->freelist.recno].next;
return loc;
}
bool buffindexed_groupdel(char *group) {
GROUPLOC gloc;
GROUPENTRY *ge;
gloc = GROUPfind(group, false);
if (GROUPLOCempty(gloc)) {
return true;
}
GROUPlock(gloc, INN_LOCK_WRITE);
ge = &GROUPentries[gloc.recno];
ge->deleted = time(NULL);
HashClear(&ge->hash);
GROUPlock(gloc, INN_LOCK_UNLOCK);
return true;
}
static void GROUPLOCclear(GROUPLOC *loc) {
loc->recno = -1;
}
static bool GROUPLOCempty(GROUPLOC loc) {
return (loc.recno < 0);
}
static bool GROUPlockhash(enum inn_locktype type) {
return inn_lock_range(GROUPfd, type, true, 0, sizeof(GROUPHEADER));
}
static bool GROUPlock(GROUPLOC gloc, enum inn_locktype type) {
return inn_lock_range(GROUPfd,
type,
true,
sizeof(GROUPHEADER) + (sizeof(GROUPENTRY) * gloc.recno),
sizeof(GROUPENTRY));
}
#ifdef OV_DEBUG
static bool ovsetcurindexblock(GROUPENTRY *ge, GROUPENTRY *georig) {
#else
static bool ovsetcurindexblock(GROUPENTRY *ge) {
#endif /* OV_DEBUG */
OVBUFF *ovbuff;
OV ov;
OVINDEXHEAD ovindexhead;
/* there is no index */
#ifdef OV_DEBUG
ov = ovblocknew(georig ? georig : ge);
#else
ov = ovblocknew();
#endif /* OV_DEBUG */
if (ov.index == NULLINDEX) {
syslog(L_ERROR, "%s: ovsetcurindexblock could not get new block", LocalLogName);
return false;
}
if ((ovbuff = getovbuff(ov)) == NULL) {
syslog(L_ERROR, "%s: ovsetcurindexblock could not get ovbuff block for new, %d, %d", LocalLogName, ov.index, ov.blocknum);
return false;
}
ovindexhead.next = ovnull;
ovindexhead.low = 0;
ovindexhead.high = 0;
#ifdef MMAP_MISSES_WRITES
if (mmapwrite(ovbuff->fd, &ovindexhead, sizeof(OVINDEXHEAD), ovbuff->base + ov.blocknum * OV_BLOCKSIZE) != sizeof(OVINDEXHEAD)) {
#else
if (pwrite(ovbuff->fd, &ovindexhead, sizeof(OVINDEXHEAD), ovbuff->base + ov.blocknum * OV_BLOCKSIZE) != sizeof(OVINDEXHEAD)) {
#endif /* MMAP_MISSES_WRITES */
syslog(L_ERROR, "%s: could not write index record index '%d', blocknum '%d': %m", LocalLogName, ge->curindex.index, ge->curindex.blocknum);
return true;
}
if (ge->baseindex.index == NULLINDEX) {
ge->baseindex = ov;
} else {
if ((ovbuff = getovbuff(ge->curindex)) == NULL)
return false;
#ifdef OV_DEBUG
if (!ovusedblock(ovbuff, ge->curindex.blocknum, false, false)) {
syslog(L_FATAL, "%s: block(%d, %d) not occupied (index)", LocalLogName, ovbuff->index, ge->curindex.blocknum);
abort();
}
#endif /* OV_DEBUG */
ovindexhead.next = ov;
ovindexhead.low = ge->curlow;
ovindexhead.high = ge->curhigh;
#ifdef MMAP_MISSES_WRITES
if (mmapwrite(ovbuff->fd, &ovindexhead, sizeof(OVINDEXHEAD), ovbuff->base + ge->curindex.blocknum * OV_BLOCKSIZE) != sizeof(OVINDEXHEAD)) {
#else
if (pwrite(ovbuff->fd, &ovindexhead, sizeof(OVINDEXHEAD), ovbuff->base + ge->curindex.blocknum * OV_BLOCKSIZE) != sizeof(OVINDEXHEAD)) {
#endif /* MMAP_MISSES_WRITES */
syslog(L_ERROR, "%s: could not write index record index '%d', blocknum '%d': %m", LocalLogName, ge->curindex.index, ge->curindex.blocknum);
return false;
}
}
ge->curindex = ov;
ge->curindexoffset = 0;
ge->curlow = 0;
ge->curhigh = 0;
return true;
}
#ifdef OV_DEBUG
static bool ovaddrec(GROUPENTRY *ge, ARTNUM artnum, TOKEN token, char *data, int len, time_t arrived, time_t expires, GROUPENTRY *georig) {
#else
static bool ovaddrec(GROUPENTRY *ge, ARTNUM artnum, TOKEN token, char *data, int len, time_t arrived, time_t expires) {
#endif /* OV_DEBUG */
OV ov;
OVINDEX ie;
OVBUFF *ovbuff;
OVINDEXHEAD ovindexhead;
bool needupdate = false;
#ifdef OV_DEBUG
int recno;
#endif /* OV_DEBUG */
Nospace = false;
if (OV_BLOCKSIZE < len) {
syslog(L_ERROR, "%s: overview data must be under %d (%d)", LocalLogName, OV_BLOCKSIZE, len);
return false;
}
if (ge->curdata.index == NULLINDEX) {
/* no data block allocated */
#ifdef OV_DEBUG
ov = ovblocknew(georig ? georig : ge);
#else
ov = ovblocknew();
#endif /* OV_DEBUG */
if (ov.index == NULLINDEX) {
syslog(L_ERROR, "%s: ovaddrec could not get new block", LocalLogName);
return false;
}
if ((ovbuff = getovbuff(ov)) == NULL) {
syslog(L_ERROR, "%s: ovaddrec could not get ovbuff block for new, %d, %d, %ld", LocalLogName, ov.index, ov.blocknum, artnum);
return false;
}
ge->curdata = ov;
ge->curoffset = 0;
} else if ((ovbuff = getovbuff(ge->curdata)) == NULL)
return false;
else if (OV_BLOCKSIZE - ge->curoffset < len) {
/* too short to store data, allocate new block */
#ifdef OV_DEBUG
ov = ovblocknew(georig ? georig : ge);
#else
ov = ovblocknew();
#endif /* OV_DEBUG */
if (ov.index == NULLINDEX) {
syslog(L_ERROR, "%s: ovaddrec could not get new block", LocalLogName);
return false;
}
if ((ovbuff = getovbuff(ov)) == NULL) {
syslog(L_ERROR, "%s: ovaddrec could not get ovbuff block for new, %d, %d, %ld", LocalLogName, ov.index, ov.blocknum, artnum);
return false;
}
ge->curdata = ov;
ge->curoffset = 0;
}
#ifdef OV_DEBUG
if (!ovusedblock(ovbuff, ge->curdata.blocknum, false, false)) {
syslog(L_FATAL, "%s: block(%d, %d) not occupied", LocalLogName, ovbuff->index, ge->curdata.blocknum);
buffindexed_close();
abort();
}
#endif /* OV_DEBUG */
#ifdef MMAP_MISSES_WRITES
if (mmapwrite(ovbuff->fd, data, len, ovbuff->base + ge->curdata.blocknum * OV_BLOCKSIZE + ge->curoffset) != len) {
#else
if (pwrite(ovbuff->fd, data, len, ovbuff->base + ge->curdata.blocknum * OV_BLOCKSIZE + ge->curoffset) != len) {
#endif /* MMAP_MISSES_WRITES */
syslog(L_ERROR, "%s: could not append overview record index '%d', blocknum '%d': %m", LocalLogName, ge->curdata.index, ge->curdata.blocknum);
return false;
}
memset(&ie, '\0', sizeof(ie));
ie.artnum = artnum;
ie.len = len;
ie.index = ge->curdata.index;
ie.blocknum = ge->curdata.blocknum;
ie.offset = ge->curoffset;
ie.token = token;
ie.arrived = arrived;
ie.expires = expires;
if (ge->baseindex.index == NULLINDEX || ge->curindexoffset == OVINDEXMAX) {
#ifdef OV_DEBUG
if (!ovsetcurindexblock(ge, georig)) {
#else
if (!ovsetcurindexblock(ge)) {
#endif /* OV_DEBUG */
syslog(L_ERROR, "%s: could not set current index", LocalLogName);
return false;
}
}
if ((ovbuff = getovbuff(ge->curindex)) == NULL)
return false;
#ifdef OV_DEBUG
if (!ovusedblock(ovbuff, ge->curindex.blocknum, false, false)) {
syslog(L_FATAL, "%s: block(%d, %d) not occupied (index)", LocalLogName, ovbuff->index, ge->curindex.blocknum);
buffindexed_close();
abort();
}
#endif /* OV_DEBUG */
#ifdef MMAP_MISSES_WRITES
if (mmapwrite(ovbuff->fd, &ie, sizeof(ie), ovbuff->base + ge->curindex.blocknum * OV_BLOCKSIZE + sizeof(OVINDEXHEAD) + sizeof(ie) * ge->curindexoffset) != sizeof(ie)) {
#else
if (pwrite(ovbuff->fd, &ie, sizeof(ie), ovbuff->base + ge->curindex.blocknum * OV_BLOCKSIZE + sizeof(OVINDEXHEAD) + sizeof(ie) * ge->curindexoffset) != sizeof(ie)) {
#endif /* MMAP_MISSES_WRITES */
syslog(L_ERROR, "%s: could not write index record index '%d', blocknum '%d': %m", LocalLogName, ge->curindex.index, ge->curindex.blocknum);
return true;
}
if ((ge->curlow <= 0) || (ge->curlow > artnum)) {
ge->curlow = artnum;
needupdate = true;
}
if ((ge->curhigh <= 0) || (ge->curhigh < artnum)) {
ge->curhigh = artnum;
needupdate = true;
}
if (needupdate) {
ovindexhead.next = ovnull;
ovindexhead.low = ge->curlow;
ovindexhead.high = ge->curhigh;
#ifdef MMAP_MISSES_WRITES
if (mmapwrite(ovbuff->fd, &ovindexhead, sizeof(OVINDEXHEAD), ovbuff->base + ge->curindex.blocknum * OV_BLOCKSIZE) != sizeof(OVINDEXHEAD)) {
#else
if (pwrite(ovbuff->fd, &ovindexhead, sizeof(OVINDEXHEAD), ovbuff->base + ge->curindex.blocknum * OV_BLOCKSIZE) != sizeof(OVINDEXHEAD)) {
#endif /* MMAP_MISSES_WRITES */
syslog(L_ERROR, "%s: could not write index record index '%d', blocknum '%d': %m", LocalLogName, ge->curindex.index, ge->curindex.blocknum);
return true;
}
}
if ((ge->low <= 0) || (ge->low > artnum))
ge->low = artnum;
if ((ge->high <= 0) || (ge->high < artnum))
ge->high = artnum;
ge->curindexoffset++;
ge->curoffset += len;
ge->count++;
return true;
}
bool buffindexed_add(char *group, ARTNUM artnum, TOKEN token, char *data, int len, time_t arrived, time_t expires) {
GROUPLOC gloc;
GROUPENTRY *ge;
if (len > OV_BLOCKSIZE) {
syslog(L_ERROR, "%s: overview data is too large %d", LocalLogName, len);
return true;
}
gloc = GROUPfind(group, false);
if (GROUPLOCempty(gloc)) {
return true;
}
GROUPlock(gloc, INN_LOCK_WRITE);
/* prepend block(s) if needed. */
ge = &GROUPentries[gloc.recno];
if (Cutofflow && ge->low > artnum) {
GROUPlock(gloc, INN_LOCK_UNLOCK);
return true;
}
#ifdef OV_DEBUG
if (!ovaddrec(ge, artnum, token, data, len, arrived, expires, NULL)) {
#else
if (!ovaddrec(ge, artnum, token, data, len, arrived, expires)) {
#endif /* OV_DEBUG */
if (Nospace) {
GROUPlock(gloc, INN_LOCK_UNLOCK);
syslog(L_ERROR, "%s: no space left for buffer, adding '%s'", LocalLogName, group);
return false;
}
syslog(L_ERROR, "%s: could not add overview for '%s'", LocalLogName, group);
}
GROUPlock(gloc, INN_LOCK_UNLOCK);
return true;
}
bool buffindexed_cancel(TOKEN token UNUSED) {
return true;
}
#ifdef OV_DEBUG
static void freegroupblock(GROUPENTRY *ge) {
#else
static void freegroupblock(void) {
#endif /* OV_DEBUG */
GROUPDATABLOCK *gdb;
int i;
GIBLIST *giblist;
for (giblist = Giblist ; giblist != NULL ; giblist = giblist->next) {
#ifdef OV_DEBUG
ovblockfree(giblist->ov, ge);
#else
ovblockfree(giblist->ov);
#endif /* OV_DEBUG */
}
for (i = 0 ; i < GROUPDATAHASHSIZE ; i++) {
for (gdb = groupdatablock[i] ; gdb != NULL ; gdb = gdb->next) {
#ifdef OV_DEBUG
ovblockfree(gdb->datablk, ge);
#else
ovblockfree(gdb->datablk);
#endif /* OV_DEBUG */
}
}
}
static void ovgroupunmap(void) {
GROUPDATABLOCK *gdb, *gdbnext;
int i;
GIBLIST *giblist, *giblistnext;
for (i = 0 ; i < GROUPDATAHASHSIZE ; i++) {
for (gdb = groupdatablock[i] ; gdb != NULL ; gdb = gdbnext) {
gdbnext = gdb->next;
free(gdb);
}
groupdatablock[i] = NULL;
}
for (giblist = Giblist ; giblist != NULL ; giblist = giblistnext) {
giblistnext = giblist->next;
free(giblist);
}
Giblist = NULL;
if (!Cache && (Gib != NULL)) {
free(Gib);
Gib = NULL;
if (Cachesearch != NULL) {
free(Cachesearch->group);
free(Cachesearch);
Cachesearch = NULL;
}
}
}
static void insertgdb(OV *ov, GROUPDATABLOCK *gdb) {
gdb->next = groupdatablock[(ov->index + ov->blocknum) % GROUPDATAHASHSIZE];
groupdatablock[(ov->index + ov->blocknum) % GROUPDATAHASHSIZE] = gdb;
return;
}
static GROUPDATABLOCK *searchgdb(OV *ov) {
GROUPDATABLOCK *gdb;
gdb = groupdatablock[(ov->index + ov->blocknum) % GROUPDATAHASHSIZE];
for (; gdb != NULL ; gdb = gdb->next) {
if (ov->index == gdb->datablk.index && ov->blocknum == gdb->datablk.blocknum)
break;
}
return gdb;
}
static int INDEXcompare(const void *p1, const void *p2) {
OVINDEX *oi1;
OVINDEX *oi2;
oi1 = (OVINDEX *)p1;
oi2 = (OVINDEX *)p2;
return oi1->artnum - oi2->artnum;
}
static bool ovgroupmmap(GROUPENTRY *ge, int low, int high, bool needov) {
OV ov = ge->baseindex;
OVBUFF *ovbuff;
GROUPDATABLOCK *gdb;
int pagefudge, limit, i, count, len;
off_t offset, mmapoffset;
OVBLOCK *ovblock;
void * addr;
GIBLIST *giblist;
if (high - low < 0) {
Gibcount = 0;
return true;
}
Gibcount = ge->count;
if (Gibcount == 0)
return true;
Gib = xmalloc(Gibcount * sizeof(OVINDEX));
count = 0;
while (ov.index != NULLINDEX) {
ovbuff = getovbuff(ov);
if (ovbuff == NULL) {
syslog(L_ERROR, "%s: ovgroupmmap ovbuff is null(ovindex is %d, ovblock is %d", LocalLogName, ov.index, ov.blocknum);
ovgroupunmap();
return false;
}
offset = ovbuff->base + (ov.blocknum * OV_BLOCKSIZE);
pagefudge = offset % pagesize;
mmapoffset = offset - pagefudge;
len = pagefudge + OV_BLOCKSIZE;
if ((addr = mmap(NULL, len, PROT_READ, MAP_SHARED, ovbuff->fd, mmapoffset)) == MAP_FAILED) {
syslog(L_ERROR, "%s: ovgroupmmap could not mmap index block: %m", LocalLogName);
ovgroupunmap();
return false;
}
ovblock = (OVBLOCK *)((char *)addr + pagefudge);
if (ov.index == ge->curindex.index && ov.blocknum == ge->curindex.blocknum) {
limit = ge->curindexoffset;
} else {
limit = OVINDEXMAX;
}
for (i = 0 ; i < limit ; i++) {
if (Gibcount == count) {
Gibcount += OV_FUDGE;
Gib = xrealloc(Gib, Gibcount * sizeof(OVINDEX));
}
Gib[count++] = ovblock->ovindex[i];
}
giblist = xmalloc(sizeof(GIBLIST));
giblist->ov = ov;
giblist->next = Giblist;
Giblist = giblist;
ov = ovblock->ovindexhead.next;
munmap(addr, len);
}
Gibcount = count;
qsort(Gib, Gibcount, sizeof(OVINDEX), INDEXcompare);
/* Remove duplicates. */
for (i = 0; i < Gibcount - 1; i++) {
if (Gib[i].artnum == Gib[i+1].artnum) {
/* lower position is removed */
Gib[i].artnum = 0;
}
}
if (!needov)
return true;
count = 0;
for (i = 0 ; i < Gibcount ; i++) {
if (Gib[i].artnum == 0 || Gib[i].artnum < low || Gib[i].artnum > high)
continue;
ov.index = Gib[i].index;
ov.blocknum = Gib[i].blocknum;
gdb = searchgdb(&ov);
if (gdb != NULL)
continue;
ovbuff = getovbuff(ov);
if (ovbuff == NULL)
continue;
gdb = xmalloc(sizeof(GROUPDATABLOCK));
gdb->datablk = ov;
gdb->next = NULL;
gdb->mmapped = false;
insertgdb(&ov, gdb);
count++;
}
if (count == 0)
return true;
if (count * OV_BLOCKSIZE > innconf->keepmmappedthreshold * 1024)
/* large retrieval, mmap is done in ovsearch() */
return true;
for (i = 0 ; i < GROUPDATAHASHSIZE ; i++) {
for (gdb = groupdatablock[i] ; gdb != NULL ; gdb = gdb->next) {
ov = gdb->datablk;
ovbuff = getovbuff(ov);
offset = ovbuff->base + (ov.blocknum * OV_BLOCKSIZE);
pagefudge = offset % pagesize;
mmapoffset = offset - pagefudge;
gdb->len = pagefudge + OV_BLOCKSIZE;
if ((gdb->addr = mmap(NULL, gdb->len, PROT_READ, MAP_SHARED, ovbuff->fd, mmapoffset)) == MAP_FAILED) {
syslog(L_ERROR, "%s: ovgroupmmap could not mmap data block: %m", LocalLogName);
free(gdb);
ovgroupunmap();
return false;
}
gdb->data = (char *)gdb->addr + pagefudge;
gdb->mmapped = true;
}
}
return true;
}
static void *ovopensearch(char *group, int low, int high, bool needov) {
GROUPLOC gloc;
GROUPENTRY *ge;
OVSEARCH *search;
gloc = GROUPfind(group, false);
if (GROUPLOCempty(gloc))
return NULL;
ge = &GROUPentries[gloc.recno];
if (low < ge->low)
low = ge->low;
if (high > ge->high)
high = ge->high;
if (!ovgroupmmap(ge, low, high, needov)) {
return NULL;
}
search = xmalloc(sizeof(OVSEARCH));
search->hi = high;
search->lo = low;
search->cur = 0;
search->group = xstrdup(group);
search->needov = needov;
search->gloc = gloc;
search->count = ge->count;
search->gdb.mmapped = false;
return (void *)search;
}
void *buffindexed_opensearch(char *group, int low, int high) {
GROUPLOC gloc;
void *handle;
if (Gib != NULL) {
free(Gib);
Gib = NULL;
if (Cachesearch != NULL) {
free(Cachesearch->group);
free(Cachesearch);
Cachesearch = NULL;
}
}
gloc = GROUPfind(group, false);
if (GROUPLOCempty(gloc)) {
return NULL;
}
GROUPlock(gloc, INN_LOCK_WRITE);
if ((handle = ovopensearch(group, low, high, true)) == NULL)
GROUPlock(gloc, INN_LOCK_UNLOCK);
return(handle);
}
static bool ovsearch(void *handle, ARTNUM *artnum, char **data, int *len, TOKEN *token, time_t *arrived, time_t *expires) {
OVSEARCH *search = (OVSEARCH *)handle;
OV srchov;
GROUPDATABLOCK *gdb;
off_t offset, mmapoffset;
OVBUFF *ovbuff;
int pagefudge;
bool newblock;
if (search->cur == Gibcount) {
return false;
}
while (Gib[search->cur].artnum == 0 || Gib[search->cur].artnum < search->lo) {
search->cur++;
if (search->cur == Gibcount)
return false;
}
if (Gib[search->cur].artnum > search->hi)
return false;
if (search->needov) {
if (Gib[search->cur].index == NULLINDEX) {
if (len)
*len = 0;
if (artnum)
*artnum = Gib[search->cur].artnum;
} else {
if (artnum)
*artnum = Gib[search->cur].artnum;
if (len)
*len = Gib[search->cur].len;
if (arrived)
*arrived = Gib[search->cur].arrived;
if (expires)
*expires = Gib[search->cur].expires;
if (data) {
srchov.index = Gib[search->cur].index;
srchov.blocknum = Gib[search->cur].blocknum;
gdb = searchgdb(&srchov);
if (gdb == NULL) {
if (len)
*len = 0;
search->cur++;
return true;
}
if (!gdb->mmapped) {
/* block needs to be mmapped */
if (search->gdb.mmapped) {
/* check previous mmapped area */
if (search->gdb.datablk.blocknum != srchov.blocknum || search->gdb.datablk.index != srchov.index) {
/* different one, release previous one */
munmap(search->gdb.addr, search->gdb.len);
newblock = true;
} else
newblock = false;
} else
newblock = true;
if (newblock) {
search->gdb.datablk.blocknum = srchov.blocknum;
search->gdb.datablk.index = srchov.index;
ovbuff = getovbuff(srchov);
offset = ovbuff->base + (srchov.blocknum * OV_BLOCKSIZE);
pagefudge = offset % pagesize;
mmapoffset = offset - pagefudge;
search->gdb.len = pagefudge + OV_BLOCKSIZE;
if ((search->gdb.addr = mmap(NULL, search->gdb.len, PROT_READ, MAP_SHARED, ovbuff->fd, mmapoffset)) == MAP_FAILED) {
syslog(L_ERROR, "%s: ovsearch could not mmap data block: %m", LocalLogName);
return false;
}
gdb->data = search->gdb.data = (char *)search->gdb.addr + pagefudge;
search->gdb.mmapped = true;
}
}
*data = (char *)gdb->data + Gib[search->cur].offset;
}
}
}
if (token) {
if (Gib[search->cur].index == NULLINDEX && !search->needov) {
search->cur++;
return false;
}
*token = Gib[search->cur].token;
}
search->cur++;
return true;
}
bool buffindexed_search(void *handle, ARTNUM *artnum, char **data, int *len, TOKEN *token, time_t *arrived) {
return(ovsearch(handle, artnum, data, len, token, arrived, NULL));
}
static void ovclosesearch(void *handle, bool freeblock) {
OVSEARCH *search = (OVSEARCH *)handle;
GROUPDATABLOCK *gdb;
int i;
#ifdef OV_DEBUG
GROUPENTRY *ge;
GROUPLOC gloc;
#endif /* OV_DEBUG */
for (i = 0 ; i < GROUPDATAHASHSIZE ; i++) {
for (gdb = groupdatablock[i] ; gdb != NULL ; gdb = gdb->next) {
if (gdb->mmapped)
munmap(gdb->addr, gdb->len);
}
}
if (search->gdb.mmapped)
munmap(search->gdb.addr, search->gdb.len);
if (freeblock) {
#ifdef OV_DEBUG
gloc = GROUPfind(search->group, false);
if (!GROUPLOCempty(gloc)) {
ge = &GROUPentries[gloc.recno];
freegroupblock(ge);
}
#else
freegroupblock();
#endif /* OV_DEBUG */
}
ovgroupunmap();
if (Cache) {
Cachesearch = search;
} else {
free(search->group);
free(search);
}
return;
}
void buffindexed_closesearch(void *handle) {
OVSEARCH *search = (OVSEARCH *)handle;
GROUPLOC gloc;
gloc = search->gloc;
ovclosesearch(handle, false);
GROUPlock(gloc, INN_LOCK_UNLOCK);
}
/* get token from sorted index */
static bool gettoken(ARTNUM artnum, TOKEN *token) {
int i, j, offset, limit;
offset = 0;
limit = Gibcount;
for (i = (limit - offset) / 2 ; i > 0 ; i = (limit - offset) / 2) {
if (Gib[offset + i].artnum == artnum) {
*token = Gib[offset + i].token;
return true;
} else if (Gib[offset + i].artnum == 0) {
/* case for duplicated index */
for (j = offset + i - 1; j >= offset ; j --) {
if (Gib[j].artnum != 0)
break;
}
if (j < offset) {
/* article not found */
return false;
}
if (Gib[j].artnum == artnum) {
*token = Gib[j].token;
return true;
} else if (Gib[j].artnum < artnum) {
/* limit is not changed */
offset += i + 1;
} else {
/* offset is not changed */
limit = j;
}
} else if (Gib[offset + i].artnum < artnum) {
/* limit is unchanged */
offset += i + 1;
} else {
/* offset is unchanged */
limit = offset + i;
}
}
/* i == 0 */
if (Gib[offset].artnum != artnum) {
/* article not found */
return false;
}
*token = Gib[offset].token;
return true;
}
bool buffindexed_getartinfo(char *group, ARTNUM artnum, TOKEN *token) {
GROUPLOC gloc;
void *handle;
bool retval, grouplocked = false;
if (Gib != NULL) {
if (Cachesearch != NULL && strcmp(Cachesearch->group, group) != 0) {
free(Gib);
Gib = NULL;
free(Cachesearch->group);
free(Cachesearch);
Cachesearch = NULL;
} else {
if (gettoken(artnum, token))
return true;
else {
/* examine to see if overview index are increased */
gloc = GROUPfind(group, false);
if (GROUPLOCempty(gloc)) {
return false;
}
GROUPlock(gloc, INN_LOCK_WRITE);
if ((Cachesearch != NULL) && (GROUPentries[gloc.recno].count == Cachesearch->count)) {
/* no new overview data is stored */
GROUPlock(gloc, INN_LOCK_UNLOCK);
return false;
} else {
grouplocked = true;
free(Gib);
Gib = NULL;
if (Cachesearch != NULL) {
free(Cachesearch->group);
free(Cachesearch);
Cachesearch = NULL;
}
}
}
}
}
if (!grouplocked) {
gloc = GROUPfind(group, false);
if (GROUPLOCempty(gloc)) {
return false;
}
GROUPlock(gloc, INN_LOCK_WRITE);
}
if (!(handle = ovopensearch(group, artnum, artnum, false))) {
GROUPlock(gloc, INN_LOCK_UNLOCK);
return false;
}
retval = buffindexed_search(handle, NULL, NULL, NULL, token, NULL);
ovclosesearch(handle, false);
GROUPlock(gloc, INN_LOCK_UNLOCK);
return retval;
}
bool buffindexed_expiregroup(char *group, int *lo, struct history *h) {
void *handle;
GROUPENTRY newge, *ge;
GROUPLOC gloc, next;
char *data;
int i, j, len;
TOKEN token;
ARTNUM artnum, low, high;
ARTHANDLE *ah;
char flag;
HASH hash;
time_t arrived, expires;
if (group == NULL) {
for (i = 0 ; i < GROUPheader->freelist.recno ; i++) {
gloc.recno = i;
GROUPlock(gloc, INN_LOCK_WRITE);
ge = &GROUPentries[gloc.recno];
if (ge->expired >= OVrealnow || ge->count == 0) {
GROUPlock(gloc, INN_LOCK_UNLOCK);
continue;
}
if (!ovgroupmmap(ge, ge->low, ge->high, true)) {
GROUPlock(gloc, INN_LOCK_UNLOCK);
syslog(L_ERROR, "%s: could not mmap overview for hidden groups(%d)", LocalLogName, i);
continue;
}
for (j = 0 ; j < Gibcount ; j++) {
if (Gib[j].artnum == 0)
continue;
/* this may be duplicated, but ignore it in this case */
OVEXPremove(Gib[j].token, true, NULL, 0);
}
#ifdef OV_DEBUG
freegroupblock(ge);
#else
freegroupblock();
#endif
ovgroupunmap();
ge->expired = time(NULL);
ge->count = 0;
GROUPlock(gloc, INN_LOCK_UNLOCK);
}
return true;
}
gloc = GROUPfind(group, false);
if (GROUPLOCempty(gloc)) {
return false;
}
GROUPlock(gloc, INN_LOCK_WRITE);
ge = &GROUPentries[gloc.recno];
if (ge->count == 0) {
if (lo)
*lo = ge->low;
ge->expired = time(NULL);
GROUPlock(gloc, INN_LOCK_UNLOCK);
return true;
}
flag = ge->flag;
hash = ge->hash;
next = ge->next;
low = ge->low;
high = ge->high;
newge.low = 0;
setinitialge(&newge, hash, &flag, next, 0, high);
if ((handle = ovopensearch(group, low, high, true)) == NULL) {
ge->expired = time(NULL);
GROUPlock(gloc, INN_LOCK_UNLOCK);
syslog(L_ERROR, "%s: could not open overview for '%s'", LocalLogName, group);
return false;
}
while (ovsearch(handle, &artnum, &data, &len, &token, &arrived, &expires)) {
ah = NULL;
if (len == 0)
continue;
if (!SMprobe(EXPENSIVESTAT, &token, NULL) || OVstatall) {
if ((ah = SMretrieve(token, RETR_STAT)) == NULL)
continue;
SMfreearticle(ah);
} else {
if (!OVhisthasmsgid(h, data))
continue;
}
if (innconf->groupbaseexpiry && OVgroupbasedexpire(token, group, data, len, arrived, expires))
continue;
#ifdef OV_DEBUG
if (!ovaddrec(&newge, artnum, token, data, len, arrived, expires, ge)) {
#else
if (!ovaddrec(&newge, artnum, token, data, len, arrived, expires)) {
#endif /* OV_DEBUG */
ovclosesearch(handle, true);
ge->expired = time(NULL);
GROUPlock(gloc, INN_LOCK_UNLOCK);
syslog(L_ERROR, "%s: could not add new overview for '%s'", LocalLogName, group);
return false;
}
}
if (newge.low == 0)
/* no article for the group */
newge.low = newge.high;
*ge = newge;
if (lo) {
if (ge->count == 0)
/* lomark should be himark + 1, if no article for the group */
*lo = ge->low + 1;
else
*lo = ge->low;
}
ovclosesearch(handle, true);
ge->expired = time(NULL);
GROUPlock(gloc, INN_LOCK_UNLOCK);
return true;
}
bool buffindexed_ctl(OVCTLTYPE type, void *val) {
int total, used, *i, j;
OVBUFF *ovbuff = ovbufftab;
OVSORTTYPE *sorttype;
bool *boolval;
GROUPDATABLOCK *gdb;
switch (type) {
case OVSPACE:
for (total = used = 0 ; ovbuff != (OVBUFF *)NULL ; ovbuff = ovbuff->next) {
ovlock(ovbuff, INN_LOCK_READ);
ovreadhead(ovbuff);
total += ovbuff->totalblk;
used += ovbuff->usedblk;
ovlock(ovbuff, INN_LOCK_UNLOCK);
}
i = (int *)val;
*i = (used * 100) / total;
return true;
case OVSORT:
sorttype = (OVSORTTYPE *)val;
*sorttype = OVNOSORT;
return true;
case OVCUTOFFLOW:
Cutofflow = *(bool *)val;
return true;
case OVSTATICSEARCH:
i = (int *)val;
*i = true;
for (j = 0 ; j < GROUPDATAHASHSIZE ; j++) {
for (gdb = groupdatablock[j] ; gdb != NULL ; gdb = gdb->next) {
if (gdb->mmapped) {
*i = false;
return true;
}
}
}
return true;
case OVCACHEKEEP:
Cache = *(bool *)val;
return true;
case OVCACHEFREE:
boolval = (bool *)val;
*boolval = true;
if (Gib != NULL) {
free(Gib);
Gib = NULL;
if (Cachesearch != NULL) {
free(Cachesearch->group);
free(Cachesearch);
Cachesearch = NULL;
}
}
return true;
default:
return false;
}
}
void buffindexed_close(void) {
struct stat sb;
OVBUFF *ovbuffnext, *ovbuff = ovbufftab;
#ifdef OV_DEBUG
FILE *F = NULL;
pid_t pid;
char *path = NULL;
int i,j;
struct ov_trace_array *trace;
struct ov_name_table *ntp;
size_t length;
#endif /* OV_DEBUG */
#ifdef OV_DEBUG
for (; ovbuff != (OVBUFF *)NULL; ovbuff = ovbuff->next) {
for (i = 0 ; i < ovbuff->totalblk ; i++) {
trace = &ovbuff->trace[i];
if (trace->ov_trace == NULL)
continue;
for (j = 0 ; j <= trace->cur && j < trace->max ; j++) {
if (trace->ov_trace[j].occupied != 0 ||
trace->ov_trace[j].freed != 0) {
if (F == NULL) {
length = strlen(innconf->pathtmp) + 11;
path = xmalloc(length);
pid = getpid();
snprintf(path, length, "%s/%d", innconf->pathtmp, pid);
if ((F = fopen(path, "w")) == NULL) {
syslog(L_ERROR, "%s: could not open %s: %m", LocalLogName, path);
break;
}
}
fprintf(F, "%d: % 6d, % 2d: 0x%08x, % 10d, % 10d\n", ovbuff->index, i, j,
trace->ov_trace[j].gloc.recno,
trace->ov_trace[j].occupied,
trace->ov_trace[j].freed);
}
}
}
}
if ((ntp = name_table) != NULL) {
if (F == NULL) {
length = strlen(innconf->pathtmp) + 11;
path = xmalloc(length);
pid = getpid();
sprintf(path, length, "%s/%d", innconf->pathtmp, pid);
if ((F = fopen(path, "w")) == NULL) {
syslog(L_ERROR, "%s: could not open %s: %m", LocalLogName, path);
}
}
if (F != NULL) {
while(ntp) {
fprintf(F, "0x%08x: %s\n", ntp->recno, ntp->name);
ntp = ntp->next;
}
}
}
if (F != NULL)
fclose(F);
if (path != NULL)
free(path);
#endif /* OV_DEBUG */
if (Gib != NULL) {
free(Gib);
Gib = NULL;
if (Cachesearch != NULL) {
free(Cachesearch->group);
free(Cachesearch);
Cachesearch = NULL;
}
}
if (fstat(GROUPfd, &sb) < 0)
return;
close(GROUPfd);
if (GROUPheader) {
if (munmap((void *)GROUPheader, GROUPfilesize(GROUPcount)) < 0) {
syslog(L_FATAL, "%s: could not munmap group.index in buffindexed_close: %m", LocalLogName);
return;
}
}
for (; ovbuff != (OVBUFF *)NULL; ovbuff = ovbuffnext) {
ovbuffnext = ovbuff->next;
free(ovbuff);
}
ovbufftab = NULL;
}
#ifdef BUFF_DEBUG
static int countgdb(void) {
int i, count = 0;
GROUPDATABLOCK *gdb;
for (i = 0 ; i < GROUPDATAHASHSIZE ; i++) {
for (gdb = groupdatablock[i] ; gdb != NULL ; gdb = gdb->next)
count++;
}
return count;
}
main(int argc, char **argv) {
char *group, flag[2], buff[OV_BLOCKSIZE];
int lo, hi, count, flags, i;
OVSEARCH *search;
OVBLOCK *ovblock;
OVINDEX *ovindex;
OVBUFF *ovbuff;
GROUPENTRY *ge;
GROUPLOC gloc;
GIBLIST *giblist;
if (argc != 2) {
fprintf(stderr, "only one argument can be specified\n");
exit(1);
}
/* if innconf isn't already read in, do so. */
if (innconf == NULL) {
if (!innconf_read(NULL)) {
fprintf(stderr, "reading inn.conf failed\n");
exit(1);
}
}
if (!buffindexed_open(OV_READ)) {
fprintf(stderr, "buffindexed_open failed\n");
exit(1);
}
fprintf(stdout, "GROUPheader->freelist.recno is %d(0x%08x)\n", GROUPheader->freelist.recno, GROUPheader->freelist.recno);
group = argv[1];
if (isdigit(*group)) {
gloc.recno = atoi(group);
ge = &GROUPentries[gloc.recno];
fprintf(stdout, "left articles are %d for %d, last expiry is %d\n", ge->count, gloc.recno, ge->expired);
if (ge->count == 0) {
GROUPlock(gloc, INN_LOCK_UNLOCK);
exit(0);
}
if (!ovgroupmmap(ge, ge->low, ge->high, true)) {
fprintf(stderr, "ovgroupmmap failed\n");
GROUPlock(gloc, INN_LOCK_UNLOCK);
}
for (giblist = Giblist, i = 0 ; giblist != NULL ; giblist = giblist->next, i++);
fprintf(stdout, "%d index block(s)\n", i);
fprintf(stdout, "%d data block(s)\n", countgdb());
for (giblist = Giblist ; giblist != NULL ; giblist = giblist->next) {
fprintf(stdout, " % 8d(% 5d)\n", giblist->ov.blocknum, giblist->ov.index);
}
for (i = 0 ; i < Gibcount ; i++) {
if (Gib[i].artnum == 0)
continue;
if (Gib[i].index == NULLINDEX)
fprintf(stdout, " %d empty\n");
else {
fprintf(stdout, " %d %d\n", Gib[i].offset, Gib[i].len);
}
}
ovgroupunmap();
GROUPlock(gloc, INN_LOCK_UNLOCK);
exit(0);
}
gloc = GROUPfind(group, false);
if (GROUPLOCempty(gloc)) {
fprintf(stderr, "gloc is null\n");
}
GROUPlock(gloc, INN_LOCK_READ);
ge = &GROUPentries[gloc.recno];
fprintf(stdout, "base %d(%d), cur %d(%d), expired at %s\n", ge->baseindex.blocknum, ge->baseindex.index, ge->curindex.blocknum, ge->curindex.index, ge->expired == 0 ? "none\n" : ctime(&ge->expired));
if (!buffindexed_groupstats(group, &lo, &hi, &count, &flags)) {
fprintf(stderr, "buffindexed_groupstats failed for group %s\n", group);
exit(1);
}
flag[0] = (char)flags;
flag[1] = '\0';
fprintf(stdout, "%s: low is %d, high is %d, count is %d, flag is '%s'\n", group, lo, hi, count, flag);
if ((search = (OVSEARCH *)ovopensearch(group, lo, hi, true)) == NULL) {
fprintf(stderr, "ovopensearch failed for group %s\n", group);
exit(1);
}
fprintf(stdout, " gloc is %d(0x%08x)\n", search->gloc.recno, search->gloc.recno);
for (giblist = Giblist, i = 0 ; giblist != NULL ; giblist = giblist->next, i++);
fprintf(stdout, "%d index block(s)\n", i);
fprintf(stdout, "%d data block(s)\n", countgdb());
for (giblist = Giblist ; giblist != NULL ; giblist = giblist->next) {
fprintf(stdout, " % 8d(% 5d)\n", giblist->ov.blocknum, giblist->ov.index);
}
for (i = 0 ; i < Gibcount ; i++) {
if (Gib[i].artnum == 0)
continue;
if (Gib[i].index == NULLINDEX)
fprintf(stdout, " %d empty\n");
else {
fprintf(stdout, " %d %d\n", Gib[i].offset, Gib[i].len);
}
}
{
ARTNUM artnum;
char *data;
int len;
TOKEN token;
while (buffindexed_search((void *)search, &artnum, &data, &len, &token, NULL)) {
if (len == 0)
fprintf(stdout, "%d: len is 0\n", artnum);
else {
memcpy(buff, data, len);
buff[len] = '\0';
fprintf(stdout, "%d: %s\n", artnum, buff);
}
}
}
}
#endif /* BUFF_DEBUG */
syntax highlighted by Code2HTML, v. 0.9.1