index.c

#include "index.h"
#include "similarity.h"
#include "helper.h"
#include "array.h"
#include "priorityqueue.h"
#include <string.h>
#include <limits.h>
#include <ctype.h>

#define GET_LOCK(lock, name, store, err_msg) do {\
    lock = store->open_lock(store, name);\
    if (!lock->obtain(lock)) {\
        RAISE(LOCK_ERROR, err_msg);\
    }\
} while(0)

#define RELEASE_LOCK(lock, store) do {\
    lock->release(lock);\
    store->close_lock(lock);\
} while (0)

const Config default_config = {
    0x100000,       /* chunk size is 1Mb */
    0x1000000,      /* Max memory used for buffer is 16 Mb */
    INDEX_INTERVAL, /* index interval */
    SKIP_INTERVAL,  /* skip interval */
    10,             /* default merge factor */
    10000,          /* max_buffered_docs */
    INT_MAX,        /* max_merge_docs */
    10000,          /* maximum field length (number of terms) */
    true            /* use compound file by default */
};

static void ste_reset(TermEnum *te);
static char *ste_next(TermEnum *te);

#define FORMAT 0
#define SEGMENTS_GEN_FILE_NAME "segments"
#define MAX_EXT_LEN 10

/* *** Must be three characters *** */
const char *INDEX_EXTENSIONS[] = {
    "frq", "prx", "fdx", "fdt", "tfx", "tix", "tis", "del", "gen", "cfs"
};

/* *** Must be three characters *** */
const char *COMPOUND_EXTENSIONS[] = {
    "frq", "prx", "fdx", "fdt", "tfx", "tix", "tis"
};


static const char BASE36_DIGITMAP[] = "0123456789abcdefghijklmnopqrstuvwxyz";

static char *u64_to_str36(char *buf, int buf_size, f_u64 u)
{
    int i = buf_size - 1;
    buf[i] = '\0';
    for (i--; i >= 0; i--) {
        buf[i] = BASE36_DIGITMAP[u % 36];
        u /= 36;
        if (0 == u) {
            break;
        }
    }
    if (0 < u) {
        RAISE(EXCEPTION, "Max length of segment filename has been reached. "
              "Perhaps it's time to re-index.\n");
    }
    return buf + i;
}