Commit 4a859c17 authored by Jean-Francois Dockes's avatar Jean-Francois Dockes
Browse files

trcircache web cache mgmt utility/test driver: add option to append entries from other file

--HG--
branch : RECOLL_1_21_MAINT
parent 89e21cf9
......@@ -111,14 +111,14 @@ public:
UCHAR h[UDIHLEN];
UdiH(const string& udi)
{
MD5_CTX ctx;
MD5Init(&ctx);
MD5Update(&ctx, (const UCHAR*)udi.c_str(), udi.length());
UCHAR md[16];
MD5Final(md, &ctx);
memcpy(h, md, UDIHLEN);
}
{
MD5_CTX ctx;
MD5Init(&ctx);
MD5Update(&ctx, (const UCHAR*)udi.c_str(), udi.length());
UCHAR md[16];
MD5Final(md, &ctx);
memcpy(h, md, UDIHLEN);
}
string asHexString() const {
static const char hex[]="0123456789abcdef";
......@@ -130,22 +130,22 @@ public:
return out;
}
bool operator==(const UdiH& r) const
{
for (int i = 0; i < UDIHLEN; i++)
if (h[i] != r.h[i])
return false;
return true;
}
{
for (int i = 0; i < UDIHLEN; i++)
if (h[i] != r.h[i])
return false;
return true;
}
bool operator<(const UdiH& r) const
{
for (int i = 0; i < UDIHLEN; i++) {
if (h[i] < r.h[i])
return true;
if (h[i] > r.h[i])
return false;
{
for (int i = 0; i < UDIHLEN; i++) {
if (h[i] < r.h[i])
return true;
if (h[i] > r.h[i])
return false;
}
return false;
}
return false;
}
};
typedef multimap<UdiH, off_t> kh_type;
typedef multimap<UdiH, off_t>::value_type kh_value_type;
......@@ -185,415 +185,415 @@ public:
// Add udi->offset translation to map
bool khEnter(const string& udi, off_t ofs)
{
UdiH h(udi);
LOGDEB2(("Circache::khEnter: h %s offs %lu udi [%s]\n",
h.asHexString().c_str(), (ULONG)ofs, udi.c_str()));
pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);
if (p.first != m_ofskh.end() && p.first->first == h) {
for (kh_type::iterator it = p.first; it != p.second; it++) {
LOGDEB2(("Circache::khEnter: col h %s, ofs %lu\n",
it->first.asHexString().c_str(),
(ULONG)it->second));
if (it->second == ofs) {
// (h,offs) already there. Happens
LOGDEB2(("Circache::khEnter: already there\n"));
return true;
{
UdiH h(udi);
LOGDEB2(("Circache::khEnter: h %s offs %lu udi [%s]\n",
h.asHexString().c_str(), (ULONG)ofs, udi.c_str()));
pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);
if (p.first != m_ofskh.end() && p.first->first == h) {
for (kh_type::iterator it = p.first; it != p.second; it++) {
LOGDEB2(("Circache::khEnter: col h %s, ofs %lu\n",
it->first.asHexString().c_str(),
(ULONG)it->second));
if (it->second == ofs) {
// (h,offs) already there. Happens
LOGDEB2(("Circache::khEnter: already there\n"));
return true;
}
}
}
m_ofskh.insert(kh_value_type(h, ofs));
LOGDEB2(("Circache::khEnter: inserted\n"));
return true;
}
m_ofskh.insert(kh_value_type(h, ofs));
LOGDEB2(("Circache::khEnter: inserted\n"));
return true;
}
void khDump()
{
for (kh_type::const_iterator it = m_ofskh.begin();
it != m_ofskh.end(); it++) {
LOGDEB(("Circache::KHDUMP: %s %d\n",
it->first.asHexString().c_str(), (ULONG)it->second));
{
for (kh_type::const_iterator it = m_ofskh.begin();
it != m_ofskh.end(); it++) {
LOGDEB(("Circache::KHDUMP: %s %d\n",
it->first.asHexString().c_str(), (ULONG)it->second));
}
}
}
// Return vector of candidate offsets for udi (possibly several
// because there may be hash collisions, and also multiple
// instances).
bool khFind(const string& udi, vector<off_t>& ofss)
{
ofss.clear();
{
ofss.clear();
UdiH h(udi);
UdiH h(udi);
LOGDEB2(("Circache::khFind: h %s udi [%s]\n",
h.asHexString().c_str(), udi.c_str()));
LOGDEB2(("Circache::khFind: h %s udi [%s]\n",
h.asHexString().c_str(), udi.c_str()));
pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);
pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);
#if 0
if (p.first == m_ofskh.end()) LOGDEB(("KHFIND: FIRST END()\n"));
if (p.second == m_ofskh.end()) LOGDEB(("KHFIND: SECOND END()\n"));
if (!(p.first->first == h))
LOGDEB(("KHFIND: NOKEY: %s %s\n",
p.first->first.asHexString().c_str(),
p.second->first.asHexString().c_str()));
if (p.first == m_ofskh.end()) LOGDEB(("KHFIND: FIRST END()\n"));
if (p.second == m_ofskh.end()) LOGDEB(("KHFIND: SECOND END()\n"));
if (!(p.first->first == h))
LOGDEB(("KHFIND: NOKEY: %s %s\n",
p.first->first.asHexString().c_str(),
p.second->first.asHexString().c_str()));
#endif
if (p.first == m_ofskh.end() || !(p.first->first == h))
return false;
if (p.first == m_ofskh.end() || !(p.first->first == h))
return false;
for (kh_type::iterator it = p.first; it != p.second; it++) {
ofss.push_back(it->second);
for (kh_type::iterator it = p.first; it != p.second; it++) {
ofss.push_back(it->second);
}
return true;
}
return true;
}
// Clear entry for udi/offs
bool khClear(const pair<string, off_t>& ref)
{
UdiH h(ref.first);
pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);
if (p.first != m_ofskh.end() && (p.first->first == h)) {
for (kh_type::iterator it = p.first; it != p.second; ) {
kh_type::iterator tmp = it++;
if (tmp->second == ref.second)
m_ofskh.erase(tmp);
{
UdiH h(ref.first);
pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);
if (p.first != m_ofskh.end() && (p.first->first == h)) {
for (kh_type::iterator it = p.first; it != p.second; ) {
kh_type::iterator tmp = it++;
if (tmp->second == ref.second)
m_ofskh.erase(tmp);
}
}
return true;
}
return true;
}
// Clear entries for vector of udi/offs
bool khClear(const vector<pair<string, off_t> >& udis)
{
for (vector<pair<string, off_t> >::const_iterator it = udis.begin();
it != udis.end(); it++)
khClear(*it);
return true;
}
{
for (vector<pair<string, off_t> >::const_iterator it = udis.begin();
it != udis.end(); it++)
khClear(*it);
return true;
}
// Clear all entries for udi
bool khClear(const string& udi)
{
UdiH h(udi);
pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);
if (p.first != m_ofskh.end() && (p.first->first == h)) {
for (kh_type::iterator it = p.first; it != p.second; ) {
kh_type::iterator tmp = it++;
m_ofskh.erase(tmp);
{
UdiH h(udi);
pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);
if (p.first != m_ofskh.end() && (p.first->first == h)) {
for (kh_type::iterator it = p.first; it != p.second; ) {
kh_type::iterator tmp = it++;
m_ofskh.erase(tmp);
}
}
return true;
}
return true;
}
CirCacheInternal()
: m_fd(-1), m_maxsize(-1), m_oheadoffs(-1),
m_nheadoffs(0), m_npadsize(0), m_uniquentries(false),
m_buffer(0), m_bufsiz(0), m_ofskhcplt(false)
{}
{}
~CirCacheInternal()
{
if (m_fd >= 0)
close(m_fd);
if (m_buffer)
free(m_buffer);
}
{
if (m_fd >= 0)
close(m_fd);
if (m_buffer)
free(m_buffer);
}
char *buf(size_t sz)
{
if (m_bufsiz >= sz)
{
if (m_bufsiz >= sz)
return m_buffer;
if ((m_buffer = (char *)realloc(m_buffer, sz))) {
m_bufsiz = sz;
} else {
m_reason << "CirCache:: realloc(" << sz << ") failed";
m_bufsiz = 0;
}
return m_buffer;
if ((m_buffer = (char *)realloc(m_buffer, sz))) {
m_bufsiz = sz;
} else {
m_reason << "CirCache:: realloc(" << sz << ") failed";
m_bufsiz = 0;
}
return m_buffer;
}
// Name for the cache file
string datafn(const string& d)
{
return path_cat(d, "circache.crch");
}
{
return path_cat(d, "circache.crch");
}
bool writefirstblock()
{
if (m_fd < 0) {
m_reason << "writefirstblock: not open ";
return false;
}
ostringstream s;
s <<
"maxsize = " << m_maxsize << "\n" <<
"oheadoffs = " << m_oheadoffs << "\n" <<
"nheadoffs = " << m_nheadoffs << "\n" <<
"npadsize = " << m_npadsize << "\n" <<
"unient = " << m_uniquentries << "\n" <<
" " <<
" " <<
" " <<
"\0";
int sz = int(s.str().size());
assert(sz < CIRCACHE_FIRSTBLOCK_SIZE);
lseek(m_fd, 0, 0);
if (write(m_fd, s.str().c_str(), sz) != sz) {
m_reason << "writefirstblock: write() failed: errno " << errno;
return false;
{
if (m_fd < 0) {
m_reason << "writefirstblock: not open ";
return false;
}
ostringstream s;
s <<
"maxsize = " << m_maxsize << "\n" <<
"oheadoffs = " << m_oheadoffs << "\n" <<
"nheadoffs = " << m_nheadoffs << "\n" <<
"npadsize = " << m_npadsize << "\n" <<
"unient = " << m_uniquentries << "\n" <<
" " <<
" " <<
" " <<
"\0";
int sz = int(s.str().size());
assert(sz < CIRCACHE_FIRSTBLOCK_SIZE);
lseek(m_fd, 0, 0);
if (write(m_fd, s.str().c_str(), sz) != sz) {
m_reason << "writefirstblock: write() failed: errno " << errno;
return false;
}
return true;
}
return true;
}
bool readfirstblock()
{
if (m_fd < 0) {
m_reason << "readfirstblock: not open ";
return false;
}
char bf[CIRCACHE_FIRSTBLOCK_SIZE];
lseek(m_fd, 0, 0);
if (read(m_fd, bf, CIRCACHE_FIRSTBLOCK_SIZE) !=
CIRCACHE_FIRSTBLOCK_SIZE) {
m_reason << "readfirstblock: read() failed: errno " << errno;
return false;
}
string s(bf, CIRCACHE_FIRSTBLOCK_SIZE);
ConfSimple conf(s, 1);
string value;
if (!conf.get("maxsize", value, cstr_null)) {
m_reason << "readfirstblock: conf get maxsize failed";
return false;
}
m_maxsize = atoll(value.c_str());
if (!conf.get("oheadoffs", value, cstr_null)) {
m_reason << "readfirstblock: conf get oheadoffs failed";
return false;
}
m_oheadoffs = atoll(value.c_str());
if (!conf.get("nheadoffs", value, cstr_null)) {
m_reason << "readfirstblock: conf get nheadoffs failed";
return false;
}
m_nheadoffs = atoll(value.c_str());
if (!conf.get("npadsize", value, cstr_null)) {
m_reason << "readfirstblock: conf get npadsize failed";
return false;
}
m_npadsize = atoll(value.c_str());
if (!conf.get("unient", value, cstr_null)) {
m_uniquentries = false;
} else {
m_uniquentries = stringToBool(value);
}
return true;
}
{
if (m_fd < 0) {
m_reason << "readfirstblock: not open ";
return false;
}
char bf[CIRCACHE_FIRSTBLOCK_SIZE];
lseek(m_fd, 0, 0);
if (read(m_fd, bf, CIRCACHE_FIRSTBLOCK_SIZE) !=
CIRCACHE_FIRSTBLOCK_SIZE) {
m_reason << "readfirstblock: read() failed: errno " << errno;
return false;
}
string s(bf, CIRCACHE_FIRSTBLOCK_SIZE);
ConfSimple conf(s, 1);
string value;
if (!conf.get("maxsize", value, cstr_null)) {
m_reason << "readfirstblock: conf get maxsize failed";
return false;
}
m_maxsize = atoll(value.c_str());
if (!conf.get("oheadoffs", value, cstr_null)) {
m_reason << "readfirstblock: conf get oheadoffs failed";
return false;
}
m_oheadoffs = atoll(value.c_str());
if (!conf.get("nheadoffs", value, cstr_null)) {
m_reason << "readfirstblock: conf get nheadoffs failed";
return false;
}
m_nheadoffs = atoll(value.c_str());
if (!conf.get("npadsize", value, cstr_null)) {
m_reason << "readfirstblock: conf get npadsize failed";
return false;
}
m_npadsize = atoll(value.c_str());
if (!conf.get("unient", value, cstr_null)) {
m_uniquentries = false;
} else {
m_uniquentries = stringToBool(value);
}
return true;
}
bool writeEntryHeader(off_t offset, const EntryHeaderData& d)
{
if (m_fd < 0) {
m_reason << "writeEntryHeader: not open ";
return false;
}
char bf[CIRCACHE_HEADER_SIZE];
memset(bf, 0, CIRCACHE_HEADER_SIZE);
snprintf(bf, CIRCACHE_HEADER_SIZE,
headerformat, d.dicsize, d.datasize, d.padsize, d.flags);
if (lseek(m_fd, offset, 0) != offset) {
m_reason << "CirCache::weh: lseek(" << offset <<
") failed: errno " << errno;
return false;
}
if (write(m_fd, bf, CIRCACHE_HEADER_SIZE) != CIRCACHE_HEADER_SIZE) {
m_reason << "CirCache::weh: write failed. errno " << errno;
return false;
{
if (m_fd < 0) {
m_reason << "writeEntryHeader: not open ";
return false;
}
char bf[CIRCACHE_HEADER_SIZE];
memset(bf, 0, CIRCACHE_HEADER_SIZE);
snprintf(bf, CIRCACHE_HEADER_SIZE,
headerformat, d.dicsize, d.datasize, d.padsize, d.flags);
if (lseek(m_fd, offset, 0) != offset) {
m_reason << "CirCache::weh: lseek(" << offset <<
") failed: errno " << errno;
return false;
}
if (write(m_fd, bf, CIRCACHE_HEADER_SIZE) != CIRCACHE_HEADER_SIZE) {
m_reason << "CirCache::weh: write failed. errno " << errno;
return false;
}
return true;
}
return true;
}
CCScanHook::status readEntryHeader(off_t offset, EntryHeaderData& d)
{
if (m_fd < 0) {
m_reason << "readEntryHeader: not open ";
return CCScanHook::Error;
}
if (lseek(m_fd, offset, 0) != offset) {
m_reason << "readEntryHeader: lseek(" << offset <<
") failed: errno " << errno;
return CCScanHook::Error;
}
char bf[CIRCACHE_HEADER_SIZE];
{
if (m_fd < 0) {
m_reason << "readEntryHeader: not open ";
return CCScanHook::Error;
}
int ret = read(m_fd, bf, CIRCACHE_HEADER_SIZE);
if (ret == 0) {
// Eof
m_reason << " Eof ";
return CCScanHook::Eof;
}
if (ret != CIRCACHE_HEADER_SIZE) {
m_reason << " readheader: read failed errno " << errno;
return CCScanHook::Error;
}
if (sscanf(bf, headerformat, &d.dicsize, &d.datasize,
&d.padsize, &d.flags) != 4) {
m_reason << " readEntryHeader: bad header at " <<
offset << " [" << bf << "]";
return CCScanHook::Error;
if (lseek(m_fd, offset, 0) != offset) {
m_reason << "readEntryHeader: lseek(" << offset <<
") failed: errno " << errno;
return CCScanHook::Error;
}
char bf[CIRCACHE_HEADER_SIZE];
int ret = read(m_fd, bf, CIRCACHE_HEADER_SIZE);
if (ret == 0) {
// Eof
m_reason << " Eof ";
return CCScanHook::Eof;
}
if (ret != CIRCACHE_HEADER_SIZE) {
m_reason << " readheader: read failed errno " << errno;
return CCScanHook::Error;
}
if (sscanf(bf, headerformat, &d.dicsize, &d.datasize,
&d.padsize, &d.flags) != 4) {
m_reason << " readEntryHeader: bad header at " <<
offset << " [" << bf << "]";
return CCScanHook::Error;
}
LOGDEB2(("Circache:readEntryHeader: dcsz %u dtsz %u pdsz %u flgs %hu\n",
d.dicsize, d.datasize, d.padsize, d.flags));
return CCScanHook::Continue;
}
LOGDEB2(("Circache:readEntryHeader: dcsz %u dtsz %u pdsz %u flgs %hu\n",
d.dicsize, d.datasize, d.padsize, d.flags));
return CCScanHook::Continue;
}
CCScanHook::status scan(off_t startoffset, CCScanHook *user,
bool fold = false)
{
if (m_fd < 0) {
m_reason << "scan: not open ";
return CCScanHook::Error;
}
off_t so0 = startoffset;
bool already_folded = false;
while (true) {
if (already_folded && startoffset == so0) {
m_ofskhcplt = true;
return CCScanHook::Eof;
{
if (m_fd < 0) {
m_reason << "scan: not open ";
return CCScanHook::Error;
}
EntryHeaderData d;
CCScanHook::status st;
switch ((st = readEntryHeader(startoffset, d))) {
case CCScanHook::Continue: break;
case CCScanHook::Eof:
if (fold && !already_folded) {
already_folded = true;
startoffset = CIRCACHE_FIRSTBLOCK_SIZE;
continue;
}
/* FALLTHROUGH */
default:
return st;
}
string udi;
if (d.dicsize) {
// d.dicsize is 0 for erased entries
char *bf;
if ((bf = buf(d.dicsize+1)) == 0) {
return CCScanHook::Error;
off_t so0 = startoffset;
bool already_folded = false;
while (true) {
if (already_folded && startoffset == so0) {
m_ofskhcplt = true;
return CCScanHook::Eof;
}
bf[d.dicsize] = 0;
if (read(m_fd, bf, d.dicsize) != int(d.dicsize)) {
m_reason << "scan: read failed errno " << errno;
return CCScanHook::Error;
EntryHeaderData d;
CCScanHook::status st;
switch ((st = readEntryHeader(startoffset, d))) {
case CCScanHook::Continue: break;
case CCScanHook::Eof:
if (fold && !already_folded) {
already_folded = true;
startoffset = CIRCACHE_FIRSTBLOCK_SIZE;
continue;
}
/* FALLTHROUGH */
default:
return st;
}