Commit b2033861 authored by Frank de Lange's avatar Frank de Lange
Browse files

Add option to use more than one confdir (RECOLL_EXTRACONFDIRS)

This change adds the possibility to run queries over more than one database by pointing the program at extra recoll configuration directories using the RECOLL_EXTRACONFDIR environment variable. This variable can contain space-separated recoll configuration directories (i.e. directories which contain `recoll.conf`) which are parsed to find out the indexed *topdirs* and the location of the database directory. The _topdirs_ are added to the directory tree, the databases are added to the `extradbs` list. When running a query over the entire tree (using `<all>`, the default) all databases are searched. When the query is limited to a subdirectory the searched set is limited to only those databases which cover the related _topdir_, thus reducing search time and overhead.

The raison d'ètre for this change is to allow the web interface to be used to search a large index split over several databases, e.g. _fiction_, _nonfiction_ and _audio_. This in turn is used in the _recoll engine_ for the _Searx_ meta-search engine, see .

This is a further development of an earlier change I submitted to Github, most of which was merged but for the extra databases.
parent 503999a5
......@@ -107,6 +107,27 @@ def normalise_filename(fn):
out += "_"
return out
def get_topdirs(confdir):
rclconf = rclconfig.RclConfig(confdir)
return rclconf.getConfParam('topdirs')
# get database directory from recoll.conf, defaults to
# confdir/xapiandb
def get_dbdir(confdir):
rclconf = rclconfig.RclConfig(confdir)
dbdir = rclconf.getConfParam('dbdir')
cachedir = rclconf.getConfParam('cachedir')
basename = 'xapiandb'
dirname = confdir
if cachedir:
dirname = cachedir
if dbdir:
basename = dbdir
if os.path.isabs(dbdir):
dirname = ''
# recoll API expects bytes, not strings
return bytes(os.path.normpath(os.path.join(dirname, basename)),'utf-8')
#{{{ get_config
def get_config():
......@@ -116,12 +137,21 @@ def get_config():
rclconf = rclconfig.RclConfig(envdir)
config['confdir'] = rclconf.getConfDir()
extradbs = bottle.request.environ.get('RECOLL_EXTRADBS')
extraconfdirs = os.environ['RECOLL_EXTRACONFDIRS']
if extradbs:
config['extradbs'] = shlex.split(extradbs)
config['extradbs'] = None
config['dirs'] = [os.path.expanduser(d) for d in
config['dirs'] = dict.fromkeys([os.path.expanduser(d) for d in
shlex.split(rclconf.getConfParam('topdirs'))], config['confdir'])
# add topdirs from extra config dirs
if extraconfdirs:
config['extraconfdirs'] = shlex.split(extraconfdirs)
for e in config['extraconfdirs']:
config['dirs'].update(dict.fromkeys([os.path.expanduser(d) for d in
config['extraconfdirs'] = None
config['stemlang'] = rclconf.getConfParam('indexstemminglanguages')
# get config from cookies or defaults
for k, v in DEFAULTS.items():
......@@ -198,10 +228,38 @@ def query_to_recoll_string(q):
#{{{ recoll_initsearch
def recoll_initsearch(q):
config = get_config()
confdir = config['confdir']
dbs = []
""" The reason for this somewhat elaborate scheme is to keep the
set size as small as possible by searching only those databases
with matching topdirs """
if q['dir'] == '<all>':
if config['extraconfdirs']:
confdirs = []
for d,conf in config['dirs'].items():
if os.path.commonprefix([os.path.basename(d),q['dir']]) == q['dir']:
if len(confdirs) == 0:
# should not happen, using non-existing q['dir']?
abort(400, 'no matching database for topdir ' + q['dir'])
elif len(confdirs) == 1:
# only one config (most common situation)
confdir = confdirs[0]
# more than one config with matching topdir, use 'm all
confdir = confdirs[0]
if config['extradbs']:
db = recoll.connect(config['confdir'], extra_dbs = config['extradbs'])
if dbs:
db = recoll.connect(confdir,dbs)
db = recoll.connect(config['confdir'])
db = recoll.connect(confdir)
db.setAbstractParams(config['maxchars'], config['context'])
query = db.query()
query.sortby(q['sort'], q['ascending'])
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment